mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,265 @@
|
||||
import {
|
||||
ApprovalHumanDecisionRequiredError,
|
||||
ApprovalPolicyDeniedError,
|
||||
ApprovalRequestNotFoundError,
|
||||
ApprovalSelfDecisionError,
|
||||
approvalRequestEffectiveStatus,
|
||||
assertApprovalMutationId,
|
||||
assertApprovalReasonCode,
|
||||
assertApprovalRequestId,
|
||||
assertApprovalRequestVersion,
|
||||
assertApprovalTimestamp,
|
||||
normalizeApprovalActionBinding,
|
||||
normalizeApprovalRequestRecord,
|
||||
normalizeApprovalPolicyFence,
|
||||
sameApprovalSubject,
|
||||
type ApprovalActionBinding,
|
||||
type ApprovalDecision,
|
||||
type ApprovalRequestEffectiveStatus,
|
||||
type ApprovalRequestRecord,
|
||||
type ApprovedActionDispatchRecord,
|
||||
type ApprovalRisk,
|
||||
} from '../domain/approvalRequest';
|
||||
import {
|
||||
assertProjectPolicyProjectId,
|
||||
normalizePolicySubject,
|
||||
type PolicySubject,
|
||||
} from '../domain/projectPolicy';
|
||||
import type { ApprovalRequestRepository } from '../ports/approvalRequestRepository';
|
||||
import type { ProjectPolicyEngine } from './projectPolicyEngine';
|
||||
|
||||
export interface CreateApprovalRequestInput {
|
||||
id: string;
|
||||
projectId: string;
|
||||
action: ApprovalActionBinding;
|
||||
risk: ApprovalRisk;
|
||||
requestedBy: PolicySubject;
|
||||
requestedAtMs: number;
|
||||
expiresAtMs: number;
|
||||
}
|
||||
|
||||
export interface DecideApprovalRequestInput {
|
||||
requestId: string;
|
||||
expectedVersion: number;
|
||||
decisionId: string;
|
||||
decision: ApprovalDecision;
|
||||
reasonCode: string;
|
||||
decidedBy: PolicySubject;
|
||||
decidedAtMs: number;
|
||||
}
|
||||
|
||||
export interface ConsumeApprovalRequestInput {
|
||||
requestId: string;
|
||||
expectedVersion: number;
|
||||
consumptionId: string;
|
||||
dispatchId: string;
|
||||
action: ApprovalActionBinding;
|
||||
requestedBy: PolicySubject;
|
||||
consumedBy: PolicySubject;
|
||||
consumedAtMs: number;
|
||||
}
|
||||
|
||||
export interface ApprovalRequestView {
|
||||
request: Readonly<ApprovalRequestRecord>;
|
||||
effectiveStatus: ApprovalRequestEffectiveStatus;
|
||||
}
|
||||
|
||||
function assertExactKeys(
|
||||
name: string,
|
||||
value: object,
|
||||
expected: readonly string[],
|
||||
): void {
|
||||
const keys = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
if (
|
||||
keys.length !== canonical.length ||
|
||||
keys.some((key, index) => key !== canonical[index])
|
||||
) {
|
||||
throw new TypeError(`${name} shape is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertInput(name: string, value: unknown): asserts value is object {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new TypeError(`${name} must be an object`);
|
||||
}
|
||||
}
|
||||
|
||||
export class ApprovalRequestService {
|
||||
constructor(
|
||||
private readonly repository: ApprovalRequestRepository,
|
||||
private readonly policy: ProjectPolicyEngine,
|
||||
) {}
|
||||
|
||||
async create(
|
||||
input: CreateApprovalRequestInput,
|
||||
): Promise<Readonly<ApprovalRequestRecord>> {
|
||||
assertInput('Approval create input', input);
|
||||
assertExactKeys('Approval create input', input, [
|
||||
'id',
|
||||
'projectId',
|
||||
'action',
|
||||
'risk',
|
||||
'requestedBy',
|
||||
'requestedAtMs',
|
||||
'expiresAtMs',
|
||||
]);
|
||||
const action = normalizeApprovalActionBinding(input.action);
|
||||
const requestedBy = normalizePolicySubject(input.requestedBy);
|
||||
const request = normalizeApprovalRequestRecord({
|
||||
id: input.id,
|
||||
projectId: input.projectId,
|
||||
version: 1,
|
||||
state: 'pending',
|
||||
action,
|
||||
risk: input.risk,
|
||||
requestedBy,
|
||||
requestedAtMs: input.requestedAtMs,
|
||||
expiresAtMs: input.expiresAtMs,
|
||||
decisionId: null,
|
||||
decision: null,
|
||||
decisionReasonCode: null,
|
||||
decidedBy: null,
|
||||
decidedAtMs: null,
|
||||
consumptionId: null,
|
||||
dispatchId: null,
|
||||
consumedBy: null,
|
||||
consumedAtMs: null,
|
||||
});
|
||||
const authorization = await this.policy.decideWithFence({
|
||||
projectId: request.projectId,
|
||||
subject: requestedBy,
|
||||
permission: action.permission,
|
||||
});
|
||||
if (
|
||||
authorization.decision.effect !== 'require_approval' ||
|
||||
!authorization.fence
|
||||
) {
|
||||
throw new ApprovalPolicyDeniedError();
|
||||
}
|
||||
const result = await this.repository.create({
|
||||
request,
|
||||
authorizationFence: normalizeApprovalPolicyFence(authorization.fence),
|
||||
});
|
||||
return result.request;
|
||||
}
|
||||
|
||||
async decide(
|
||||
input: DecideApprovalRequestInput,
|
||||
): Promise<Readonly<ApprovalRequestRecord>> {
|
||||
assertInput('Approval decision input', input);
|
||||
assertExactKeys('Approval decision input', input, [
|
||||
'requestId',
|
||||
'expectedVersion',
|
||||
'decisionId',
|
||||
'decision',
|
||||
'reasonCode',
|
||||
'decidedBy',
|
||||
'decidedAtMs',
|
||||
]);
|
||||
assertApprovalRequestId(input.requestId);
|
||||
assertApprovalRequestVersion(input.expectedVersion);
|
||||
assertApprovalMutationId(input.decisionId);
|
||||
assertApprovalReasonCode(input.reasonCode);
|
||||
assertApprovalTimestamp('decidedAtMs', input.decidedAtMs);
|
||||
const decidedBy = normalizePolicySubject(input.decidedBy);
|
||||
if (decidedBy.type !== 'user') {
|
||||
throw new ApprovalHumanDecisionRequiredError();
|
||||
}
|
||||
const existing = await this.repository.findById(input.requestId);
|
||||
if (!existing) throw new ApprovalRequestNotFoundError();
|
||||
const request = normalizeApprovalRequestRecord(existing);
|
||||
if (sameApprovalSubject(request.requestedBy, decidedBy)) {
|
||||
throw new ApprovalSelfDecisionError();
|
||||
}
|
||||
const authorization = await this.policy.decideWithFence({
|
||||
projectId: request.projectId,
|
||||
subject: decidedBy,
|
||||
permission: 'approval.decide',
|
||||
});
|
||||
if (authorization.decision.effect !== 'allow' || !authorization.fence) {
|
||||
throw new ApprovalPolicyDeniedError();
|
||||
}
|
||||
const result = await this.repository.decide({
|
||||
requestId: input.requestId,
|
||||
expectedVersion: input.expectedVersion,
|
||||
decisionId: input.decisionId,
|
||||
decision: input.decision,
|
||||
reasonCode: input.reasonCode,
|
||||
decidedBy,
|
||||
decidedAtMs: input.decidedAtMs,
|
||||
authorizationFence: normalizeApprovalPolicyFence(authorization.fence),
|
||||
});
|
||||
return result.request;
|
||||
}
|
||||
|
||||
async consume(input: ConsumeApprovalRequestInput): Promise<{
|
||||
request: Readonly<ApprovalRequestRecord>;
|
||||
dispatch: Readonly<ApprovedActionDispatchRecord>;
|
||||
}> {
|
||||
assertInput('Approval consumption input', input);
|
||||
assertExactKeys('Approval consumption input', input, [
|
||||
'requestId',
|
||||
'expectedVersion',
|
||||
'consumptionId',
|
||||
'dispatchId',
|
||||
'action',
|
||||
'requestedBy',
|
||||
'consumedBy',
|
||||
'consumedAtMs',
|
||||
]);
|
||||
assertApprovalRequestId(input.requestId);
|
||||
assertApprovalRequestVersion(input.expectedVersion);
|
||||
assertApprovalMutationId(input.consumptionId);
|
||||
assertApprovalMutationId(input.dispatchId);
|
||||
assertApprovalTimestamp('consumedAtMs', input.consumedAtMs);
|
||||
const action = normalizeApprovalActionBinding(input.action);
|
||||
const requestedBy = normalizePolicySubject(input.requestedBy);
|
||||
const consumedBy = normalizePolicySubject(input.consumedBy);
|
||||
if (consumedBy.type !== 'system' && consumedBy.type !== 'worker') {
|
||||
throw new ApprovalPolicyDeniedError();
|
||||
}
|
||||
const existing = await this.repository.findById(input.requestId);
|
||||
if (!existing) throw new ApprovalRequestNotFoundError();
|
||||
const request = normalizeApprovalRequestRecord(existing);
|
||||
const authorization = await this.policy.decideWithFence({
|
||||
projectId: request.projectId,
|
||||
subject: requestedBy,
|
||||
permission: action.permission,
|
||||
});
|
||||
if (
|
||||
(authorization.decision.effect !== 'allow' &&
|
||||
authorization.decision.effect !== 'require_approval') ||
|
||||
!authorization.fence
|
||||
) {
|
||||
throw new ApprovalPolicyDeniedError();
|
||||
}
|
||||
const result = await this.repository.consume({
|
||||
requestId: input.requestId,
|
||||
expectedVersion: input.expectedVersion,
|
||||
consumptionId: input.consumptionId,
|
||||
dispatchId: input.dispatchId,
|
||||
action,
|
||||
requestedBy,
|
||||
consumedBy,
|
||||
consumedAtMs: input.consumedAtMs,
|
||||
authorizationFence: normalizeApprovalPolicyFence(authorization.fence),
|
||||
});
|
||||
return Object.freeze({
|
||||
request: result.request,
|
||||
dispatch: result.dispatch,
|
||||
});
|
||||
}
|
||||
|
||||
async get(requestId: string, nowMs: number): Promise<ApprovalRequestView> {
|
||||
assertApprovalRequestId(requestId);
|
||||
assertApprovalTimestamp('nowMs', nowMs);
|
||||
const request = await this.repository.findById(requestId);
|
||||
if (!request) throw new ApprovalRequestNotFoundError();
|
||||
const normalized = normalizeApprovalRequestRecord(request);
|
||||
return Object.freeze({
|
||||
request: normalized,
|
||||
effectiveStatus: approvalRequestEffectiveStatus(normalized, nowMs),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import {
|
||||
assertApprovedActionLeaseDuration,
|
||||
assertApprovedActionLeaseIdentity,
|
||||
assertApprovedActionPageSize,
|
||||
assertApprovedActionResultCode,
|
||||
type ApprovedActionDispatchCursor,
|
||||
type ApprovedActionDispatchExecutionRecord,
|
||||
} from '../domain/approvedActionDispatchExecution';
|
||||
import type { ApprovedActionHandler } from '../ports/approvedActionHandler';
|
||||
import type { ApprovedActionDispatchRepository } from '../ports/approvedActionDispatchRepository';
|
||||
|
||||
const DEFAULT_LEASE_DURATION_MS = 30_000;
|
||||
const DEFAULT_RETRY_BASE_MS = 1_000;
|
||||
const DEFAULT_RETRY_MAX_MS = 60_000;
|
||||
|
||||
export interface ApprovedActionDispatcherOptions {
|
||||
owner: string;
|
||||
leaseDurationMs?: number;
|
||||
retryBaseMs?: number;
|
||||
retryMaxMs?: number;
|
||||
clock?: () => number;
|
||||
createId?: () => string;
|
||||
}
|
||||
|
||||
export interface ApprovedActionDispatchBatchSummary {
|
||||
scanned: number;
|
||||
claimed: number;
|
||||
started: number;
|
||||
succeeded: number;
|
||||
failed: number;
|
||||
blocked: number;
|
||||
retrying: number;
|
||||
deferred: number;
|
||||
recoveryRequired: number;
|
||||
alreadyTerminal: number;
|
||||
unavailable: number;
|
||||
truncated: boolean;
|
||||
nextCursor?: Readonly<ApprovedActionDispatchCursor>;
|
||||
}
|
||||
|
||||
function assertPositiveInteger(name: string, value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 1) {
|
||||
throw new RangeError(`${name} must be a positive safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
function exactKeys(value: object, expected: readonly string[]): boolean {
|
||||
const keys = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
return (
|
||||
keys.length === canonical.length &&
|
||||
keys.every((key, index) => key === canonical[index])
|
||||
);
|
||||
}
|
||||
|
||||
export class ApprovedActionDispatcher {
|
||||
private readonly handlers = new Map<string, ApprovedActionHandler>();
|
||||
private readonly owner: string;
|
||||
private readonly leaseDurationMs: number;
|
||||
private readonly retryBaseMs: number;
|
||||
private readonly retryMaxMs: number;
|
||||
private readonly clock: () => number;
|
||||
private readonly createId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly repository: ApprovedActionDispatchRepository,
|
||||
handlers: readonly ApprovedActionHandler[],
|
||||
options: ApprovedActionDispatcherOptions,
|
||||
) {
|
||||
assertApprovedActionLeaseIdentity(options.owner);
|
||||
this.owner = options.owner;
|
||||
this.leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
|
||||
this.retryBaseMs = options.retryBaseMs ?? DEFAULT_RETRY_BASE_MS;
|
||||
this.retryMaxMs = options.retryMaxMs ?? DEFAULT_RETRY_MAX_MS;
|
||||
this.clock = options.clock ?? Date.now;
|
||||
this.createId = options.createId ?? uuidV7;
|
||||
assertApprovedActionLeaseDuration(this.leaseDurationMs);
|
||||
assertPositiveInteger('retryBaseMs', this.retryBaseMs);
|
||||
assertPositiveInteger('retryMaxMs', this.retryMaxMs);
|
||||
if (this.retryMaxMs < this.retryBaseMs) {
|
||||
throw new RangeError(
|
||||
'retryMaxMs must be greater than or equal to retryBaseMs',
|
||||
);
|
||||
}
|
||||
for (const handler of handlers) {
|
||||
if (
|
||||
!handler ||
|
||||
typeof handler !== 'object' ||
|
||||
typeof handler.actionType !== 'string' ||
|
||||
handler.actionType.length < 1 ||
|
||||
handler.actionType.length > 64 ||
|
||||
typeof handler.inspect !== 'function' ||
|
||||
typeof handler.execute !== 'function'
|
||||
) {
|
||||
throw new TypeError('Approved action handler is invalid');
|
||||
}
|
||||
if (this.handlers.has(handler.actionType)) {
|
||||
throw new TypeError(
|
||||
`Duplicate approved action handler: ${handler.actionType}`,
|
||||
);
|
||||
}
|
||||
this.handlers.set(handler.actionType, handler);
|
||||
}
|
||||
}
|
||||
|
||||
async dispatchBatch(
|
||||
options: { cursor?: ApprovedActionDispatchCursor; limit?: number } = {},
|
||||
): Promise<Readonly<ApprovedActionDispatchBatchSummary>> {
|
||||
if (!options || typeof options !== 'object' || Array.isArray(options)) {
|
||||
throw new TypeError('Approved action dispatch options must be an object');
|
||||
}
|
||||
if (
|
||||
!exactKeys(
|
||||
options,
|
||||
options.cursor === undefined && options.limit === undefined
|
||||
? []
|
||||
: [
|
||||
...(options.cursor === undefined ? [] : ['cursor']),
|
||||
...(options.limit === undefined ? [] : ['limit']),
|
||||
],
|
||||
)
|
||||
) {
|
||||
throw new TypeError('Approved action dispatch options shape is invalid');
|
||||
}
|
||||
const limit = options.limit ?? 16;
|
||||
assertApprovedActionPageSize(limit);
|
||||
const observedAtMs = this.now();
|
||||
const page = await this.repository.listDue({
|
||||
nowMs: observedAtMs,
|
||||
limit,
|
||||
...(options.cursor ? { cursor: options.cursor } : {}),
|
||||
});
|
||||
const summary: ApprovedActionDispatchBatchSummary = {
|
||||
scanned: page.dispatches.length,
|
||||
claimed: 0,
|
||||
started: 0,
|
||||
succeeded: 0,
|
||||
failed: 0,
|
||||
blocked: 0,
|
||||
retrying: 0,
|
||||
deferred: 0,
|
||||
recoveryRequired: 0,
|
||||
alreadyTerminal: 0,
|
||||
unavailable: 0,
|
||||
truncated: page.truncated,
|
||||
...(page.nextCursor ? { nextCursor: page.nextCursor } : {}),
|
||||
};
|
||||
for (const candidate of page.dispatches) {
|
||||
await this.dispatchOne(candidate.dispatch.id, summary);
|
||||
}
|
||||
return Object.freeze(summary);
|
||||
}
|
||||
|
||||
private async dispatchOne(
|
||||
dispatchId: string,
|
||||
summary: ApprovedActionDispatchBatchSummary,
|
||||
): Promise<void> {
|
||||
const claimedAtMs = this.now();
|
||||
let claim;
|
||||
try {
|
||||
claim = await this.repository.claim({
|
||||
dispatchId,
|
||||
owner: this.owner,
|
||||
leaseToken: this.createId(),
|
||||
nowMs: claimedAtMs,
|
||||
leaseDurationMs: this.leaseDurationMs,
|
||||
});
|
||||
} catch {
|
||||
summary.unavailable += 1;
|
||||
return;
|
||||
}
|
||||
if (claim.status === 'not_found') {
|
||||
summary.unavailable += 1;
|
||||
return;
|
||||
}
|
||||
if (claim.status !== 'claimed') {
|
||||
if (claim.status === 'recovery_required') summary.recoveryRequired += 1;
|
||||
else if (
|
||||
claim.status === 'succeeded' ||
|
||||
claim.status === 'failed' ||
|
||||
claim.status === 'blocked'
|
||||
) {
|
||||
summary.alreadyTerminal += 1;
|
||||
} else summary.deferred += 1;
|
||||
return;
|
||||
}
|
||||
summary.claimed += 1;
|
||||
const handler = this.handlers.get(
|
||||
claim.snapshot.dispatch.action.actionType,
|
||||
);
|
||||
if (!handler) {
|
||||
await this.releasePreflight(
|
||||
claim.snapshot.execution,
|
||||
'handler_unavailable',
|
||||
true,
|
||||
summary,
|
||||
);
|
||||
return;
|
||||
}
|
||||
let inspection;
|
||||
try {
|
||||
inspection = await handler.inspect(claim.snapshot.dispatch);
|
||||
this.assertInspection(inspection);
|
||||
} catch {
|
||||
await this.releasePreflight(
|
||||
claim.snapshot.execution,
|
||||
'handler_inspection_failed',
|
||||
true,
|
||||
summary,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (inspection.status !== 'ready') {
|
||||
await this.releasePreflight(
|
||||
claim.snapshot.execution,
|
||||
inspection.resultCode,
|
||||
inspection.status === 'retry',
|
||||
summary,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
inspection.actionDigest !== claim.snapshot.dispatch.action.actionDigest
|
||||
) {
|
||||
await this.releasePreflight(
|
||||
claim.snapshot.execution,
|
||||
'action_digest_mismatch',
|
||||
false,
|
||||
summary,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let started;
|
||||
try {
|
||||
const startedAtMs = this.now();
|
||||
started = await this.repository.start({
|
||||
dispatchId,
|
||||
approvalRequestId: claim.snapshot.dispatch.approvalRequestId,
|
||||
actionDigest: inspection.actionDigest,
|
||||
owner: this.owner,
|
||||
leaseToken: claim.snapshot.execution.leaseToken!,
|
||||
expectedVersion: claim.snapshot.execution.version,
|
||||
startedAtMs,
|
||||
});
|
||||
summary.started += 1;
|
||||
} catch {
|
||||
summary.unavailable += 1;
|
||||
return;
|
||||
}
|
||||
|
||||
let outcome: 'succeeded' | 'failed' | 'indeterminate';
|
||||
let resultCode: string;
|
||||
try {
|
||||
const result = await handler.execute(
|
||||
Object.freeze({
|
||||
dispatch: started.dispatch,
|
||||
execution: started.execution,
|
||||
idempotencyKey: started.dispatch.id,
|
||||
fence: Object.freeze({
|
||||
owner: this.owner,
|
||||
leaseToken: started.execution.leaseToken!,
|
||||
version: started.execution.version,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
this.assertExecutionResult(result);
|
||||
outcome = result.outcome;
|
||||
resultCode = result.resultCode;
|
||||
} catch {
|
||||
outcome = 'indeterminate';
|
||||
resultCode = 'handler_failed_after_start';
|
||||
}
|
||||
try {
|
||||
const completed = await this.repository.complete({
|
||||
dispatchId,
|
||||
owner: this.owner,
|
||||
leaseToken: started.execution.leaseToken!,
|
||||
expectedVersion: started.execution.version,
|
||||
resultMutationId: this.createId(),
|
||||
outcome,
|
||||
resultCode,
|
||||
completedAtMs: this.now(),
|
||||
});
|
||||
if (completed.execution.status === 'succeeded') summary.succeeded += 1;
|
||||
else if (completed.execution.status === 'failed') summary.failed += 1;
|
||||
else summary.blocked += 1;
|
||||
} catch {
|
||||
summary.unavailable += 1;
|
||||
summary.recoveryRequired += 1;
|
||||
}
|
||||
}
|
||||
|
||||
private async releasePreflight(
|
||||
execution: Readonly<ApprovedActionDispatchExecutionRecord>,
|
||||
resultCode: string,
|
||||
retry: boolean,
|
||||
summary: ApprovedActionDispatchBatchSummary,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const atMs = this.now();
|
||||
const released = await this.repository.releaseBeforeStart({
|
||||
dispatchId: execution.dispatchId,
|
||||
owner: this.owner,
|
||||
leaseToken: execution.leaseToken!,
|
||||
expectedVersion: execution.version,
|
||||
resultMutationId: this.createId(),
|
||||
resultCode,
|
||||
atMs,
|
||||
...(retry
|
||||
? { retryAtMs: this.nextRetryAt(atMs, execution.attemptCount) }
|
||||
: {}),
|
||||
});
|
||||
if (released.execution.status === 'retry_wait') summary.retrying += 1;
|
||||
else summary.blocked += 1;
|
||||
} catch {
|
||||
summary.unavailable += 1;
|
||||
}
|
||||
}
|
||||
|
||||
private assertInspection(
|
||||
value: unknown,
|
||||
): asserts value is Awaited<ReturnType<ApprovedActionHandler['inspect']>> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new TypeError('Approved action inspection is invalid');
|
||||
}
|
||||
if (
|
||||
'status' in value &&
|
||||
value.status === 'ready' &&
|
||||
exactKeys(value, ['status', 'actionDigest']) &&
|
||||
'actionDigest' in value &&
|
||||
typeof value.actionDigest === 'string' &&
|
||||
/^[0-9a-f]{64}$/.test(value.actionDigest)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
'status' in value &&
|
||||
(value.status === 'retry' || value.status === 'blocked') &&
|
||||
exactKeys(value, ['status', 'resultCode']) &&
|
||||
'resultCode' in value &&
|
||||
typeof value.resultCode === 'string'
|
||||
) {
|
||||
assertApprovedActionResultCode(value.resultCode);
|
||||
return;
|
||||
}
|
||||
throw new TypeError('Approved action inspection is invalid');
|
||||
}
|
||||
|
||||
private assertExecutionResult(
|
||||
value: unknown,
|
||||
): asserts value is Awaited<ReturnType<ApprovedActionHandler['execute']>> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
!exactKeys(value, ['outcome', 'resultCode']) ||
|
||||
!('outcome' in value) ||
|
||||
!['succeeded', 'failed', 'indeterminate'].includes(
|
||||
value.outcome as string,
|
||||
) ||
|
||||
!('resultCode' in value) ||
|
||||
typeof value.resultCode !== 'string'
|
||||
) {
|
||||
throw new TypeError('Approved action execution result is invalid');
|
||||
}
|
||||
assertApprovedActionResultCode(value.resultCode);
|
||||
}
|
||||
|
||||
private nextRetryAt(atMs: number, attemptCount: number): number {
|
||||
const exponent = Math.max(0, Math.min(attemptCount - 1, 30));
|
||||
const delay = Math.min(this.retryMaxMs, this.retryBaseMs * 2 ** exponent);
|
||||
return Math.min(Number.MAX_SAFE_INTEGER, atMs + delay);
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new RangeError('clock must return a non-negative safe integer');
|
||||
}
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
import {
|
||||
assertAuthenticatedPrincipalActive,
|
||||
normalizeAuthenticatedPrincipal,
|
||||
type AuthenticatedPrincipal,
|
||||
} from '../domain/authenticatedPrincipal';
|
||||
import {
|
||||
APPROVED_ACTION_RECOVERY_STRONG_ASSURANCES,
|
||||
ApprovedActionRecoveryAuthorizationDeniedError,
|
||||
ApprovedActionRecoveryHumanRequiredError,
|
||||
ApprovedActionRecoveryNotFoundError,
|
||||
ApprovedActionRecoveryStrongAuthenticationRequiredError,
|
||||
MAX_APPROVED_ACTION_RECOVERY_AUTH_AGE_MS,
|
||||
createApprovedActionRecoveryAuthorizationFact,
|
||||
} from '../domain/approvedActionRecoveryAuthorization';
|
||||
import {
|
||||
assertApprovedActionEvidenceDigest,
|
||||
type ApprovedActionRecoveryDecision,
|
||||
} from '../domain/approvedActionRecovery';
|
||||
import { assertApprovedActionResultCode } from '../domain/approvedActionDispatchExecution';
|
||||
import { assertApprovalMutationId } from '../domain/approvalRequest';
|
||||
import type {
|
||||
ApprovedActionRecoveryRepository,
|
||||
ResolveApprovedActionRecoveryResult,
|
||||
} from '../ports/approvedActionRecoveryRepository';
|
||||
import type { ProjectPolicyEngine } from './projectPolicyEngine';
|
||||
|
||||
export interface ManuallyResolveApprovedActionRecoveryInput {
|
||||
dispatchId: string;
|
||||
expectedExecutionVersion: number;
|
||||
expectedRecoveryVersion: number;
|
||||
mutationId: string;
|
||||
decision: ApprovedActionRecoveryDecision;
|
||||
evidenceDigest?: string;
|
||||
reasonCode: string;
|
||||
principal: AuthenticatedPrincipal;
|
||||
}
|
||||
|
||||
function exactKeys(value: object, expected: readonly string[]): void {
|
||||
const keys = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
if (
|
||||
keys.length !== canonical.length ||
|
||||
keys.some((key, index) => key !== canonical[index])
|
||||
) {
|
||||
throw new TypeError('Manual recovery input shape is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function assertVersion(name: string, value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0 || value > 2_147_483_647) {
|
||||
throw new RangeError(`${name} is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
export class ApprovedActionManualRecoveryService {
|
||||
constructor(
|
||||
private readonly repository: ApprovedActionRecoveryRepository,
|
||||
private readonly policy: ProjectPolicyEngine,
|
||||
private readonly clock: () => number = Date.now,
|
||||
) {}
|
||||
|
||||
async resolve(
|
||||
input: ManuallyResolveApprovedActionRecoveryInput,
|
||||
): Promise<ResolveApprovedActionRecoveryResult> {
|
||||
if (!input || typeof input !== 'object' || Array.isArray(input)) {
|
||||
throw new TypeError('Manual recovery input must be an object');
|
||||
}
|
||||
exactKeys(input, [
|
||||
'dispatchId',
|
||||
'expectedExecutionVersion',
|
||||
'expectedRecoveryVersion',
|
||||
'mutationId',
|
||||
'decision',
|
||||
...(input.evidenceDigest === undefined ? [] : ['evidenceDigest']),
|
||||
'reasonCode',
|
||||
'principal',
|
||||
]);
|
||||
assertApprovalMutationId(input.dispatchId);
|
||||
assertVersion('expectedExecutionVersion', input.expectedExecutionVersion);
|
||||
assertVersion('expectedRecoveryVersion', input.expectedRecoveryVersion);
|
||||
assertApprovalMutationId(input.mutationId);
|
||||
if (
|
||||
!['confirm_succeeded', 'confirm_failed', 'abandon_unknown'].includes(
|
||||
input.decision,
|
||||
)
|
||||
) {
|
||||
throw new TypeError('Manual recovery decision is invalid');
|
||||
}
|
||||
if (input.evidenceDigest !== undefined) {
|
||||
assertApprovedActionEvidenceDigest(input.evidenceDigest);
|
||||
}
|
||||
assertApprovedActionResultCode(input.reasonCode);
|
||||
const resolvedAtMs = this.now();
|
||||
const principal = normalizeAuthenticatedPrincipal(input.principal);
|
||||
assertAuthenticatedPrincipalActive(principal, resolvedAtMs);
|
||||
if (principal.subject.type !== 'user') {
|
||||
throw new ApprovedActionRecoveryHumanRequiredError();
|
||||
}
|
||||
if (
|
||||
!APPROVED_ACTION_RECOVERY_STRONG_ASSURANCES.includes(
|
||||
principal.assurance as (typeof APPROVED_ACTION_RECOVERY_STRONG_ASSURANCES)[number],
|
||||
) ||
|
||||
resolvedAtMs - principal.authenticatedAtMs >
|
||||
MAX_APPROVED_ACTION_RECOVERY_AUTH_AGE_MS
|
||||
) {
|
||||
throw new ApprovedActionRecoveryStrongAuthenticationRequiredError();
|
||||
}
|
||||
const snapshot = await this.repository.findById(input.dispatchId);
|
||||
if (!snapshot) throw new ApprovedActionRecoveryNotFoundError();
|
||||
const authorization = await this.policy.decideWithFence({
|
||||
projectId: snapshot.action.dispatch.projectId,
|
||||
subject: principal.subject,
|
||||
permission: 'approval.recover',
|
||||
});
|
||||
if (
|
||||
authorization.decision.effect !== 'allow' ||
|
||||
!authorization.fence ||
|
||||
authorization.fence.bindingVersion === null
|
||||
) {
|
||||
throw new ApprovedActionRecoveryAuthorizationDeniedError();
|
||||
}
|
||||
const authorizationFact = createApprovedActionRecoveryAuthorizationFact({
|
||||
dispatchId: input.dispatchId,
|
||||
projectId: snapshot.action.dispatch.projectId,
|
||||
mutationId: input.mutationId,
|
||||
resolvedBy: principal.subject,
|
||||
authenticationId: principal.authenticationId,
|
||||
assurance:
|
||||
principal.assurance as (typeof APPROVED_ACTION_RECOVERY_STRONG_ASSURANCES)[number],
|
||||
authenticatedAtMs: principal.authenticatedAtMs,
|
||||
projectVersion: authorization.fence.projectVersion,
|
||||
bindingVersion: authorization.fence.bindingVersion,
|
||||
authorizedAtMs: resolvedAtMs,
|
||||
});
|
||||
return this.repository.resolve({
|
||||
dispatchId: input.dispatchId,
|
||||
expectedExecutionVersion: input.expectedExecutionVersion,
|
||||
expectedRecoveryVersion: input.expectedRecoveryVersion,
|
||||
mutationId: input.mutationId,
|
||||
source: 'human',
|
||||
decision: input.decision,
|
||||
...(input.evidenceDigest === undefined
|
||||
? {}
|
||||
: { evidenceDigest: input.evidenceDigest }),
|
||||
reasonCode: input.reasonCode,
|
||||
resolvedBy: principal.subject,
|
||||
resolvedAtMs,
|
||||
authorizationFact,
|
||||
});
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new RangeError('clock must return a non-negative safe integer');
|
||||
}
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import {
|
||||
assertApprovedActionLeaseIdentity,
|
||||
assertApprovedActionResultCode,
|
||||
} from '../domain/approvedActionDispatchExecution';
|
||||
import {
|
||||
assertApprovedActionEvidenceDigest,
|
||||
assertApprovedActionRecoveryLeaseDuration,
|
||||
assertApprovedActionRecoveryPageSize,
|
||||
type ApprovedActionRecoveryCursor,
|
||||
} from '../domain/approvedActionRecovery';
|
||||
import type {
|
||||
ApprovedActionRecoveryEvidence,
|
||||
ApprovedActionRecoveryEvidenceProvider,
|
||||
} from '../ports/approvedActionRecoveryEvidenceProvider';
|
||||
import type { ApprovedActionRecoveryRepository } from '../ports/approvedActionRecoveryRepository';
|
||||
|
||||
const DEFAULT_LEASE_DURATION_MS = 30_000;
|
||||
const DEFAULT_RETRY_BASE_MS = 5_000;
|
||||
const DEFAULT_RETRY_MAX_MS = 5 * 60_000;
|
||||
|
||||
export interface ApprovedActionRecoveryReconcilerOptions {
|
||||
owner: string;
|
||||
leaseDurationMs?: number;
|
||||
retryBaseMs?: number;
|
||||
retryMaxMs?: number;
|
||||
clock?: () => number;
|
||||
createId?: () => string;
|
||||
}
|
||||
|
||||
export interface ApprovedActionRecoveryBatchSummary {
|
||||
scanned: number;
|
||||
claimed: number;
|
||||
verifiedSucceeded: number;
|
||||
verifiedFailed: number;
|
||||
deferred: number;
|
||||
manualRequired: number;
|
||||
executionActive: number;
|
||||
alreadyResolved: number;
|
||||
unavailable: number;
|
||||
truncated: boolean;
|
||||
nextCursor?: Readonly<ApprovedActionRecoveryCursor>;
|
||||
}
|
||||
|
||||
function assertPositiveInteger(name: string, value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 1) {
|
||||
throw new RangeError(`${name} must be a positive safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
function exactKeys(value: object, expected: readonly string[]): boolean {
|
||||
const keys = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
return (
|
||||
keys.length === canonical.length &&
|
||||
keys.every((key, index) => key === canonical[index])
|
||||
);
|
||||
}
|
||||
|
||||
export class ApprovedActionRecoveryReconciler {
|
||||
private readonly providers = new Map<
|
||||
string,
|
||||
ApprovedActionRecoveryEvidenceProvider
|
||||
>();
|
||||
private readonly owner: string;
|
||||
private readonly leaseDurationMs: number;
|
||||
private readonly retryBaseMs: number;
|
||||
private readonly retryMaxMs: number;
|
||||
private readonly clock: () => number;
|
||||
private readonly createId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly repository: ApprovedActionRecoveryRepository,
|
||||
providers: readonly ApprovedActionRecoveryEvidenceProvider[],
|
||||
options: ApprovedActionRecoveryReconcilerOptions,
|
||||
) {
|
||||
assertApprovedActionLeaseIdentity(options.owner);
|
||||
this.owner = options.owner;
|
||||
this.leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
|
||||
this.retryBaseMs = options.retryBaseMs ?? DEFAULT_RETRY_BASE_MS;
|
||||
this.retryMaxMs = options.retryMaxMs ?? DEFAULT_RETRY_MAX_MS;
|
||||
this.clock = options.clock ?? Date.now;
|
||||
this.createId = options.createId ?? uuidV7;
|
||||
assertApprovedActionRecoveryLeaseDuration(this.leaseDurationMs);
|
||||
assertPositiveInteger('retryBaseMs', this.retryBaseMs);
|
||||
assertPositiveInteger('retryMaxMs', this.retryMaxMs);
|
||||
if (this.retryMaxMs < this.retryBaseMs) {
|
||||
throw new RangeError(
|
||||
'retryMaxMs must be greater than or equal to retryBaseMs',
|
||||
);
|
||||
}
|
||||
for (const provider of providers) {
|
||||
if (
|
||||
!provider ||
|
||||
typeof provider !== 'object' ||
|
||||
typeof provider.actionType !== 'string' ||
|
||||
provider.actionType.length < 1 ||
|
||||
provider.actionType.length > 64 ||
|
||||
!['automatic', 'manual_only'].includes(provider.capability) ||
|
||||
typeof provider.inspect !== 'function'
|
||||
) {
|
||||
throw new TypeError('Approved action recovery provider is invalid');
|
||||
}
|
||||
if (this.providers.has(provider.actionType)) {
|
||||
throw new TypeError(
|
||||
`Duplicate approved action recovery provider: ${provider.actionType}`,
|
||||
);
|
||||
}
|
||||
this.providers.set(provider.actionType, provider);
|
||||
}
|
||||
}
|
||||
|
||||
async reconcileBatch(
|
||||
options: { cursor?: ApprovedActionRecoveryCursor; limit?: number } = {},
|
||||
): Promise<Readonly<ApprovedActionRecoveryBatchSummary>> {
|
||||
if (!options || typeof options !== 'object' || Array.isArray(options)) {
|
||||
throw new TypeError('Approved action recovery options must be an object');
|
||||
}
|
||||
const expectedKeys = [
|
||||
...(options.cursor === undefined ? [] : ['cursor']),
|
||||
...(options.limit === undefined ? [] : ['limit']),
|
||||
];
|
||||
if (!exactKeys(options, expectedKeys)) {
|
||||
throw new TypeError('Approved action recovery options shape is invalid');
|
||||
}
|
||||
const limit = options.limit ?? 16;
|
||||
assertApprovedActionRecoveryPageSize(limit);
|
||||
const observedAtMs = this.now();
|
||||
const page = await this.repository.listDue({
|
||||
nowMs: observedAtMs,
|
||||
limit,
|
||||
...(options.cursor ? { cursor: options.cursor } : {}),
|
||||
});
|
||||
const summary: ApprovedActionRecoveryBatchSummary = {
|
||||
scanned: page.recoveries.length,
|
||||
claimed: 0,
|
||||
verifiedSucceeded: 0,
|
||||
verifiedFailed: 0,
|
||||
deferred: 0,
|
||||
manualRequired: 0,
|
||||
executionActive: 0,
|
||||
alreadyResolved: 0,
|
||||
unavailable: 0,
|
||||
truncated: page.truncated,
|
||||
...(page.nextCursor ? { nextCursor: page.nextCursor } : {}),
|
||||
};
|
||||
for (const candidate of page.recoveries) {
|
||||
await this.reconcileOne(candidate.action.dispatch.id, summary);
|
||||
}
|
||||
return Object.freeze(summary);
|
||||
}
|
||||
|
||||
private async reconcileOne(
|
||||
dispatchId: string,
|
||||
summary: ApprovedActionRecoveryBatchSummary,
|
||||
): Promise<void> {
|
||||
let claim;
|
||||
try {
|
||||
claim = await this.repository.claim({
|
||||
dispatchId,
|
||||
owner: this.owner,
|
||||
leaseToken: this.createId(),
|
||||
nowMs: this.now(),
|
||||
leaseDurationMs: this.leaseDurationMs,
|
||||
});
|
||||
} catch {
|
||||
summary.unavailable += 1;
|
||||
return;
|
||||
}
|
||||
if (claim.status === 'not_found') {
|
||||
summary.unavailable += 1;
|
||||
return;
|
||||
}
|
||||
if (claim.status !== 'claimed') {
|
||||
if (claim.status === 'manual_required') summary.manualRequired += 1;
|
||||
else if (claim.status === 'resolved') summary.alreadyResolved += 1;
|
||||
else if (claim.status === 'execution_active')
|
||||
summary.executionActive += 1;
|
||||
else summary.deferred += 1;
|
||||
return;
|
||||
}
|
||||
summary.claimed += 1;
|
||||
const snapshot = claim.snapshot;
|
||||
const provider = this.providers.get(
|
||||
snapshot.action.dispatch.action.actionType,
|
||||
);
|
||||
let evidence: ApprovedActionRecoveryEvidence;
|
||||
if (!provider || provider.capability === 'manual_only') {
|
||||
evidence = {
|
||||
finding: 'unsupported',
|
||||
resultCode: 'automatic_recovery_unsupported',
|
||||
};
|
||||
} else {
|
||||
try {
|
||||
evidence = await provider.inspect(
|
||||
Object.freeze({
|
||||
snapshot,
|
||||
idempotencyKey: snapshot.action.dispatch.id,
|
||||
observedAtMs: this.now(),
|
||||
}),
|
||||
);
|
||||
} catch {
|
||||
evidence = {
|
||||
finding: 'unavailable',
|
||||
resultCode: 'recovery_evidence_unavailable',
|
||||
};
|
||||
}
|
||||
try {
|
||||
this.assertEvidence(evidence);
|
||||
} catch {
|
||||
evidence = {
|
||||
finding: 'conflict',
|
||||
resultCode: 'recovery_evidence_invalid',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
evidence.finding === 'verified_succeeded' ||
|
||||
evidence.finding === 'verified_failed'
|
||||
) {
|
||||
try {
|
||||
const resolved = await this.repository.resolve({
|
||||
dispatchId,
|
||||
expectedExecutionVersion: snapshot.action.execution.version,
|
||||
expectedRecoveryVersion: snapshot.recovery.version,
|
||||
owner: this.owner,
|
||||
leaseToken: snapshot.recovery.leaseToken!,
|
||||
mutationId: this.createId(),
|
||||
source: 'automatic_evidence',
|
||||
decision:
|
||||
evidence.finding === 'verified_succeeded'
|
||||
? 'confirm_succeeded'
|
||||
: 'confirm_failed',
|
||||
evidenceDigest: evidence.evidenceDigest,
|
||||
reasonCode: evidence.resultCode,
|
||||
resolvedAtMs: this.now(),
|
||||
});
|
||||
if (resolved.status === 'not_found') {
|
||||
summary.unavailable += 1;
|
||||
} else if (resolved.status === 'already_terminal') {
|
||||
summary.alreadyResolved += 1;
|
||||
} else if (resolved.snapshot.action.execution.status === 'succeeded') {
|
||||
summary.verifiedSucceeded += 1;
|
||||
} else {
|
||||
summary.verifiedFailed += 1;
|
||||
}
|
||||
} catch {
|
||||
summary.unavailable += 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const retry = ['still_running', 'missing', 'unavailable'].includes(
|
||||
evidence.finding,
|
||||
);
|
||||
try {
|
||||
const observedAtMs = this.now();
|
||||
const recorded = await this.repository.recordFinding({
|
||||
dispatchId,
|
||||
expectedExecutionVersion: snapshot.action.execution.version,
|
||||
expectedRecoveryVersion: snapshot.recovery.version,
|
||||
owner: this.owner,
|
||||
leaseToken: snapshot.recovery.leaseToken!,
|
||||
findingMutationId: this.createId(),
|
||||
finding: evidence.finding,
|
||||
resultCode: evidence.resultCode,
|
||||
...(evidence.evidenceDigest
|
||||
? { evidenceDigest: evidence.evidenceDigest }
|
||||
: {}),
|
||||
observedAtMs,
|
||||
...(retry
|
||||
? {
|
||||
retryAtMs: this.nextRetryAt(
|
||||
observedAtMs,
|
||||
snapshot.recovery.findingCount + 1,
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
if (recorded.recovery.status === 'manual_required') {
|
||||
summary.manualRequired += 1;
|
||||
} else {
|
||||
summary.deferred += 1;
|
||||
}
|
||||
} catch {
|
||||
summary.unavailable += 1;
|
||||
}
|
||||
}
|
||||
|
||||
private assertEvidence(
|
||||
value: unknown,
|
||||
): asserts value is ApprovedActionRecoveryEvidence {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new TypeError('Approved action recovery evidence is invalid');
|
||||
}
|
||||
if (
|
||||
!('finding' in value) ||
|
||||
![
|
||||
'verified_succeeded',
|
||||
'verified_failed',
|
||||
'still_running',
|
||||
'missing',
|
||||
'conflict',
|
||||
'unsupported',
|
||||
'unavailable',
|
||||
].includes(value.finding as string) ||
|
||||
!('resultCode' in value) ||
|
||||
typeof value.resultCode !== 'string'
|
||||
) {
|
||||
throw new TypeError('Approved action recovery evidence is invalid');
|
||||
}
|
||||
assertApprovedActionResultCode(value.resultCode);
|
||||
const verified =
|
||||
value.finding === 'verified_succeeded' ||
|
||||
value.finding === 'verified_failed';
|
||||
if (
|
||||
!exactKeys(
|
||||
value,
|
||||
verified || 'evidenceDigest' in value
|
||||
? ['finding', 'resultCode', 'evidenceDigest']
|
||||
: ['finding', 'resultCode'],
|
||||
)
|
||||
) {
|
||||
throw new TypeError('Approved action recovery evidence is invalid');
|
||||
}
|
||||
if (verified && !('evidenceDigest' in value)) {
|
||||
throw new TypeError('Verified recovery evidence has no digest');
|
||||
}
|
||||
if ('evidenceDigest' in value) {
|
||||
if (typeof value.evidenceDigest !== 'string') {
|
||||
throw new TypeError('Approved action recovery evidence is invalid');
|
||||
}
|
||||
assertApprovedActionEvidenceDigest(value.evidenceDigest);
|
||||
}
|
||||
}
|
||||
|
||||
private nextRetryAt(atMs: number, findingCount: number): number {
|
||||
const exponent = Math.max(0, Math.min(findingCount - 1, 30));
|
||||
const delay = Math.min(this.retryMaxMs, this.retryBaseMs * 2 ** exponent);
|
||||
return Math.min(Number.MAX_SAFE_INTEGER, atMs + delay);
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new RangeError('clock must return a non-negative safe integer');
|
||||
}
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
import type { ApprovedActionDispatchCursor } from '../domain/approvedActionDispatchExecution';
|
||||
import type { ApprovedActionRecoveryCursor } from '../domain/approvedActionRecovery';
|
||||
import type {
|
||||
ApprovedActionDispatchCycleOptions,
|
||||
ApprovedActionRecoveryCycleOptions,
|
||||
ApprovedActionRuntimeCycleSummary,
|
||||
ApprovedActionRuntimeSupervisor,
|
||||
} from './approvedActionRuntimeSupervisor';
|
||||
|
||||
export const MIN_APPROVED_ACTION_RUNTIME_INTERVAL_MS = 250;
|
||||
export const MAX_APPROVED_ACTION_RUNTIME_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_APPROVED_ACTION_RUNTIME_INITIAL_DELAY_MS =
|
||||
24 * 60 * 60 * 1_000;
|
||||
export const MAX_APPROVED_ACTION_RUNTIME_STOP_TIMEOUT_MS = 60_000;
|
||||
|
||||
interface ScheduledTimer {
|
||||
unref?: () => void;
|
||||
}
|
||||
|
||||
export interface ApprovedActionRuntimeLifecycleScheduler {
|
||||
setTimeout(callback: () => void, delayMs: number): ScheduledTimer;
|
||||
clearTimeout(timer: ScheduledTimer): void;
|
||||
}
|
||||
|
||||
export interface ApprovedActionRuntimeLifecycleOptions {
|
||||
intervalMs: number;
|
||||
initialDelayMs?: number;
|
||||
stopTimeoutMs?: number;
|
||||
cycle?: {
|
||||
dispatch?: ApprovedActionDispatchCycleOptions;
|
||||
recovery?: ApprovedActionRecoveryCycleOptions;
|
||||
};
|
||||
scheduler?: ApprovedActionRuntimeLifecycleScheduler;
|
||||
onCycle?: (summary: Readonly<ApprovedActionRuntimeCycleSummary>) => void;
|
||||
onError?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export type ApprovedActionRuntimeStopResult = 'drained' | 'timed_out';
|
||||
|
||||
const defaultScheduler: ApprovedActionRuntimeLifecycleScheduler = {
|
||||
setTimeout(callback, delayMs) {
|
||||
return setTimeout(callback, delayMs);
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
clearTimeout(timer as ReturnType<typeof setTimeout>);
|
||||
},
|
||||
};
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One timer serializes recovery and new dispatch work. It remains inert until
|
||||
* start(), never overlaps a slow cycle, resumes bounded keyset cursors, and
|
||||
* waits only a bounded time during shutdown.
|
||||
*/
|
||||
export class ApprovedActionRuntimeLifecycle {
|
||||
private readonly intervalMs: number;
|
||||
private readonly initialDelayMs: number;
|
||||
private readonly stopTimeoutMs: number;
|
||||
private readonly dispatchOptions: Omit<
|
||||
ApprovedActionDispatchCycleOptions,
|
||||
'cursor'
|
||||
>;
|
||||
private readonly recoveryOptions: Omit<
|
||||
ApprovedActionRecoveryCycleOptions,
|
||||
'cursor'
|
||||
>;
|
||||
private readonly scheduler: ApprovedActionRuntimeLifecycleScheduler;
|
||||
private readonly onCycle?: (
|
||||
summary: Readonly<ApprovedActionRuntimeCycleSummary>,
|
||||
) => void;
|
||||
private readonly onError?: (error: unknown) => void;
|
||||
private started = false;
|
||||
private timer?: ScheduledTimer;
|
||||
private inFlight?: Promise<void>;
|
||||
private dispatchCursor?: ApprovedActionDispatchCursor;
|
||||
private recoveryCursor?: ApprovedActionRecoveryCursor;
|
||||
|
||||
constructor(
|
||||
private readonly supervisor: Pick<
|
||||
ApprovedActionRuntimeSupervisor,
|
||||
'runCycle'
|
||||
>,
|
||||
options: ApprovedActionRuntimeLifecycleOptions,
|
||||
) {
|
||||
this.intervalMs = options.intervalMs;
|
||||
this.initialDelayMs = options.initialDelayMs ?? 0;
|
||||
this.stopTimeoutMs = options.stopTimeoutMs ?? 5_000;
|
||||
this.dispatchOptions = {
|
||||
...(options.cycle?.dispatch?.pageSize === undefined
|
||||
? {}
|
||||
: { pageSize: options.cycle.dispatch.pageSize }),
|
||||
...(options.cycle?.dispatch?.maxPages === undefined
|
||||
? {}
|
||||
: { maxPages: options.cycle.dispatch.maxPages }),
|
||||
};
|
||||
this.recoveryOptions = {
|
||||
...(options.cycle?.recovery?.pageSize === undefined
|
||||
? {}
|
||||
: { pageSize: options.cycle.recovery.pageSize }),
|
||||
...(options.cycle?.recovery?.maxPages === undefined
|
||||
? {}
|
||||
: { maxPages: options.cycle.recovery.maxPages }),
|
||||
};
|
||||
this.dispatchCursor =
|
||||
options.cycle?.dispatch?.cursor === undefined
|
||||
? undefined
|
||||
: { ...options.cycle.dispatch.cursor };
|
||||
this.recoveryCursor =
|
||||
options.cycle?.recovery?.cursor === undefined
|
||||
? undefined
|
||||
: { ...options.cycle.recovery.cursor };
|
||||
this.scheduler = options.scheduler ?? defaultScheduler;
|
||||
this.onCycle = options.onCycle;
|
||||
this.onError = options.onError;
|
||||
assertIntegerBetween(
|
||||
'intervalMs',
|
||||
this.intervalMs,
|
||||
MIN_APPROVED_ACTION_RUNTIME_INTERVAL_MS,
|
||||
MAX_APPROVED_ACTION_RUNTIME_INTERVAL_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'initialDelayMs',
|
||||
this.initialDelayMs,
|
||||
0,
|
||||
MAX_APPROVED_ACTION_RUNTIME_INITIAL_DELAY_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'stopTimeoutMs',
|
||||
this.stopTimeoutMs,
|
||||
1,
|
||||
MAX_APPROVED_ACTION_RUNTIME_STOP_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
start(): boolean {
|
||||
if (this.started || this.inFlight) return false;
|
||||
this.started = true;
|
||||
this.schedule(this.initialDelayMs);
|
||||
return true;
|
||||
}
|
||||
|
||||
async stop(): Promise<ApprovedActionRuntimeStopResult> {
|
||||
this.started = false;
|
||||
if (this.timer) {
|
||||
this.scheduler.clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
const inFlight = this.inFlight;
|
||||
if (!inFlight) return 'drained';
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const result = await Promise.race<ApprovedActionRuntimeStopResult>([
|
||||
inFlight.then(() => 'drained' as const),
|
||||
new Promise<'timed_out'>((resolve) => {
|
||||
timeout = setTimeout(() => resolve('timed_out'), this.stopTimeoutMs);
|
||||
}),
|
||||
]);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
return result;
|
||||
}
|
||||
|
||||
private schedule(delayMs: number): void {
|
||||
if (!this.started || this.timer) return;
|
||||
const timer = this.scheduler.setTimeout(() => {
|
||||
if (this.timer === timer) this.timer = undefined;
|
||||
this.run();
|
||||
}, delayMs);
|
||||
this.timer = timer;
|
||||
timer.unref?.();
|
||||
}
|
||||
|
||||
private run(): void {
|
||||
if (!this.started || this.inFlight) return;
|
||||
const inFlight = this.supervisor
|
||||
.runCycle({
|
||||
recovery: {
|
||||
...this.recoveryOptions,
|
||||
...(this.recoveryCursor === undefined
|
||||
? {}
|
||||
: { cursor: { ...this.recoveryCursor } }),
|
||||
},
|
||||
dispatch: {
|
||||
...this.dispatchOptions,
|
||||
...(this.dispatchCursor === undefined
|
||||
? {}
|
||||
: { cursor: { ...this.dispatchCursor } }),
|
||||
},
|
||||
})
|
||||
.then((summary) => {
|
||||
this.recoveryCursor =
|
||||
summary.recovery.remaining && summary.recovery.nextCursor
|
||||
? { ...summary.recovery.nextCursor }
|
||||
: undefined;
|
||||
this.dispatchCursor =
|
||||
summary.dispatch.remaining && summary.dispatch.nextCursor
|
||||
? { ...summary.dispatch.nextCursor }
|
||||
: undefined;
|
||||
this.notifyCycle(summary);
|
||||
})
|
||||
.catch((error) => this.notifyError(error))
|
||||
.then(() => undefined)
|
||||
.finally(() => {
|
||||
if (this.inFlight === inFlight) this.inFlight = undefined;
|
||||
if (this.started) this.schedule(this.intervalMs);
|
||||
});
|
||||
this.inFlight = inFlight;
|
||||
}
|
||||
|
||||
private notifyCycle(
|
||||
summary: Readonly<ApprovedActionRuntimeCycleSummary>,
|
||||
): void {
|
||||
try {
|
||||
this.onCycle?.(summary);
|
||||
} catch (error) {
|
||||
this.notifyError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private notifyError(error: unknown): void {
|
||||
try {
|
||||
this.onError?.(error);
|
||||
} catch {
|
||||
// Diagnostics must not create another scheduler failure loop.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
import {
|
||||
assertApprovedActionPageSize,
|
||||
type ApprovedActionDispatchCursor,
|
||||
} from '../domain/approvedActionDispatchExecution';
|
||||
import {
|
||||
assertApprovedActionRecoveryPageSize,
|
||||
type ApprovedActionRecoveryCursor,
|
||||
} from '../domain/approvedActionRecovery';
|
||||
import type {
|
||||
ApprovedActionDispatchBatchSummary,
|
||||
ApprovedActionDispatcher,
|
||||
} from './approvedActionDispatcher';
|
||||
import type {
|
||||
ApprovedActionRecoveryBatchSummary,
|
||||
ApprovedActionRecoveryReconciler,
|
||||
} from './approvedActionRecoveryReconciler';
|
||||
|
||||
export const MAX_APPROVED_ACTION_RUNTIME_PAGES_PER_PHASE = 64;
|
||||
|
||||
export type ApprovedActionRuntimePhaseStopReason =
|
||||
| 'complete'
|
||||
| 'page_limit'
|
||||
| 'cursor_stalled';
|
||||
|
||||
export interface ApprovedActionDispatchCycleOptions {
|
||||
cursor?: ApprovedActionDispatchCursor;
|
||||
pageSize?: number;
|
||||
maxPages?: number;
|
||||
}
|
||||
|
||||
export interface ApprovedActionRecoveryCycleOptions {
|
||||
cursor?: ApprovedActionRecoveryCursor;
|
||||
pageSize?: number;
|
||||
maxPages?: number;
|
||||
}
|
||||
|
||||
export interface ApprovedActionRuntimeCycleOptions {
|
||||
dispatch?: ApprovedActionDispatchCycleOptions;
|
||||
recovery?: ApprovedActionRecoveryCycleOptions;
|
||||
}
|
||||
|
||||
export interface ApprovedActionDispatchCycleSummary
|
||||
extends Omit<ApprovedActionDispatchBatchSummary, 'truncated' | 'nextCursor'> {
|
||||
pages: number;
|
||||
stopReason: ApprovedActionRuntimePhaseStopReason;
|
||||
remaining: boolean;
|
||||
nextCursor?: Readonly<ApprovedActionDispatchCursor>;
|
||||
}
|
||||
|
||||
export interface ApprovedActionRecoveryCycleSummary
|
||||
extends Omit<ApprovedActionRecoveryBatchSummary, 'truncated' | 'nextCursor'> {
|
||||
pages: number;
|
||||
stopReason: ApprovedActionRuntimePhaseStopReason;
|
||||
remaining: boolean;
|
||||
nextCursor?: Readonly<ApprovedActionRecoveryCursor>;
|
||||
}
|
||||
|
||||
export interface ApprovedActionRuntimeCycleSummary {
|
||||
recovery: Readonly<ApprovedActionRecoveryCycleSummary>;
|
||||
dispatch: Readonly<ApprovedActionDispatchCycleSummary>;
|
||||
}
|
||||
|
||||
function exactKeys(value: object, expected: readonly string[]): boolean {
|
||||
const keys = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
return (
|
||||
keys.length === canonical.length &&
|
||||
keys.every((key, index) => key === canonical[index])
|
||||
);
|
||||
}
|
||||
|
||||
function assertOptionsObject(
|
||||
name: string,
|
||||
value: unknown,
|
||||
): asserts value is object {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new TypeError(`${name} must be an object`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertMaxPages(value: number): void {
|
||||
if (
|
||||
!Number.isSafeInteger(value) ||
|
||||
value < 1 ||
|
||||
value > MAX_APPROVED_ACTION_RUNTIME_PAGES_PER_PHASE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'maxPages must be between 1 and MAX_APPROVED_ACTION_RUNTIME_PAGES_PER_PHASE',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function sameDispatchCursor(
|
||||
left: ApprovedActionDispatchCursor | undefined,
|
||||
right: Readonly<ApprovedActionDispatchCursor>,
|
||||
): boolean {
|
||||
return (
|
||||
left !== undefined &&
|
||||
left.eligibleAtMs === right.eligibleAtMs &&
|
||||
left.dispatchId === right.dispatchId
|
||||
);
|
||||
}
|
||||
|
||||
function sameRecoveryCursor(
|
||||
left: ApprovedActionRecoveryCursor | undefined,
|
||||
right: Readonly<ApprovedActionRecoveryCursor>,
|
||||
): boolean {
|
||||
return (
|
||||
left !== undefined &&
|
||||
left.nextScanAtMs === right.nextScanAtMs &&
|
||||
left.dispatchId === right.dispatchId
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeDispatchOptions(
|
||||
value: ApprovedActionDispatchCycleOptions = {},
|
||||
): Required<Pick<ApprovedActionDispatchCycleOptions, 'pageSize' | 'maxPages'>> &
|
||||
Pick<ApprovedActionDispatchCycleOptions, 'cursor'> {
|
||||
assertOptionsObject('dispatch cycle options', value);
|
||||
const expectedKeys = [
|
||||
...(value.cursor === undefined ? [] : ['cursor']),
|
||||
...(value.pageSize === undefined ? [] : ['pageSize']),
|
||||
...(value.maxPages === undefined ? [] : ['maxPages']),
|
||||
];
|
||||
if (!exactKeys(value, expectedKeys)) {
|
||||
throw new TypeError('dispatch cycle options shape is invalid');
|
||||
}
|
||||
const pageSize = value.pageSize ?? 16;
|
||||
const maxPages = value.maxPages ?? 4;
|
||||
assertApprovedActionPageSize(pageSize);
|
||||
assertMaxPages(maxPages);
|
||||
return {
|
||||
pageSize,
|
||||
maxPages,
|
||||
...(value.cursor === undefined ? {} : { cursor: { ...value.cursor } }),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeRecoveryOptions(
|
||||
value: ApprovedActionRecoveryCycleOptions = {},
|
||||
): Required<Pick<ApprovedActionRecoveryCycleOptions, 'pageSize' | 'maxPages'>> &
|
||||
Pick<ApprovedActionRecoveryCycleOptions, 'cursor'> {
|
||||
assertOptionsObject('recovery cycle options', value);
|
||||
const expectedKeys = [
|
||||
...(value.cursor === undefined ? [] : ['cursor']),
|
||||
...(value.pageSize === undefined ? [] : ['pageSize']),
|
||||
...(value.maxPages === undefined ? [] : ['maxPages']),
|
||||
];
|
||||
if (!exactKeys(value, expectedKeys)) {
|
||||
throw new TypeError('recovery cycle options shape is invalid');
|
||||
}
|
||||
const pageSize = value.pageSize ?? 16;
|
||||
const maxPages = value.maxPages ?? 4;
|
||||
assertApprovedActionRecoveryPageSize(pageSize);
|
||||
assertMaxPages(maxPages);
|
||||
return {
|
||||
pageSize,
|
||||
maxPages,
|
||||
...(value.cursor === undefined ? {} : { cursor: { ...value.cursor } }),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs one bounded SQLite control-plane cycle. Recovery is deliberately first:
|
||||
* if the recovery index cannot be read, the cycle does not create more action
|
||||
* side effects. The class owns no timer and is safe to embed in other profiles.
|
||||
*/
|
||||
export class ApprovedActionRuntimeSupervisor {
|
||||
constructor(
|
||||
private readonly dispatcher: Pick<
|
||||
ApprovedActionDispatcher,
|
||||
'dispatchBatch'
|
||||
>,
|
||||
private readonly reconciler: Pick<
|
||||
ApprovedActionRecoveryReconciler,
|
||||
'reconcileBatch'
|
||||
>,
|
||||
) {}
|
||||
|
||||
async runCycle(
|
||||
options: ApprovedActionRuntimeCycleOptions = {},
|
||||
): Promise<Readonly<ApprovedActionRuntimeCycleSummary>> {
|
||||
assertOptionsObject('approved action runtime options', options);
|
||||
const expectedKeys = [
|
||||
...(options.dispatch === undefined ? [] : ['dispatch']),
|
||||
...(options.recovery === undefined ? [] : ['recovery']),
|
||||
];
|
||||
if (!exactKeys(options, expectedKeys)) {
|
||||
throw new TypeError('approved action runtime options shape is invalid');
|
||||
}
|
||||
const recoveryOptions = normalizeRecoveryOptions(options.recovery);
|
||||
const dispatchOptions = normalizeDispatchOptions(options.dispatch);
|
||||
const recovery = await this.runRecovery(recoveryOptions);
|
||||
const dispatch = await this.runDispatch(dispatchOptions);
|
||||
return Object.freeze({ recovery, dispatch });
|
||||
}
|
||||
|
||||
private async runDispatch(
|
||||
options: ReturnType<typeof normalizeDispatchOptions>,
|
||||
): Promise<Readonly<ApprovedActionDispatchCycleSummary>> {
|
||||
const total: ApprovedActionDispatchCycleSummary = {
|
||||
pages: 0,
|
||||
scanned: 0,
|
||||
claimed: 0,
|
||||
started: 0,
|
||||
succeeded: 0,
|
||||
failed: 0,
|
||||
blocked: 0,
|
||||
retrying: 0,
|
||||
deferred: 0,
|
||||
recoveryRequired: 0,
|
||||
alreadyTerminal: 0,
|
||||
unavailable: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
let cursor = options.cursor;
|
||||
for (let pageNumber = 0; pageNumber < options.maxPages; pageNumber += 1) {
|
||||
const page = await this.dispatcher.dispatchBatch({
|
||||
...(cursor === undefined ? {} : { cursor }),
|
||||
limit: options.pageSize,
|
||||
});
|
||||
total.pages += 1;
|
||||
total.scanned += page.scanned;
|
||||
total.claimed += page.claimed;
|
||||
total.started += page.started;
|
||||
total.succeeded += page.succeeded;
|
||||
total.failed += page.failed;
|
||||
total.blocked += page.blocked;
|
||||
total.retrying += page.retrying;
|
||||
total.deferred += page.deferred;
|
||||
total.recoveryRequired += page.recoveryRequired;
|
||||
total.alreadyTerminal += page.alreadyTerminal;
|
||||
total.unavailable += page.unavailable;
|
||||
if (!page.truncated) return Object.freeze(total);
|
||||
if (!page.nextCursor || sameDispatchCursor(cursor, page.nextCursor)) {
|
||||
total.stopReason = 'cursor_stalled';
|
||||
total.remaining = true;
|
||||
if (page.nextCursor) total.nextCursor = { ...page.nextCursor };
|
||||
return Object.freeze(total);
|
||||
}
|
||||
cursor = { ...page.nextCursor };
|
||||
if (pageNumber === options.maxPages - 1) {
|
||||
total.stopReason = 'page_limit';
|
||||
total.remaining = true;
|
||||
total.nextCursor = cursor;
|
||||
return Object.freeze(total);
|
||||
}
|
||||
}
|
||||
return Object.freeze(total);
|
||||
}
|
||||
|
||||
private async runRecovery(
|
||||
options: ReturnType<typeof normalizeRecoveryOptions>,
|
||||
): Promise<Readonly<ApprovedActionRecoveryCycleSummary>> {
|
||||
const total: ApprovedActionRecoveryCycleSummary = {
|
||||
pages: 0,
|
||||
scanned: 0,
|
||||
claimed: 0,
|
||||
verifiedSucceeded: 0,
|
||||
verifiedFailed: 0,
|
||||
deferred: 0,
|
||||
manualRequired: 0,
|
||||
executionActive: 0,
|
||||
alreadyResolved: 0,
|
||||
unavailable: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
let cursor = options.cursor;
|
||||
for (let pageNumber = 0; pageNumber < options.maxPages; pageNumber += 1) {
|
||||
const page = await this.reconciler.reconcileBatch({
|
||||
...(cursor === undefined ? {} : { cursor }),
|
||||
limit: options.pageSize,
|
||||
});
|
||||
total.pages += 1;
|
||||
total.scanned += page.scanned;
|
||||
total.claimed += page.claimed;
|
||||
total.verifiedSucceeded += page.verifiedSucceeded;
|
||||
total.verifiedFailed += page.verifiedFailed;
|
||||
total.deferred += page.deferred;
|
||||
total.manualRequired += page.manualRequired;
|
||||
total.executionActive += page.executionActive;
|
||||
total.alreadyResolved += page.alreadyResolved;
|
||||
total.unavailable += page.unavailable;
|
||||
if (!page.truncated) return Object.freeze(total);
|
||||
if (!page.nextCursor || sameRecoveryCursor(cursor, page.nextCursor)) {
|
||||
total.stopReason = 'cursor_stalled';
|
||||
total.remaining = true;
|
||||
if (page.nextCursor) total.nextCursor = { ...page.nextCursor };
|
||||
return Object.freeze(total);
|
||||
}
|
||||
cursor = { ...page.nextCursor };
|
||||
if (pageNumber === options.maxPages - 1) {
|
||||
total.stopReason = 'page_limit';
|
||||
total.remaining = true;
|
||||
total.nextCursor = cursor;
|
||||
return Object.freeze(total);
|
||||
}
|
||||
}
|
||||
return Object.freeze(total);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import {
|
||||
APPROVED_RUN_ACTION_TYPE,
|
||||
APPROVED_RUN_RECEIPT_RESULT_CODE,
|
||||
ApprovedRunActionBindingConflictError,
|
||||
InvalidApprovedRunActionError,
|
||||
digestApprovedRunCreationPlan,
|
||||
normalizeApprovedRunCreationPlan,
|
||||
type ApprovedRunCreationPlan,
|
||||
} from '../domain/approvedRunAction';
|
||||
import type { ApprovedActionDispatchRecord } from '../domain/approvalRequest';
|
||||
import type {
|
||||
ApprovedActionExecutionContext,
|
||||
ApprovedActionExecutionResult,
|
||||
ApprovedActionHandler,
|
||||
ApprovedActionInspectionResult,
|
||||
} from '../ports/approvedActionHandler';
|
||||
import type { ApprovedRunActionPlanResolver } from '../ports/approvedRunActionPlanResolver';
|
||||
import type { ApprovedRunActionRepository } from '../ports/approvedRunActionRepository';
|
||||
|
||||
export class ApprovedRunActionHandler implements ApprovedActionHandler {
|
||||
readonly actionType = APPROVED_RUN_ACTION_TYPE;
|
||||
|
||||
constructor(
|
||||
private readonly plans: ApprovedRunActionPlanResolver,
|
||||
private readonly repository: ApprovedRunActionRepository,
|
||||
) {}
|
||||
|
||||
async inspect(
|
||||
dispatch: Readonly<ApprovedActionDispatchRecord>,
|
||||
): Promise<ApprovedActionInspectionResult> {
|
||||
if (dispatch.action.actionType !== this.actionType) {
|
||||
return { status: 'blocked', resultCode: 'approved_run_type_mismatch' };
|
||||
}
|
||||
const plan = await this.plans.resolve(dispatch.action.actionRef);
|
||||
if (!plan) {
|
||||
return { status: 'retry', resultCode: 'approved_run_plan_missing' };
|
||||
}
|
||||
try {
|
||||
const normalized = normalizeApprovedRunCreationPlan(plan);
|
||||
if (
|
||||
normalized.actionRef !== dispatch.action.actionRef ||
|
||||
normalized.projectId !== dispatch.projectId
|
||||
) {
|
||||
return {
|
||||
status: 'blocked',
|
||||
resultCode: 'approved_run_plan_binding_invalid',
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: 'ready',
|
||||
actionDigest: digestApprovedRunCreationPlan(normalized),
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidApprovedRunActionError) {
|
||||
return { status: 'blocked', resultCode: 'approved_run_plan_invalid' };
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async execute(
|
||||
context: Readonly<ApprovedActionExecutionContext>,
|
||||
): Promise<ApprovedActionExecutionResult> {
|
||||
if (!this.contextIsBound(context)) {
|
||||
return { outcome: 'failed', resultCode: 'approved_run_fence_invalid' };
|
||||
}
|
||||
const plan = await this.plans.resolve(context.dispatch.action.actionRef);
|
||||
if (!plan) {
|
||||
return {
|
||||
outcome: 'failed',
|
||||
resultCode: 'approved_run_plan_disappeared',
|
||||
};
|
||||
}
|
||||
let normalized: Readonly<ApprovedRunCreationPlan>;
|
||||
try {
|
||||
normalized = normalizeApprovedRunCreationPlan(plan);
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidApprovedRunActionError) {
|
||||
return { outcome: 'failed', resultCode: 'approved_run_plan_changed' };
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (
|
||||
normalized.actionRef !== context.dispatch.action.actionRef ||
|
||||
normalized.projectId !== context.dispatch.projectId ||
|
||||
digestApprovedRunCreationPlan(normalized) !==
|
||||
context.dispatch.action.actionDigest
|
||||
) {
|
||||
return { outcome: 'failed', resultCode: 'approved_run_plan_changed' };
|
||||
}
|
||||
try {
|
||||
await this.repository.create({
|
||||
snapshot: Object.freeze({
|
||||
dispatch: context.dispatch,
|
||||
execution: context.execution,
|
||||
}),
|
||||
plan: normalized,
|
||||
});
|
||||
return {
|
||||
outcome: 'succeeded',
|
||||
resultCode: APPROVED_RUN_RECEIPT_RESULT_CODE,
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof ApprovedRunActionBindingConflictError) {
|
||||
return {
|
||||
outcome: 'failed',
|
||||
resultCode: 'approved_run_receipt_conflict',
|
||||
};
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private contextIsBound(
|
||||
context: Readonly<ApprovedActionExecutionContext>,
|
||||
): boolean {
|
||||
return (
|
||||
context.dispatch.action.actionType === this.actionType &&
|
||||
context.execution.status === 'executing' &&
|
||||
context.execution.dispatchId === context.dispatch.id &&
|
||||
context.execution.projectId === context.dispatch.projectId &&
|
||||
context.execution.startedAtMs !== null &&
|
||||
context.execution.leaseOwner === context.fence.owner &&
|
||||
context.execution.leaseToken === context.fence.leaseToken &&
|
||||
context.execution.version === context.fence.version &&
|
||||
context.idempotencyKey === context.dispatch.id
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import type {
|
||||
AcknowledgeRemoteRunRunningCommand,
|
||||
AcknowledgeRemoteRunStartingCommand,
|
||||
FailRemoteRunStartCommand,
|
||||
RemoteRunActivationResult,
|
||||
} from './remoteRunActivationService';
|
||||
import { RemoteRunActivationService } from './remoteRunActivationService';
|
||||
import type { AuthenticatedWorkerPrincipal } from './workerControlService';
|
||||
import type { WorkerRemoteRunActivationClient } from '../ports/workerRemoteRunActivationClient';
|
||||
|
||||
/** The transport authenticates once; Worker request bodies cannot select a principal. */
|
||||
export class BoundWorkerRemoteRunActivationClient
|
||||
implements WorkerRemoteRunActivationClient
|
||||
{
|
||||
constructor(
|
||||
private readonly service: RemoteRunActivationService,
|
||||
private readonly principal: AuthenticatedWorkerPrincipal,
|
||||
) {}
|
||||
|
||||
acknowledgeStarting(
|
||||
command: AcknowledgeRemoteRunStartingCommand,
|
||||
): Promise<RemoteRunActivationResult> {
|
||||
return this.service.acknowledgeStarting(this.principal, command);
|
||||
}
|
||||
|
||||
acknowledgeRunning(
|
||||
command: AcknowledgeRemoteRunRunningCommand,
|
||||
): Promise<RemoteRunActivationResult> {
|
||||
return this.service.acknowledgeRunning(this.principal, command);
|
||||
}
|
||||
|
||||
failStart(
|
||||
command: FailRemoteRunStartCommand,
|
||||
): Promise<RemoteRunActivationResult> {
|
||||
return this.service.failStart(this.principal, command);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import type { WorkerRemoteRunCompletionClient } from '../ports/workerRemoteRunCompletionClient';
|
||||
import type { PrimaryRunCompletionResult } from './primaryRunCompletionService';
|
||||
import {
|
||||
RemoteRunCompletionService,
|
||||
type RemoteRunCompletionCommand,
|
||||
} from './remoteRunCompletionService';
|
||||
import type { AuthenticatedWorkerPrincipal } from './workerControlService';
|
||||
|
||||
/** The transport authenticates once; Worker request bodies cannot select a principal. */
|
||||
export class BoundWorkerRemoteRunCompletionClient
|
||||
implements WorkerRemoteRunCompletionClient
|
||||
{
|
||||
constructor(
|
||||
private readonly service: RemoteRunCompletionService,
|
||||
private readonly principal: AuthenticatedWorkerPrincipal,
|
||||
) {}
|
||||
|
||||
complete(
|
||||
command: RemoteRunCompletionCommand,
|
||||
): Promise<PrimaryRunCompletionResult> {
|
||||
return this.service.complete(this.principal, command);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import type { RunDispatchLeaseRecord } from '../domain/runDispatchLease';
|
||||
import type { ReleaseRunDispatchLeaseResult } from '../ports/runDispatchLeaseRepository';
|
||||
import type { WorkerRunLeaseClient } from '../ports/workerRunLeaseClient';
|
||||
import type {
|
||||
FencedRunDispatchLeaseRequest,
|
||||
ReleaseRunDispatchLeaseRequest,
|
||||
} from './runDispatchLeaseService';
|
||||
import { RunDispatchLeaseService } from './runDispatchLeaseService';
|
||||
import type { AuthenticatedWorkerPrincipal } from './workerControlService';
|
||||
|
||||
/**
|
||||
* Transport seam: the authenticated principal is fixed when the client is
|
||||
* constructed and can never be supplied by a Worker request body.
|
||||
*/
|
||||
export class BoundWorkerRunLeaseClient implements WorkerRunLeaseClient {
|
||||
constructor(
|
||||
private readonly service: RunDispatchLeaseService,
|
||||
private readonly principal: AuthenticatedWorkerPrincipal,
|
||||
) {}
|
||||
|
||||
renew(
|
||||
request: FencedRunDispatchLeaseRequest,
|
||||
): Promise<RunDispatchLeaseRecord> {
|
||||
return this.service.renew(this.principal, request);
|
||||
}
|
||||
|
||||
release(
|
||||
request: ReleaseRunDispatchLeaseRequest,
|
||||
): Promise<ReleaseRunDispatchLeaseResult> {
|
||||
return this.service.release(this.principal, request);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
import type { DeploymentProfile } from '../domain/deploymentProfile';
|
||||
|
||||
export type ClusterControlActivationState =
|
||||
| 'disabled'
|
||||
| 'schema_ready'
|
||||
| 'reconciled'
|
||||
| 'active'
|
||||
| 'failed'
|
||||
| 'stopped';
|
||||
|
||||
export interface ClusterControlReadinessEvidence {
|
||||
readonly contractName: string;
|
||||
readonly contractVersion: number;
|
||||
readonly serverMajor: number;
|
||||
readonly migrationIds: readonly string[];
|
||||
}
|
||||
|
||||
export interface ClusterControlReadinessProbe {
|
||||
assertReady(): Promise<ClusterControlReadinessEvidence>;
|
||||
}
|
||||
|
||||
export interface ClusterControlStartupRecoverySummary {
|
||||
readonly safe: boolean;
|
||||
readonly remaining: number;
|
||||
readonly failed: number;
|
||||
}
|
||||
|
||||
export type ClusterControlStopResult = 'stopped' | 'timed_out';
|
||||
|
||||
export interface ClusterControlActivationStack {
|
||||
reconcile(): Promise<ClusterControlStartupRecoverySummary>;
|
||||
startLifecycles(): Promise<boolean>;
|
||||
installAdmission(): () => void;
|
||||
stop(): Promise<ClusterControlStopResult>;
|
||||
}
|
||||
|
||||
export interface ClusterControlActivationAudit {
|
||||
readonly state: ClusterControlActivationState;
|
||||
readonly contractName?: string;
|
||||
readonly contractVersion?: number;
|
||||
readonly serverMajor?: number;
|
||||
readonly migrationCount?: number;
|
||||
readonly recovery?: ClusterControlStartupRecoverySummary;
|
||||
}
|
||||
|
||||
export interface ClusterControlRuntimeActivationOptions {
|
||||
readonly enabled?: boolean;
|
||||
readonly profile: DeploymentProfile;
|
||||
readonly readiness: ClusterControlReadinessProbe;
|
||||
readonly create: (
|
||||
evidence: ClusterControlReadinessEvidence,
|
||||
) => ClusterControlActivationStack;
|
||||
readonly audit: (
|
||||
record: ClusterControlActivationAudit,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export type ClusterControlRuntimeActivationResult =
|
||||
| { readonly status: 'disabled'; stop(): Promise<'stopped'> }
|
||||
| {
|
||||
readonly status: 'active';
|
||||
readonly evidence: ClusterControlReadinessEvidence;
|
||||
readonly recovery: ClusterControlStartupRecoverySummary;
|
||||
stop(): Promise<ClusterControlStopResult>;
|
||||
};
|
||||
|
||||
const DISABLED_STOP = async (): Promise<'stopped'> => 'stopped';
|
||||
|
||||
function auditEvidence(
|
||||
evidence: ClusterControlReadinessEvidence,
|
||||
): Pick<
|
||||
ClusterControlActivationAudit,
|
||||
'contractName' | 'contractVersion' | 'serverMajor' | 'migrationCount'
|
||||
> {
|
||||
return {
|
||||
contractName: evidence.contractName,
|
||||
contractVersion: evidence.contractVersion,
|
||||
serverMajor: evidence.serverMajor,
|
||||
migrationCount: evidence.migrationIds.length,
|
||||
};
|
||||
}
|
||||
|
||||
function assertSafeRecovery(
|
||||
recovery: ClusterControlStartupRecoverySummary,
|
||||
): void {
|
||||
if (!recovery.safe || recovery.remaining !== 0 || recovery.failed !== 0) {
|
||||
throw new Error('Cluster-control startup recovery did not converge safely');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enforces readiness -> assembly -> recovery -> lifecycle -> admission order.
|
||||
* The factory is deliberately called only after schema/role readiness, so an
|
||||
* invalid cluster database cannot even construct business repositories.
|
||||
*/
|
||||
export async function activateClusterControlRuntime(
|
||||
options: ClusterControlRuntimeActivationOptions,
|
||||
): Promise<ClusterControlRuntimeActivationResult> {
|
||||
const enabled = options.enabled ?? false;
|
||||
if (!enabled) {
|
||||
await options.audit({ state: 'disabled' });
|
||||
return { status: 'disabled', stop: DISABLED_STOP };
|
||||
}
|
||||
if (options.profile !== 'cluster-control') {
|
||||
throw new TypeError(
|
||||
`Deployment profile ${options.profile} cannot activate cluster-control`,
|
||||
);
|
||||
}
|
||||
|
||||
let evidence: ClusterControlReadinessEvidence | undefined;
|
||||
let stack: ClusterControlActivationStack | undefined;
|
||||
let disposeAdmission: (() => void) | undefined;
|
||||
try {
|
||||
evidence = await options.readiness.assertReady();
|
||||
await options.audit({ state: 'schema_ready', ...auditEvidence(evidence) });
|
||||
stack = options.create(evidence);
|
||||
const recovery = await stack.reconcile();
|
||||
assertSafeRecovery(recovery);
|
||||
await options.audit({
|
||||
state: 'reconciled',
|
||||
...auditEvidence(evidence),
|
||||
recovery,
|
||||
});
|
||||
if (!(await stack.startLifecycles())) {
|
||||
throw new Error('Cluster-control lifecycles did not start');
|
||||
}
|
||||
disposeAdmission = stack.installAdmission();
|
||||
await options.audit({
|
||||
state: 'active',
|
||||
...auditEvidence(evidence),
|
||||
recovery,
|
||||
});
|
||||
|
||||
let stopPromise: Promise<ClusterControlStopResult> | undefined;
|
||||
return {
|
||||
status: 'active',
|
||||
evidence,
|
||||
recovery,
|
||||
stop() {
|
||||
if (stopPromise) return stopPromise;
|
||||
stopPromise = (async () => {
|
||||
let admissionError: unknown;
|
||||
try {
|
||||
disposeAdmission?.();
|
||||
} catch (error) {
|
||||
admissionError = error;
|
||||
}
|
||||
disposeAdmission = undefined;
|
||||
const result = await stack!.stop();
|
||||
if (admissionError) {
|
||||
try {
|
||||
await options.audit({
|
||||
state: 'failed',
|
||||
...auditEvidence(evidence!),
|
||||
});
|
||||
} catch {
|
||||
// Preserve the admission cleanup failure after stopping the stack.
|
||||
}
|
||||
throw admissionError;
|
||||
}
|
||||
try {
|
||||
await options.audit({
|
||||
state: 'stopped',
|
||||
...auditEvidence(evidence!),
|
||||
recovery,
|
||||
});
|
||||
} catch {
|
||||
// Diagnostic failure cannot reverse stopped ownership.
|
||||
}
|
||||
return result;
|
||||
})();
|
||||
return stopPromise;
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
try {
|
||||
disposeAdmission?.();
|
||||
} catch {
|
||||
// Preserve the activation failure and continue stopping the stack.
|
||||
}
|
||||
if (stack) {
|
||||
try {
|
||||
await stack.stop();
|
||||
} catch {
|
||||
// Preserve the activation failure after best-effort cleanup.
|
||||
}
|
||||
}
|
||||
try {
|
||||
await options.audit({
|
||||
state: 'failed',
|
||||
...(evidence ? auditEvidence(evidence) : {}),
|
||||
});
|
||||
} catch {
|
||||
// Diagnostic failure cannot replace the activation failure.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { RunAttemptStatus } from '../domain/run';
|
||||
|
||||
export class RunCommandError extends Error {
|
||||
constructor(message: string, public readonly code: string) {
|
||||
super(message);
|
||||
this.name = new.target.name;
|
||||
}
|
||||
}
|
||||
|
||||
export class RunNotFoundError extends RunCommandError {
|
||||
constructor(public readonly runId: string) {
|
||||
super('Run does not exist', 'RUN_NOT_FOUND');
|
||||
}
|
||||
}
|
||||
|
||||
export class RunAttemptNotFoundError extends RunCommandError {
|
||||
constructor(public readonly attemptId: string) {
|
||||
super('RunAttempt does not exist', 'RUN_ATTEMPT_NOT_FOUND');
|
||||
}
|
||||
}
|
||||
|
||||
export class RunAttemptConcurrentWriteError extends RunCommandError {
|
||||
constructor(
|
||||
public readonly attemptId: string,
|
||||
public readonly expectedStatus: RunAttemptStatus,
|
||||
public readonly expectedCallbackSequence: number,
|
||||
) {
|
||||
super(
|
||||
'RunAttempt changed while applying the command',
|
||||
'RUN_ATTEMPT_CONCURRENT_WRITE',
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
import { isTerminalRunAttemptStatus } from '../domain/runStateMachine';
|
||||
import type {
|
||||
CompletionReceiptDirectoryEntry,
|
||||
CompletionReceiptOrphanDirectory,
|
||||
CompletionReceiptOwnership,
|
||||
CompletionReceiptOwnershipSource,
|
||||
} from '../ports/completionReceiptOrphanMaintenance';
|
||||
|
||||
export const MAX_ORPHAN_AUDIT_SHARDS = 32;
|
||||
export const MAX_ORPHAN_AUDIT_ENTRIES_PER_SHARD = 64;
|
||||
|
||||
export type CompletionReceiptOrphanAuditMode = 'audit' | 'quarantine';
|
||||
export type CompletionReceiptOrphanCategory =
|
||||
| 'journaled'
|
||||
| 'active_attempt'
|
||||
| 'young_terminal_attempt'
|
||||
| 'terminal_orphan'
|
||||
| 'young_unknown_receipt'
|
||||
| 'unknown_receipt'
|
||||
| 'young_temporary'
|
||||
| 'stale_temporary'
|
||||
| 'young_unknown_entry'
|
||||
| 'unknown_entry'
|
||||
| 'unsafe_entry';
|
||||
export type CompletionReceiptOrphanAction =
|
||||
| 'retained'
|
||||
| 'eligible'
|
||||
| 'blocked_overflow'
|
||||
| 'quarantined'
|
||||
| 'changed';
|
||||
|
||||
export interface CompletionReceiptOrphanAuditEntry {
|
||||
shard: string;
|
||||
name: string;
|
||||
category: CompletionReceiptOrphanCategory;
|
||||
action: CompletionReceiptOrphanAction;
|
||||
ageMs: number;
|
||||
attemptId?: string;
|
||||
attemptStatus?: string;
|
||||
quarantineRef?: string;
|
||||
}
|
||||
|
||||
export interface CompletionReceiptOrphanAuditReport {
|
||||
schemaVersion: 1;
|
||||
mode: CompletionReceiptOrphanAuditMode;
|
||||
observedAtMs: number;
|
||||
minimumAgeMs: number;
|
||||
startShard: string;
|
||||
nextShard: string;
|
||||
wrapped: boolean;
|
||||
shardCount: number;
|
||||
maxEntriesPerShard: number;
|
||||
scannedEntries: number;
|
||||
overflowShards: readonly string[];
|
||||
entries: readonly CompletionReceiptOrphanAuditEntry[];
|
||||
counts: Readonly<Record<CompletionReceiptOrphanCategory, number>>;
|
||||
}
|
||||
|
||||
export interface CompletionReceiptOrphanAuditorOptions {
|
||||
mode?: CompletionReceiptOrphanAuditMode;
|
||||
observedAtMs?: number;
|
||||
minimumAgeMs?: number;
|
||||
startShard?: number;
|
||||
shardCount?: number;
|
||||
maxEntriesPerShard?: number;
|
||||
clock?: { now(): number };
|
||||
}
|
||||
|
||||
const CATEGORIES: readonly CompletionReceiptOrphanCategory[] = [
|
||||
'journaled',
|
||||
'active_attempt',
|
||||
'young_terminal_attempt',
|
||||
'terminal_orphan',
|
||||
'young_unknown_receipt',
|
||||
'unknown_receipt',
|
||||
'young_temporary',
|
||||
'stale_temporary',
|
||||
'young_unknown_entry',
|
||||
'unknown_entry',
|
||||
'unsafe_entry',
|
||||
];
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
function shardName(value: number): string {
|
||||
return value.toString(16).padStart(2, '0');
|
||||
}
|
||||
|
||||
function classify(
|
||||
entry: CompletionReceiptDirectoryEntry,
|
||||
ownership: CompletionReceiptOwnership | undefined,
|
||||
oldEnough: boolean,
|
||||
): { category: CompletionReceiptOrphanCategory; eligible: boolean } {
|
||||
if (entry.kind === 'unsafe') {
|
||||
return { category: 'unsafe_entry', eligible: false };
|
||||
}
|
||||
if (entry.kind === 'temporary') {
|
||||
return oldEnough
|
||||
? { category: 'stale_temporary', eligible: true }
|
||||
: { category: 'young_temporary', eligible: false };
|
||||
}
|
||||
if (entry.kind === 'unknown') {
|
||||
return oldEnough
|
||||
? { category: 'unknown_entry', eligible: true }
|
||||
: { category: 'young_unknown_entry', eligible: false };
|
||||
}
|
||||
if (ownership?.journalState) {
|
||||
return { category: 'journaled', eligible: false };
|
||||
}
|
||||
if (ownership?.attemptStatus) {
|
||||
if (!isTerminalRunAttemptStatus(ownership.attemptStatus)) {
|
||||
return { category: 'active_attempt', eligible: false };
|
||||
}
|
||||
return oldEnough
|
||||
? { category: 'terminal_orphan', eligible: true }
|
||||
: { category: 'young_terminal_attempt', eligible: false };
|
||||
}
|
||||
return oldEnough
|
||||
? { category: 'unknown_receipt', eligible: true }
|
||||
: { category: 'young_unknown_receipt', eligible: false };
|
||||
}
|
||||
|
||||
export class CompletionReceiptOrphanAuditor {
|
||||
constructor(
|
||||
private readonly directory: CompletionReceiptOrphanDirectory,
|
||||
private readonly ownership: CompletionReceiptOwnershipSource,
|
||||
) {}
|
||||
|
||||
async run(
|
||||
options: CompletionReceiptOrphanAuditorOptions = {},
|
||||
): Promise<CompletionReceiptOrphanAuditReport> {
|
||||
const mode = options.mode ?? 'audit';
|
||||
if (mode !== 'audit' && mode !== 'quarantine') {
|
||||
throw new RangeError('mode must be audit or quarantine');
|
||||
}
|
||||
const observedAtMs =
|
||||
options.observedAtMs ?? options.clock?.now() ?? Date.now();
|
||||
const minimumAgeMs = options.minimumAgeMs ?? 5 * 60_000;
|
||||
const startShard = options.startShard ?? 0;
|
||||
const shardCount = options.shardCount ?? 8;
|
||||
const maxEntriesPerShard = options.maxEntriesPerShard ?? 32;
|
||||
assertIntegerBetween(
|
||||
'observedAtMs',
|
||||
observedAtMs,
|
||||
0,
|
||||
Number.MAX_SAFE_INTEGER,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'minimumAgeMs',
|
||||
minimumAgeMs,
|
||||
0,
|
||||
Number.MAX_SAFE_INTEGER,
|
||||
);
|
||||
assertIntegerBetween('startShard', startShard, 0, 255);
|
||||
assertIntegerBetween('shardCount', shardCount, 1, MAX_ORPHAN_AUDIT_SHARDS);
|
||||
assertIntegerBetween(
|
||||
'maxEntriesPerShard',
|
||||
maxEntriesPerShard,
|
||||
1,
|
||||
MAX_ORPHAN_AUDIT_ENTRIES_PER_SHARD,
|
||||
);
|
||||
|
||||
const counts = Object.fromEntries(
|
||||
CATEGORIES.map((category) => [category, 0]),
|
||||
) as Record<CompletionReceiptOrphanCategory, number>;
|
||||
const overflowShards: string[] = [];
|
||||
const entries: CompletionReceiptOrphanAuditEntry[] = [];
|
||||
|
||||
for (let offset = 0; offset < shardCount; offset += 1) {
|
||||
const shard = shardName((startShard + offset) % 256);
|
||||
const snapshot = await this.directory.inspectShard(
|
||||
shard,
|
||||
maxEntriesPerShard,
|
||||
);
|
||||
if (snapshot.shard !== shard) {
|
||||
throw new Error('Completion receipt directory returned another shard');
|
||||
}
|
||||
if (snapshot.entries.length > maxEntriesPerShard) {
|
||||
throw new Error('Completion receipt directory exceeded its hard limit');
|
||||
}
|
||||
if (snapshot.overflow) overflowShards.push(shard);
|
||||
|
||||
const attemptIds = snapshot.entries.flatMap((entry) =>
|
||||
entry.kind === 'receipt' && entry.attemptId ? [entry.attemptId] : [],
|
||||
);
|
||||
const ownership = await this.ownership.lookup(attemptIds);
|
||||
for (const entry of snapshot.entries) {
|
||||
const ageMs = Math.max(0, observedAtMs - entry.modifiedAtMs);
|
||||
const classification = classify(
|
||||
entry,
|
||||
entry.attemptId ? ownership.get(entry.attemptId) : undefined,
|
||||
ageMs >= minimumAgeMs,
|
||||
);
|
||||
counts[classification.category] += 1;
|
||||
let action: CompletionReceiptOrphanAction = classification.eligible
|
||||
? 'eligible'
|
||||
: 'retained';
|
||||
let quarantineRef: string | undefined;
|
||||
if (classification.eligible && mode === 'quarantine') {
|
||||
if (snapshot.overflow) {
|
||||
action = 'blocked_overflow';
|
||||
} else {
|
||||
const result = await this.directory.quarantine(entry);
|
||||
action = result.status;
|
||||
if (result.status === 'quarantined') {
|
||||
quarantineRef = result.reference;
|
||||
}
|
||||
}
|
||||
}
|
||||
entries.push({
|
||||
shard,
|
||||
name: entry.name,
|
||||
category: classification.category,
|
||||
action,
|
||||
ageMs,
|
||||
...(entry.attemptId ? { attemptId: entry.attemptId } : {}),
|
||||
...(entry.attemptId && ownership.get(entry.attemptId)?.attemptStatus
|
||||
? {
|
||||
attemptStatus: ownership.get(entry.attemptId)!.attemptStatus,
|
||||
}
|
||||
: {}),
|
||||
...(quarantineRef ? { quarantineRef } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const absoluteNextShard = startShard + shardCount;
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
mode,
|
||||
observedAtMs,
|
||||
minimumAgeMs,
|
||||
startShard: shardName(startShard),
|
||||
nextShard: shardName(absoluteNextShard % 256),
|
||||
wrapped: absoluteNextShard > 255,
|
||||
shardCount,
|
||||
maxEntriesPerShard,
|
||||
scannedEntries: entries.length,
|
||||
overflowShards,
|
||||
entries,
|
||||
counts,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,282 @@
|
||||
import { timingSafeEqual } from 'crypto';
|
||||
import {
|
||||
decodeLocalSecretPlaintext,
|
||||
decryptLocalSecretEnvelopeToBuffer,
|
||||
encryptLocalSecretEnvelope,
|
||||
type LocalSecretNonceFactory,
|
||||
} from '../adapters/crypto/aes256GcmLocalSecret';
|
||||
import {
|
||||
LOCAL_SECRET_ALGORITHM,
|
||||
LocalSecretMutationConflictError,
|
||||
LocalSecretUnavailableError,
|
||||
LocalSecretVersionConflictError,
|
||||
assertLocalSecretMutationId,
|
||||
assertLocalSecretName,
|
||||
assertLocalSecretPlaintext,
|
||||
assertLocalSecretProjectId,
|
||||
assertLocalSecretKeyId,
|
||||
createLocalSecretRef,
|
||||
parseLocalSecretRef,
|
||||
type LocalSecretEnvelope,
|
||||
} from '../domain/localSecret';
|
||||
import { assertRunDispatchCandidate } from '../domain/runDispatchCandidate';
|
||||
import type { LocalSecretEnvelopeRepository } from '../ports/localSecretEnvelopeRepository';
|
||||
import type {
|
||||
LocalSecretEnvironmentProvider,
|
||||
LocalSecretEnvironmentRequest,
|
||||
} from '../ports/localSecretEnvironmentProvider';
|
||||
import type {
|
||||
LocalSecretKeyMaterial,
|
||||
LocalSecretKeyProvider,
|
||||
} from '../ports/localSecretKeyProvider';
|
||||
|
||||
export interface PutEncryptedLocalSecretCommand {
|
||||
projectId: string;
|
||||
name: string;
|
||||
plaintext: string;
|
||||
mutationId: string;
|
||||
expectedCurrentVersion: number;
|
||||
createdAtMs: number;
|
||||
}
|
||||
|
||||
export interface PutEncryptedLocalSecretResult {
|
||||
status: 'inserted' | 'existing';
|
||||
version: number;
|
||||
secretRef: string;
|
||||
}
|
||||
|
||||
export { LocalSecretMutationConflictError, LocalSecretVersionConflictError };
|
||||
|
||||
function assertPutCommand(command: PutEncryptedLocalSecretCommand): void {
|
||||
if (!command || typeof command !== 'object' || Array.isArray(command)) {
|
||||
throw new TypeError('Local Secret write command must be an object');
|
||||
}
|
||||
assertLocalSecretProjectId(command.projectId);
|
||||
assertLocalSecretName(command.name);
|
||||
assertLocalSecretPlaintext(command.plaintext);
|
||||
assertLocalSecretMutationId(command.mutationId);
|
||||
if (
|
||||
!Number.isSafeInteger(command.expectedCurrentVersion) ||
|
||||
command.expectedCurrentVersion < 0 ||
|
||||
command.expectedCurrentVersion >= 2_147_483_647
|
||||
) {
|
||||
throw new TypeError('Local Secret expected current version is invalid');
|
||||
}
|
||||
if (!Number.isSafeInteger(command.createdAtMs) || command.createdAtMs < 0) {
|
||||
throw new TypeError('Local Secret creation time is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function ownedKeyMaterial(
|
||||
material: LocalSecretKeyMaterial | null,
|
||||
expectedKeyId?: string,
|
||||
): { keyId: string; key: Buffer } {
|
||||
if (!material || !(material.key instanceof Uint8Array)) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
try {
|
||||
assertLocalSecretKeyId(material.keyId);
|
||||
if (
|
||||
(expectedKeyId !== undefined && material.keyId !== expectedKeyId) ||
|
||||
material.key.byteLength !== 32
|
||||
) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
return { keyId: material.keyId, key: Buffer.from(material.key) };
|
||||
} catch {
|
||||
throw new LocalSecretUnavailableError();
|
||||
} finally {
|
||||
material.key.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
function plaintextMatches(
|
||||
envelope: LocalSecretEnvelope,
|
||||
key: Uint8Array,
|
||||
expected: string,
|
||||
): boolean {
|
||||
const actual = decryptLocalSecretEnvelopeToBuffer(envelope, key);
|
||||
const wanted = Buffer.from(expected, 'utf8');
|
||||
try {
|
||||
return actual.length === wanted.length && timingSafeEqual(actual, wanted);
|
||||
} finally {
|
||||
actual.fill(0);
|
||||
wanted.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
export class EncryptedLocalSecretService
|
||||
implements LocalSecretEnvironmentProvider
|
||||
{
|
||||
constructor(
|
||||
private readonly envelopes: LocalSecretEnvelopeRepository,
|
||||
private readonly keys: LocalSecretKeyProvider,
|
||||
private readonly nonceFactory?: LocalSecretNonceFactory,
|
||||
) {}
|
||||
|
||||
async put(
|
||||
command: PutEncryptedLocalSecretCommand,
|
||||
): Promise<PutEncryptedLocalSecretResult> {
|
||||
assertPutCommand(command);
|
||||
try {
|
||||
return await this.putValidated(command);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof LocalSecretVersionConflictError ||
|
||||
error instanceof LocalSecretMutationConflictError ||
|
||||
error instanceof LocalSecretUnavailableError
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
private async putValidated(
|
||||
command: PutEncryptedLocalSecretCommand,
|
||||
): Promise<PutEncryptedLocalSecretResult> {
|
||||
const existing = await this.envelopes.findByMutation(
|
||||
command.projectId,
|
||||
command.name,
|
||||
command.mutationId,
|
||||
);
|
||||
if (existing) {
|
||||
const material = ownedKeyMaterial(
|
||||
await this.keys.resolve(existing.keyId),
|
||||
existing.keyId,
|
||||
);
|
||||
try {
|
||||
if (
|
||||
existing.version !== command.expectedCurrentVersion + 1 ||
|
||||
!plaintextMatches(existing, material.key, command.plaintext)
|
||||
) {
|
||||
throw new LocalSecretMutationConflictError();
|
||||
}
|
||||
} finally {
|
||||
material.key.fill(0);
|
||||
}
|
||||
return this.result('existing', existing);
|
||||
}
|
||||
|
||||
const material = ownedKeyMaterial(await this.keys.active());
|
||||
try {
|
||||
const envelope = encryptLocalSecretEnvelope(
|
||||
{
|
||||
projectId: command.projectId,
|
||||
name: command.name,
|
||||
version: command.expectedCurrentVersion + 1,
|
||||
mutationId: command.mutationId,
|
||||
keyId: material.keyId,
|
||||
algorithm: LOCAL_SECRET_ALGORITHM,
|
||||
createdAtMs: command.createdAtMs,
|
||||
},
|
||||
command.plaintext,
|
||||
material.key,
|
||||
this.nonceFactory,
|
||||
);
|
||||
const appended = await this.envelopes.append({
|
||||
envelope,
|
||||
expectedCurrentVersion: command.expectedCurrentVersion,
|
||||
});
|
||||
if (appended.status === 'existing') {
|
||||
const existingMaterial =
|
||||
appended.envelope.keyId === material.keyId
|
||||
? material
|
||||
: ownedKeyMaterial(
|
||||
await this.keys.resolve(appended.envelope.keyId),
|
||||
appended.envelope.keyId,
|
||||
);
|
||||
try {
|
||||
if (
|
||||
appended.envelope.version !== command.expectedCurrentVersion + 1 ||
|
||||
!plaintextMatches(
|
||||
appended.envelope,
|
||||
existingMaterial.key,
|
||||
command.plaintext,
|
||||
)
|
||||
) {
|
||||
throw new LocalSecretMutationConflictError();
|
||||
}
|
||||
} finally {
|
||||
if (existingMaterial !== material) existingMaterial.key.fill(0);
|
||||
}
|
||||
}
|
||||
return this.result(appended.status, appended.envelope);
|
||||
} finally {
|
||||
material.key.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
async resolve(
|
||||
request: Readonly<LocalSecretEnvironmentRequest>,
|
||||
): Promise<readonly string[] | null> {
|
||||
const cachedKeys = new Map<string, Buffer>();
|
||||
try {
|
||||
if (!request || typeof request !== 'object' || Array.isArray(request)) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
assertRunDispatchCandidate(request.candidate);
|
||||
if (
|
||||
!Array.isArray(request.secretRefs) ||
|
||||
request.secretRefs.length > 64
|
||||
) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
const references = request.secretRefs.map(parseLocalSecretRef);
|
||||
if (
|
||||
references.some(
|
||||
(reference) => reference.projectId !== request.candidate.projectId,
|
||||
)
|
||||
) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
const envelopes = await this.envelopes.resolveMany(references);
|
||||
if (
|
||||
envelopes.length !== references.length ||
|
||||
envelopes.some((item) => !item)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const plaintext: string[] = [];
|
||||
for (const envelope of envelopes as readonly LocalSecretEnvelope[]) {
|
||||
let key = cachedKeys.get(envelope.keyId);
|
||||
if (!key) {
|
||||
const material = ownedKeyMaterial(
|
||||
await this.keys.resolve(envelope.keyId),
|
||||
envelope.keyId,
|
||||
);
|
||||
key = material.key;
|
||||
cachedKeys.set(envelope.keyId, key);
|
||||
}
|
||||
const bytes = decryptLocalSecretEnvelopeToBuffer(envelope, key);
|
||||
try {
|
||||
plaintext.push(decodeLocalSecretPlaintext(bytes));
|
||||
} finally {
|
||||
bytes.fill(0);
|
||||
}
|
||||
}
|
||||
return Object.freeze(plaintext);
|
||||
} catch (error) {
|
||||
if (error instanceof LocalSecretUnavailableError) throw error;
|
||||
throw new LocalSecretUnavailableError();
|
||||
} finally {
|
||||
for (const key of cachedKeys.values()) key.fill(0);
|
||||
cachedKeys.clear();
|
||||
}
|
||||
}
|
||||
|
||||
private result(
|
||||
status: PutEncryptedLocalSecretResult['status'],
|
||||
envelope: LocalSecretEnvelope,
|
||||
): PutEncryptedLocalSecretResult {
|
||||
return Object.freeze({
|
||||
status,
|
||||
version: envelope.version,
|
||||
secretRef: createLocalSecretRef({
|
||||
projectId: envelope.projectId,
|
||||
name: envelope.name,
|
||||
version: envelope.version,
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import type { DeploymentProfile } from '../domain/deploymentProfile';
|
||||
import type { WorkerRecord } from '../domain/worker';
|
||||
import type { WorkerExecutionDrainer } from '../ports/workerExecutionDrainer';
|
||||
import type {
|
||||
WorkerHeartbeatLifecycle,
|
||||
WorkerHeartbeatStopResult,
|
||||
} from './workerHeartbeatLifecycle';
|
||||
|
||||
export type HeadlessWorkerStopResult =
|
||||
| 'stopped'
|
||||
| 'not_started'
|
||||
| 'executions_timed_out'
|
||||
| 'heartbeat_timed_out'
|
||||
| 'heartbeat_disconnect_failed';
|
||||
|
||||
export interface HeadlessWorkerBootstrapOptions {
|
||||
enabled?: boolean;
|
||||
profile: DeploymentProfile;
|
||||
heartbeat: WorkerHeartbeatLifecycle;
|
||||
executions: WorkerExecutionDrainer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Independent Worker boot topology. It owns no HTTP panel, Scheduler, SQLite
|
||||
* control-plane repository, or local Primary router. Shutdown first advertises
|
||||
* zero capacity, then waits for the execution plane, and only then marks the
|
||||
* Worker session offline.
|
||||
*/
|
||||
export class HeadlessWorkerRuntime {
|
||||
private started = false;
|
||||
|
||||
constructor(
|
||||
private readonly heartbeat: WorkerHeartbeatLifecycle,
|
||||
private readonly executions: WorkerExecutionDrainer,
|
||||
) {}
|
||||
|
||||
currentSession(): WorkerRecord | undefined {
|
||||
return this.heartbeat.currentSession();
|
||||
}
|
||||
|
||||
async start(): Promise<boolean> {
|
||||
if (this.started) return false;
|
||||
const started = await this.heartbeat.start();
|
||||
this.started = started;
|
||||
return started;
|
||||
}
|
||||
|
||||
async drainAndStop(): Promise<HeadlessWorkerStopResult> {
|
||||
if (!this.started) return 'not_started';
|
||||
await this.heartbeat.drain();
|
||||
if ((await this.executions.drain()) === 'timed_out') {
|
||||
return 'executions_timed_out';
|
||||
}
|
||||
const heartbeatResult: WorkerHeartbeatStopResult =
|
||||
await this.heartbeat.stop();
|
||||
if (heartbeatResult === 'timed_out') return 'heartbeat_timed_out';
|
||||
if (heartbeatResult === 'disconnect_failed') {
|
||||
return 'heartbeat_disconnect_failed';
|
||||
}
|
||||
this.started = false;
|
||||
return 'stopped';
|
||||
}
|
||||
}
|
||||
|
||||
export type HeadlessWorkerBootstrapResult =
|
||||
| { status: 'disabled' }
|
||||
| { status: 'active'; runtime: HeadlessWorkerRuntime };
|
||||
|
||||
export async function bootstrapHeadlessWorkerRuntime({
|
||||
enabled = false,
|
||||
profile,
|
||||
heartbeat,
|
||||
executions,
|
||||
}: HeadlessWorkerBootstrapOptions): Promise<HeadlessWorkerBootstrapResult> {
|
||||
if (!enabled) return { status: 'disabled' };
|
||||
if (profile !== 'worker') {
|
||||
throw new TypeError(
|
||||
`Deployment profile ${profile} cannot activate the headless Worker runtime`,
|
||||
);
|
||||
}
|
||||
const runtime = new HeadlessWorkerRuntime(heartbeat, executions);
|
||||
if (!(await runtime.start())) {
|
||||
throw new Error('Headless Worker runtime did not start');
|
||||
}
|
||||
return { status: 'active', runtime };
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
import { createHash } from 'crypto';
|
||||
import jwt, { type JwtPayload } from 'jsonwebtoken';
|
||||
import {
|
||||
assertAuthenticatedPrincipalActive,
|
||||
normalizeAuthenticatedPrincipal,
|
||||
type AuthenticatedPrincipal,
|
||||
} from '../domain/authenticatedPrincipal';
|
||||
import {
|
||||
IdentityDirectoryUnavailableError,
|
||||
LEGACY_PANEL_IDENTITY_PROVIDER,
|
||||
LEGACY_PANEL_PROVIDER_SUBJECT,
|
||||
} from '../domain/identityDirectory';
|
||||
import type { IdentityDirectoryRepository } from '../ports/identityDirectoryRepository';
|
||||
import type {
|
||||
LegacyPanelPlatform,
|
||||
LegacyPanelSessionSource,
|
||||
} from '../ports/legacyPanelSessionSource';
|
||||
|
||||
const LEGACY_PANEL_JWT_PATTERN =
|
||||
/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/;
|
||||
const MAX_LEGACY_PANEL_TOKEN_LENGTH = 4096;
|
||||
const MAX_LEGACY_PANEL_JWT_DATA_LENGTH = 256;
|
||||
|
||||
export interface AuthenticateLegacyPanelSessionRequest {
|
||||
token: string;
|
||||
platform: LegacyPanelPlatform;
|
||||
nowMs: number;
|
||||
}
|
||||
|
||||
export class LegacyPanelAuthenticationRejectedError extends Error {
|
||||
readonly code = 'LEGACY_PANEL_AUTHENTICATION_REJECTED';
|
||||
|
||||
constructor() {
|
||||
super('Legacy panel authentication was rejected');
|
||||
this.name = 'LegacyPanelAuthenticationRejectedError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacyPanelAuthenticationUnavailableError extends Error {
|
||||
readonly code = 'LEGACY_PANEL_AUTHENTICATION_UNAVAILABLE';
|
||||
|
||||
constructor() {
|
||||
super('Legacy panel authentication is unavailable');
|
||||
this.name = 'LegacyPanelAuthenticationUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function assertExactRequest(request: AuthenticateLegacyPanelSessionRequest) {
|
||||
if (!request || typeof request !== 'object' || Array.isArray(request)) {
|
||||
throw new TypeError('Legacy panel authentication request is invalid');
|
||||
}
|
||||
const keys = Object.keys(request).sort();
|
||||
const expected = ['nowMs', 'platform', 'token'];
|
||||
if (
|
||||
keys.length !== expected.length ||
|
||||
keys.some((key, index) => key !== expected[index])
|
||||
) {
|
||||
throw new TypeError('Legacy panel authentication request shape is invalid');
|
||||
}
|
||||
if (
|
||||
typeof request.token !== 'string' ||
|
||||
request.token.length < 1 ||
|
||||
request.token.length > MAX_LEGACY_PANEL_TOKEN_LENGTH ||
|
||||
!LEGACY_PANEL_JWT_PATTERN.test(request.token)
|
||||
) {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
if (request.platform !== 'desktop' && request.platform !== 'mobile') {
|
||||
throw new TypeError('Legacy panel platform is invalid');
|
||||
}
|
||||
if (!Number.isSafeInteger(request.nowMs) || request.nowMs < 0) {
|
||||
throw new TypeError('Legacy panel authentication time is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function normalizePayload(value: string | JwtPayload): {
|
||||
authenticatedAtMs: number;
|
||||
expiresAtMs: number;
|
||||
} {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
const keys = Object.keys(value).sort();
|
||||
const expected = ['data', 'exp', 'iat'];
|
||||
if (
|
||||
keys.length !== expected.length ||
|
||||
keys.some((key, index) => key !== expected[index])
|
||||
) {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
if (
|
||||
typeof value.data !== 'string' ||
|
||||
value.data.length < 1 ||
|
||||
value.data.length > MAX_LEGACY_PANEL_JWT_DATA_LENGTH ||
|
||||
!Number.isSafeInteger(value.iat) ||
|
||||
value.iat! < 0 ||
|
||||
!Number.isSafeInteger(value.exp) ||
|
||||
value.exp! <= value.iat!
|
||||
) {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
const authenticatedAtMs = value.iat! * 1000;
|
||||
const expiresAtMs = value.exp! * 1000;
|
||||
if (
|
||||
!Number.isSafeInteger(authenticatedAtMs) ||
|
||||
!Number.isSafeInteger(expiresAtMs)
|
||||
) {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
return { authenticatedAtMs, expiresAtMs };
|
||||
}
|
||||
|
||||
export class LegacyPanelAuthenticationService {
|
||||
constructor(
|
||||
private readonly identityDirectory: IdentityDirectoryRepository,
|
||||
private readonly sessions: LegacyPanelSessionSource,
|
||||
private readonly jwtSecret: string,
|
||||
) {
|
||||
if (
|
||||
typeof jwtSecret !== 'string' ||
|
||||
jwtSecret.length < 1 ||
|
||||
jwtSecret.length > 4096
|
||||
) {
|
||||
throw new TypeError('Legacy panel JWT secret is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
async authenticate(
|
||||
request: AuthenticateLegacyPanelSessionRequest,
|
||||
): Promise<Readonly<AuthenticatedPrincipal>> {
|
||||
assertExactRequest(request);
|
||||
let payload: { authenticatedAtMs: number; expiresAtMs: number };
|
||||
try {
|
||||
payload = normalizePayload(
|
||||
jwt.verify(request.token, this.jwtSecret, {
|
||||
algorithms: ['HS384'],
|
||||
clockTimestamp: Math.floor(request.nowMs / 1000),
|
||||
clockTolerance: 0,
|
||||
}),
|
||||
);
|
||||
} catch {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
if (
|
||||
payload.authenticatedAtMs > request.nowMs ||
|
||||
payload.expiresAtMs <= request.nowMs
|
||||
) {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
|
||||
let active: boolean;
|
||||
try {
|
||||
active = await this.sessions.isActive(request.token, request.platform);
|
||||
} catch {
|
||||
throw new LegacyPanelAuthenticationUnavailableError();
|
||||
}
|
||||
if (!active) throw new LegacyPanelAuthenticationRejectedError();
|
||||
|
||||
let subject;
|
||||
try {
|
||||
subject = await this.identityDirectory.resolveAuthenticationSubject(
|
||||
LEGACY_PANEL_IDENTITY_PROVIDER,
|
||||
LEGACY_PANEL_PROVIDER_SUBJECT,
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof IdentityDirectoryUnavailableError) {
|
||||
throw new LegacyPanelAuthenticationUnavailableError();
|
||||
}
|
||||
throw new LegacyPanelAuthenticationUnavailableError();
|
||||
}
|
||||
if (!subject || subject.type !== 'user') {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
|
||||
const principal = normalizeAuthenticatedPrincipal({
|
||||
subject,
|
||||
authenticationId: `legacy_panel:${createHash('sha256')
|
||||
.update(request.token, 'utf8')
|
||||
.digest('hex')}`,
|
||||
authenticatedAtMs: payload.authenticatedAtMs,
|
||||
expiresAtMs: payload.expiresAtMs,
|
||||
assurance: 'single_factor',
|
||||
});
|
||||
try {
|
||||
assertAuthenticatedPrincipalActive(principal, request.nowMs);
|
||||
} catch {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
return principal;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
import type { ExecutionOrigin } from '../domain/run';
|
||||
import { selectOneLegacyExecution } from '../domain/legacyExecutionSelection';
|
||||
import type {
|
||||
LegacyExecutionCallbackFact,
|
||||
LegacyExecutionCancellationFact,
|
||||
LegacyExecutionSelector,
|
||||
} from '../ports/legacyExecutionCorrelation';
|
||||
import type {
|
||||
ActiveLegacyShadowRun,
|
||||
LegacyShadowRunLocator,
|
||||
} from '../ports/legacyShadowRunLocator';
|
||||
import type { LegacyShadowRunWriter } from './legacyShadowRunWriter';
|
||||
|
||||
export type LegacyCorrelationOperation =
|
||||
| 'cancel_all'
|
||||
| 'cancel_one'
|
||||
| 'callback';
|
||||
export type LegacyCorrelationFailureReason =
|
||||
| 'ambiguous'
|
||||
| 'truncated'
|
||||
| 'unmatched'
|
||||
| 'write_failed';
|
||||
|
||||
export interface LegacyCorrelationFailure {
|
||||
operation: LegacyCorrelationOperation;
|
||||
reason: LegacyCorrelationFailureReason;
|
||||
legacyCronId: number;
|
||||
candidateCount: number;
|
||||
}
|
||||
|
||||
export interface LegacyCorrelationReporter {
|
||||
failure(failure: LegacyCorrelationFailure): void;
|
||||
}
|
||||
|
||||
export interface LegacyCorrelationResult {
|
||||
matched: number;
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
export class LegacyShadowRunCorrelator {
|
||||
constructor(
|
||||
private readonly locator: LegacyShadowRunLocator,
|
||||
private readonly writer: LegacyShadowRunWriter,
|
||||
private readonly reporter: LegacyCorrelationReporter,
|
||||
) {}
|
||||
|
||||
async cancel(
|
||||
fact: LegacyExecutionCancellationFact,
|
||||
origins: readonly ExecutionOrigin[],
|
||||
): Promise<LegacyCorrelationResult> {
|
||||
const lookup = await this.locator.listActiveByLegacyCron({
|
||||
legacyCronId: fact.legacyCronId,
|
||||
origins,
|
||||
});
|
||||
const candidates =
|
||||
fact.scope === 'all'
|
||||
? [...lookup.candidates]
|
||||
: this.selectOne(lookup.candidates, fact);
|
||||
if (lookup.truncated) {
|
||||
this.report({
|
||||
operation: fact.scope === 'all' ? 'cancel_all' : 'cancel_one',
|
||||
reason: 'truncated',
|
||||
legacyCronId: fact.legacyCronId,
|
||||
candidateCount: lookup.candidates.length,
|
||||
});
|
||||
}
|
||||
if (fact.scope === 'one' && candidates.length !== 1) {
|
||||
this.reportSelectionFailure('cancel_one', fact, lookup.candidates);
|
||||
return { matched: 0, truncated: lookup.truncated };
|
||||
}
|
||||
|
||||
let matched = 0;
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
await this.writer.cancelled(
|
||||
{ runId: candidate.runId, attemptId: candidate.attemptId },
|
||||
{ atMs: fact.atMs, reason: fact.reason },
|
||||
);
|
||||
matched += 1;
|
||||
} catch {
|
||||
this.report({
|
||||
operation: fact.scope === 'all' ? 'cancel_all' : 'cancel_one',
|
||||
reason: 'write_failed',
|
||||
legacyCronId: fact.legacyCronId,
|
||||
candidateCount: 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
return { matched, truncated: lookup.truncated };
|
||||
}
|
||||
|
||||
async callback(
|
||||
fact: LegacyExecutionCallbackFact,
|
||||
origins: readonly ExecutionOrigin[],
|
||||
): Promise<LegacyCorrelationResult> {
|
||||
const lookup = await this.locator.listActiveByLegacyCron({
|
||||
legacyCronId: fact.legacyCronId,
|
||||
origins,
|
||||
});
|
||||
const candidates = this.selectOne(lookup.candidates, fact);
|
||||
if (lookup.truncated) {
|
||||
this.report({
|
||||
operation: 'callback',
|
||||
reason: 'truncated',
|
||||
legacyCronId: fact.legacyCronId,
|
||||
candidateCount: lookup.candidates.length,
|
||||
});
|
||||
}
|
||||
if (candidates.length !== 1) {
|
||||
this.reportSelectionFailure('callback', fact, lookup.candidates);
|
||||
return { matched: 0, truncated: lookup.truncated };
|
||||
}
|
||||
|
||||
const [candidate] = candidates;
|
||||
const reference = {
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
};
|
||||
try {
|
||||
if (fact.phase === 'running') {
|
||||
await this.writer.spawned(reference, {
|
||||
atMs: fact.atMs,
|
||||
...(fact.pid === undefined ? {} : { pid: fact.pid }),
|
||||
...(fact.logArtifactId === undefined
|
||||
? {}
|
||||
: { logArtifactId: fact.logArtifactId }),
|
||||
});
|
||||
await this.writer.running(reference, fact.atMs);
|
||||
} else {
|
||||
await this.writer.spawned(reference, {
|
||||
atMs: fact.atMs,
|
||||
...(fact.pid === undefined ? {} : { pid: fact.pid }),
|
||||
...(fact.logArtifactId === undefined
|
||||
? {}
|
||||
: { logArtifactId: fact.logArtifactId }),
|
||||
});
|
||||
await this.writer.exited(reference, {
|
||||
atMs: fact.atMs,
|
||||
exitCode: fact.exitCode ?? 0,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
this.report({
|
||||
operation: 'callback',
|
||||
reason: 'write_failed',
|
||||
legacyCronId: fact.legacyCronId,
|
||||
candidateCount: 1,
|
||||
});
|
||||
return { matched: 0, truncated: lookup.truncated };
|
||||
}
|
||||
return { matched: 1, truncated: lookup.truncated };
|
||||
}
|
||||
|
||||
private selectOne(
|
||||
candidates: readonly ActiveLegacyShadowRun[],
|
||||
selector: LegacyExecutionSelector,
|
||||
): ActiveLegacyShadowRun[] {
|
||||
return selectOneLegacyExecution(candidates, selector);
|
||||
}
|
||||
|
||||
private reportSelectionFailure(
|
||||
operation: LegacyCorrelationOperation,
|
||||
selector: LegacyExecutionSelector,
|
||||
candidates: readonly ActiveLegacyShadowRun[],
|
||||
): void {
|
||||
this.report({
|
||||
operation,
|
||||
reason: candidates.length === 0 ? 'unmatched' : 'ambiguous',
|
||||
legacyCronId: selector.legacyCronId,
|
||||
candidateCount: candidates.length,
|
||||
});
|
||||
}
|
||||
|
||||
private report(failure: LegacyCorrelationFailure): void {
|
||||
try {
|
||||
this.reporter.failure(failure);
|
||||
} catch {
|
||||
// Correlation diagnostics must not affect legacy execution.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
import type { ExecutionOrigin } from '../domain/run';
|
||||
import type { RuntimeRolloutPolicy } from '../domain/runtimeRollout';
|
||||
import type {
|
||||
LegacyExecutionAcceptedFact,
|
||||
LegacyExecutionCancelledFact,
|
||||
LegacyExecutionExitedFact,
|
||||
LegacyExecutionObservation,
|
||||
LegacyExecutionObserver,
|
||||
LegacyExecutionRunningFact,
|
||||
LegacyExecutionSpawnedFact,
|
||||
LegacyExecutionStartFailedFact,
|
||||
} from '../ports/legacyExecutionObserver';
|
||||
import type {
|
||||
LegacyShadowRunReference,
|
||||
LegacyShadowRunWriter,
|
||||
} from './legacyShadowRunWriter';
|
||||
|
||||
export type ShadowObservationOperation =
|
||||
| 'accept'
|
||||
| 'spawned'
|
||||
| 'running'
|
||||
| 'start_failed'
|
||||
| 'exited'
|
||||
| 'cancelled';
|
||||
|
||||
export interface ShadowObservationFailure {
|
||||
origin: ExecutionOrigin;
|
||||
operation: ShadowObservationOperation;
|
||||
errorCode: string;
|
||||
runId?: string;
|
||||
attemptId?: string;
|
||||
}
|
||||
|
||||
export interface ShadowObservationReporter {
|
||||
failure(failure: ShadowObservationFailure): void;
|
||||
}
|
||||
|
||||
export interface TrackedLegacyExecutionObservation
|
||||
extends LegacyExecutionObservation {
|
||||
settled(): Promise<void>;
|
||||
}
|
||||
|
||||
const NOOP_OBSERVATION: TrackedLegacyExecutionObservation = Object.freeze({
|
||||
spawned() {},
|
||||
running() {},
|
||||
startFailed() {},
|
||||
exited() {},
|
||||
cancelled() {},
|
||||
async settled() {},
|
||||
});
|
||||
|
||||
function classifyError(error: unknown): string {
|
||||
if (
|
||||
error instanceof Error &&
|
||||
'code' in error &&
|
||||
typeof (error as Error & { code?: unknown }).code === 'string'
|
||||
) {
|
||||
const code = (error as Error & { code: string }).code;
|
||||
return /^[A-Z0-9_]{1,64}$/.test(code) ? code : 'SHADOW_UNKNOWN';
|
||||
}
|
||||
return 'SHADOW_UNKNOWN';
|
||||
}
|
||||
|
||||
class SerialLegacyExecutionObservation
|
||||
implements TrackedLegacyExecutionObservation
|
||||
{
|
||||
private chain: Promise<LegacyShadowRunReference | null>;
|
||||
|
||||
constructor(
|
||||
private readonly origin: ExecutionOrigin,
|
||||
private readonly writer: LegacyShadowRunWriter,
|
||||
accepted: LegacyExecutionAcceptedFact,
|
||||
private readonly reporter: ShadowObservationReporter,
|
||||
) {
|
||||
this.chain = writer.accept(accepted).catch((error) => {
|
||||
this.report('accept', error);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
spawned(fact: LegacyExecutionSpawnedFact): void {
|
||||
this.enqueue('spawned', (reference) =>
|
||||
this.writer.spawned(reference, fact),
|
||||
);
|
||||
}
|
||||
|
||||
running(fact: LegacyExecutionRunningFact): void {
|
||||
this.enqueue('running', (reference) =>
|
||||
this.writer.running(reference, fact.atMs),
|
||||
);
|
||||
}
|
||||
|
||||
startFailed(fact: LegacyExecutionStartFailedFact): void {
|
||||
this.enqueue('start_failed', (reference) =>
|
||||
this.writer.startFailed(reference, fact),
|
||||
);
|
||||
}
|
||||
|
||||
exited(fact: LegacyExecutionExitedFact): void {
|
||||
this.enqueue('exited', (reference) => this.writer.exited(reference, fact));
|
||||
}
|
||||
|
||||
cancelled(fact: LegacyExecutionCancelledFact): void {
|
||||
this.enqueue('cancelled', (reference) =>
|
||||
this.writer.cancelled(reference, fact),
|
||||
);
|
||||
}
|
||||
|
||||
async settled(): Promise<void> {
|
||||
await this.chain;
|
||||
}
|
||||
|
||||
private enqueue(
|
||||
operation: ShadowObservationOperation,
|
||||
write: (reference: LegacyShadowRunReference) => Promise<void>,
|
||||
): void {
|
||||
this.chain = this.chain.then(async (reference) => {
|
||||
if (!reference) return null;
|
||||
try {
|
||||
await write(reference);
|
||||
} catch (error) {
|
||||
this.report(operation, error, reference);
|
||||
}
|
||||
return reference;
|
||||
});
|
||||
}
|
||||
|
||||
private report(
|
||||
operation: ShadowObservationOperation,
|
||||
error: unknown,
|
||||
reference?: LegacyShadowRunReference,
|
||||
): void {
|
||||
try {
|
||||
this.reporter.failure({
|
||||
origin: this.origin,
|
||||
operation,
|
||||
errorCode: classifyError(error),
|
||||
...(reference === undefined ? {} : reference),
|
||||
});
|
||||
} catch {
|
||||
// Shadow reporting must not become a second failure path.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacyShadowRunObserver implements LegacyExecutionObserver {
|
||||
constructor(
|
||||
private readonly policy: RuntimeRolloutPolicy,
|
||||
private readonly writer: LegacyShadowRunWriter,
|
||||
private readonly reporter: ShadowObservationReporter,
|
||||
) {}
|
||||
|
||||
begin(fact: LegacyExecutionAcceptedFact): TrackedLegacyExecutionObservation {
|
||||
const decision = this.policy.decide(fact.origin);
|
||||
if (decision.mode === 'off') return NOOP_OBSERVATION;
|
||||
if (decision.mode === 'primary') {
|
||||
throw new Error(
|
||||
'Legacy observer cannot accept a Runtime-owned primary execution',
|
||||
);
|
||||
}
|
||||
return new SerialLegacyExecutionObservation(
|
||||
fact.origin,
|
||||
this.writer,
|
||||
fact,
|
||||
this.reporter,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,546 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunAttemptStatus,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
RunStatus,
|
||||
} from '../domain/run';
|
||||
import {
|
||||
isTerminalRunAttemptStatus,
|
||||
isTerminalRunStatus,
|
||||
reserveRunEvent,
|
||||
transitionRun,
|
||||
transitionRunAttempt,
|
||||
type RunAttemptTransitionCommand,
|
||||
type RunAttemptTransitionDecision,
|
||||
type RunDomainEventDraft,
|
||||
type RunTransitionCommand,
|
||||
type RunTransitionDecision,
|
||||
} from '../domain/runStateMachine';
|
||||
import { RunVersionConflictError } from '../domain/stateMachineErrors';
|
||||
import type {
|
||||
LegacyExecutionAcceptedFact,
|
||||
LegacyExecutionCancelledFact,
|
||||
LegacyExecutionExitedFact,
|
||||
LegacyExecutionSpawnedFact,
|
||||
LegacyExecutionStartFailedFact,
|
||||
} from '../ports/legacyExecutionObserver';
|
||||
import type {
|
||||
RunRepository,
|
||||
RunRepositoryTransaction,
|
||||
} from '../ports/runRepository';
|
||||
import {
|
||||
RunAttemptConcurrentWriteError,
|
||||
RunAttemptNotFoundError,
|
||||
RunNotFoundError,
|
||||
} from './commandErrors';
|
||||
|
||||
export interface LegacyShadowRunReference {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
}
|
||||
|
||||
export type ShadowIdFactory = () => string;
|
||||
|
||||
export class LegacyShadowRunWriter {
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly createId: ShadowIdFactory = uuidV7,
|
||||
) {}
|
||||
|
||||
async accept(
|
||||
fact: LegacyExecutionAcceptedFact,
|
||||
): Promise<LegacyShadowRunReference> {
|
||||
const reference = {
|
||||
runId: this.createId(),
|
||||
attemptId: this.createId(),
|
||||
};
|
||||
const initialRun: RunRecord = {
|
||||
id: reference.runId,
|
||||
projectId: fact.projectId,
|
||||
taskId: fact.taskId,
|
||||
taskRevision: fact.taskRevision,
|
||||
...(fact.taskName === undefined ? {} : { taskName: fact.taskName }),
|
||||
...(fact.legacyCronId === undefined
|
||||
? {}
|
||||
: { legacyCronId: fact.legacyCronId }),
|
||||
triggerType: fact.triggerType,
|
||||
executionOrigin: fact.origin,
|
||||
executionOwner: 'legacy',
|
||||
...(fact.triggeredBy === undefined
|
||||
? {}
|
||||
: { triggeredBy: fact.triggeredBy }),
|
||||
...(fact.requestId === undefined ? {} : { requestId: fact.requestId }),
|
||||
...(fact.scheduledForMs === undefined
|
||||
? {}
|
||||
: { scheduledForMs: fact.scheduledForMs }),
|
||||
status: 'created',
|
||||
version: 0,
|
||||
eventSequence: 0,
|
||||
priority: 0,
|
||||
createdAtMs: fact.acceptedAtMs,
|
||||
};
|
||||
const initialAttempt: RunAttemptRecord = {
|
||||
id: reference.attemptId,
|
||||
runId: reference.runId,
|
||||
attempt: 1,
|
||||
status: 'claimed',
|
||||
executorType: 'legacy_local',
|
||||
callbackSequence: 0,
|
||||
createdAtMs: fact.acceptedAtMs,
|
||||
};
|
||||
|
||||
await this.repository.transaction(async (transaction) => {
|
||||
await transaction.insertRun(initialRun);
|
||||
await transaction.insertAttempt(initialAttempt);
|
||||
|
||||
const created = reserveRunEvent(initialRun, 0);
|
||||
const createdRun = created.run;
|
||||
const createdUpdated = await transaction.compareAndSetRun(createdRun, 0);
|
||||
if (!createdUpdated) {
|
||||
throw new RunVersionConflictError(initialRun.id, 0, initialRun.version);
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
this.event(
|
||||
createdRun,
|
||||
{
|
||||
sequence: created.sequence,
|
||||
type: 'run.created',
|
||||
payload: {
|
||||
status: 'created',
|
||||
version: createdRun.version,
|
||||
execution_owner: 'legacy',
|
||||
shadow: true,
|
||||
},
|
||||
},
|
||||
fact.acceptedAtMs,
|
||||
),
|
||||
);
|
||||
|
||||
const queued = transitionRun(createdRun, {
|
||||
to: 'queued',
|
||||
expectedVersion: createdRun.version,
|
||||
atMs: fact.acceptedAtMs,
|
||||
});
|
||||
await this.persistRunDecision(
|
||||
transaction,
|
||||
createdRun,
|
||||
queued,
|
||||
fact.acceptedAtMs,
|
||||
);
|
||||
});
|
||||
return reference;
|
||||
}
|
||||
|
||||
async spawned(
|
||||
reference: LegacyShadowRunReference,
|
||||
fact: LegacyExecutionSpawnedFact,
|
||||
): Promise<void> {
|
||||
await this.repository.transaction(async (transaction) => {
|
||||
const aggregate = await this.load(transaction, reference);
|
||||
await this.ensureSpawned(
|
||||
transaction,
|
||||
aggregate.run,
|
||||
aggregate.attempt,
|
||||
fact,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async running(
|
||||
reference: LegacyShadowRunReference,
|
||||
atMs: number,
|
||||
): Promise<void> {
|
||||
await this.repository.transaction(async (transaction) => {
|
||||
let { run, attempt } = await this.load(transaction, reference);
|
||||
if (isTerminalRunStatus(run.status)) return;
|
||||
({ run, attempt } = await this.ensureSpawned(transaction, run, attempt, {
|
||||
atMs,
|
||||
}));
|
||||
await this.ensureRunning(transaction, run, attempt, atMs);
|
||||
});
|
||||
}
|
||||
|
||||
async startFailed(
|
||||
reference: LegacyShadowRunReference,
|
||||
fact: LegacyExecutionStartFailedFact,
|
||||
): Promise<void> {
|
||||
await this.repository.transaction(async (transaction) => {
|
||||
let { run, attempt } = await this.load(transaction, reference);
|
||||
if (isTerminalRunStatus(run.status)) return;
|
||||
({ run, attempt } = await this.ensureSpawned(transaction, run, attempt, {
|
||||
atMs: fact.atMs,
|
||||
}));
|
||||
|
||||
if (!isTerminalRunAttemptStatus(attempt.status)) {
|
||||
if (attempt.status === 'claimed') {
|
||||
({ run, attempt } = await this.transitionAttempt(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
{
|
||||
to: 'starting',
|
||||
expectedRunVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
},
|
||||
));
|
||||
}
|
||||
if (attempt.status === 'starting' || attempt.status === 'running') {
|
||||
({ run, attempt } = await this.transitionAttempt(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
{
|
||||
to: 'failed',
|
||||
expectedRunVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
errorCode: fact.errorCode,
|
||||
errorSummary: 'Legacy process failed to start',
|
||||
},
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if (!isTerminalRunStatus(run.status)) {
|
||||
await this.transitionRunStatus(transaction, run, {
|
||||
to: 'failed',
|
||||
expectedVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
errorCode: fact.errorCode,
|
||||
errorSummary: 'Legacy process failed to start',
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async exited(
|
||||
reference: LegacyShadowRunReference,
|
||||
fact: LegacyExecutionExitedFact,
|
||||
): Promise<void> {
|
||||
await this.repository.transaction(async (transaction) => {
|
||||
let { run, attempt } = await this.load(transaction, reference);
|
||||
if (isTerminalRunStatus(run.status)) return;
|
||||
({ run, attempt } = await this.ensureSpawned(transaction, run, attempt, {
|
||||
atMs: fact.atMs,
|
||||
}));
|
||||
({ run, attempt } = await this.ensureRunning(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
fact.atMs,
|
||||
));
|
||||
|
||||
const succeeded = fact.exitCode === 0;
|
||||
const attemptTarget: RunAttemptStatus = succeeded
|
||||
? 'succeeded'
|
||||
: 'failed';
|
||||
const runTarget: RunStatus = succeeded ? 'succeeded' : 'failed';
|
||||
const errorCode =
|
||||
fact.exitCode === null
|
||||
? fact.signal
|
||||
? 'LEGACY_PROCESS_SIGNALLED'
|
||||
: 'LEGACY_EXIT_UNKNOWN'
|
||||
: succeeded
|
||||
? undefined
|
||||
: 'LEGACY_EXIT_NON_ZERO';
|
||||
const errorSummary = errorCode
|
||||
? 'Legacy process exited without success'
|
||||
: undefined;
|
||||
|
||||
if (!isTerminalRunAttemptStatus(attempt.status)) {
|
||||
({ run, attempt } = await this.transitionAttempt(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
{
|
||||
to: attemptTarget,
|
||||
expectedRunVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
...(fact.exitCode === null ? {} : { exitCode: fact.exitCode }),
|
||||
...(errorCode === undefined ? {} : { errorCode }),
|
||||
...(errorSummary === undefined ? {} : { errorSummary }),
|
||||
},
|
||||
fact.signal === undefined ? {} : { legacy_signal: fact.signal },
|
||||
));
|
||||
}
|
||||
|
||||
if (!isTerminalRunStatus(run.status)) {
|
||||
await this.transitionRunStatus(
|
||||
transaction,
|
||||
run,
|
||||
{
|
||||
to: runTarget,
|
||||
expectedVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
...(errorCode === undefined ? {} : { errorCode }),
|
||||
...(errorSummary === undefined ? {} : { errorSummary }),
|
||||
},
|
||||
{
|
||||
legacy_exit_code: fact.exitCode,
|
||||
...(fact.signal === undefined
|
||||
? {}
|
||||
: { legacy_signal: fact.signal }),
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async cancelled(
|
||||
reference: LegacyShadowRunReference,
|
||||
fact: LegacyExecutionCancelledFact,
|
||||
): Promise<void> {
|
||||
await this.repository.transaction(async (transaction) => {
|
||||
let { run, attempt } = await this.load(transaction, reference);
|
||||
if (isTerminalRunStatus(run.status)) return;
|
||||
|
||||
if (!isTerminalRunAttemptStatus(attempt.status)) {
|
||||
({ run, attempt } = await this.transitionAttempt(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
{
|
||||
to: 'cancelled',
|
||||
expectedRunVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
errorCode: 'LEGACY_EXECUTION_CANCELLED',
|
||||
errorSummary: 'Legacy execution was cancelled',
|
||||
},
|
||||
{ legacy_cancel_reason: fact.reason },
|
||||
));
|
||||
}
|
||||
if (!isTerminalRunStatus(run.status)) {
|
||||
await this.transitionRunStatus(
|
||||
transaction,
|
||||
run,
|
||||
{
|
||||
to: 'cancelled',
|
||||
expectedVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
errorCode: 'LEGACY_EXECUTION_CANCELLED',
|
||||
errorSummary: 'Legacy execution was cancelled',
|
||||
},
|
||||
{ legacy_cancel_reason: fact.reason },
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async load(
|
||||
transaction: RunRepositoryTransaction,
|
||||
reference: LegacyShadowRunReference,
|
||||
): Promise<{ run: RunRecord; attempt: RunAttemptRecord }> {
|
||||
const run = await transaction.findRunById(reference.runId);
|
||||
if (!run) throw new RunNotFoundError(reference.runId);
|
||||
const attempt = await transaction.findAttemptById(reference.attemptId);
|
||||
if (!attempt) throw new RunAttemptNotFoundError(reference.attemptId);
|
||||
if (attempt.runId !== run.id) {
|
||||
throw new RunAttemptConcurrentWriteError(
|
||||
attempt.id,
|
||||
attempt.status,
|
||||
attempt.callbackSequence,
|
||||
);
|
||||
}
|
||||
return { run, attempt };
|
||||
}
|
||||
|
||||
private async ensureSpawned(
|
||||
transaction: RunRepositoryTransaction,
|
||||
currentRun: RunRecord,
|
||||
currentAttempt: RunAttemptRecord,
|
||||
fact: LegacyExecutionSpawnedFact,
|
||||
): Promise<{ run: RunRecord; attempt: RunAttemptRecord }> {
|
||||
let run = currentRun;
|
||||
let attempt = currentAttempt;
|
||||
if (isTerminalRunStatus(run.status)) return { run, attempt };
|
||||
|
||||
if (run.status === 'created') {
|
||||
run = await this.transitionRunStatus(transaction, run, {
|
||||
to: 'queued',
|
||||
expectedVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
});
|
||||
}
|
||||
if (run.status === 'queued') {
|
||||
run = await this.transitionRunStatus(transaction, run, {
|
||||
to: 'dispatching',
|
||||
expectedVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
});
|
||||
}
|
||||
if (attempt.status === 'claimed') {
|
||||
({ run, attempt } = await this.transitionAttempt(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
{
|
||||
to: 'starting',
|
||||
expectedRunVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
...(fact.pid === undefined ? {} : { pid: fact.pid }),
|
||||
...(fact.executorHandle === undefined
|
||||
? {}
|
||||
: { executorHandle: fact.executorHandle }),
|
||||
...(fact.logArtifactId === undefined
|
||||
? {}
|
||||
: { logArtifactId: fact.logArtifactId }),
|
||||
},
|
||||
));
|
||||
}
|
||||
return { run, attempt };
|
||||
}
|
||||
|
||||
private async ensureRunning(
|
||||
transaction: RunRepositoryTransaction,
|
||||
currentRun: RunRecord,
|
||||
currentAttempt: RunAttemptRecord,
|
||||
atMs: number,
|
||||
): Promise<{ run: RunRecord; attempt: RunAttemptRecord }> {
|
||||
let run = currentRun;
|
||||
let attempt = currentAttempt;
|
||||
if (isTerminalRunStatus(run.status)) return { run, attempt };
|
||||
|
||||
if (attempt.status === 'starting') {
|
||||
({ run, attempt } = await this.transitionAttempt(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
{
|
||||
to: 'running',
|
||||
expectedRunVersion: run.version,
|
||||
atMs,
|
||||
},
|
||||
));
|
||||
}
|
||||
if (run.status === 'dispatching') {
|
||||
run = await this.transitionRunStatus(transaction, run, {
|
||||
to: 'running',
|
||||
expectedVersion: run.version,
|
||||
atMs,
|
||||
});
|
||||
}
|
||||
return { run, attempt };
|
||||
}
|
||||
|
||||
private async transitionRunStatus(
|
||||
transaction: RunRepositoryTransaction,
|
||||
current: RunRecord,
|
||||
command: RunTransitionCommand,
|
||||
extraPayload: Readonly<Record<string, unknown>> = {},
|
||||
): Promise<RunRecord> {
|
||||
const decision = transitionRun(current, command);
|
||||
await this.persistRunDecision(
|
||||
transaction,
|
||||
current,
|
||||
decision,
|
||||
command.atMs,
|
||||
extraPayload,
|
||||
);
|
||||
return decision.run;
|
||||
}
|
||||
|
||||
private async transitionAttempt(
|
||||
transaction: RunRepositoryTransaction,
|
||||
currentRun: RunRecord,
|
||||
currentAttempt: RunAttemptRecord,
|
||||
command: RunAttemptTransitionCommand,
|
||||
extraPayload: Readonly<Record<string, unknown>> = {},
|
||||
): Promise<{ run: RunRecord; attempt: RunAttemptRecord }> {
|
||||
const decision = transitionRunAttempt(currentRun, currentAttempt, command);
|
||||
await this.persistAttemptDecision(
|
||||
transaction,
|
||||
currentRun,
|
||||
currentAttempt,
|
||||
decision,
|
||||
command.atMs,
|
||||
extraPayload,
|
||||
);
|
||||
return { run: decision.run, attempt: decision.attempt };
|
||||
}
|
||||
|
||||
private async persistRunDecision(
|
||||
transaction: RunRepositoryTransaction,
|
||||
current: RunRecord,
|
||||
decision: RunTransitionDecision,
|
||||
atMs: number,
|
||||
extraPayload: Readonly<Record<string, unknown>> = {},
|
||||
): Promise<void> {
|
||||
const updated = await transaction.compareAndSetRun(
|
||||
decision.run,
|
||||
current.version,
|
||||
);
|
||||
if (!updated) {
|
||||
const latest = await transaction.findRunById(current.id);
|
||||
throw new RunVersionConflictError(
|
||||
current.id,
|
||||
current.version,
|
||||
latest?.version ?? current.version,
|
||||
);
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
this.event(decision.run, decision.event, atMs, extraPayload),
|
||||
);
|
||||
}
|
||||
|
||||
private async persistAttemptDecision(
|
||||
transaction: RunRepositoryTransaction,
|
||||
currentRun: RunRecord,
|
||||
currentAttempt: RunAttemptRecord,
|
||||
decision: RunAttemptTransitionDecision,
|
||||
atMs: number,
|
||||
extraPayload: Readonly<Record<string, unknown>> = {},
|
||||
): Promise<void> {
|
||||
const runUpdated = await transaction.compareAndSetRun(
|
||||
decision.run,
|
||||
currentRun.version,
|
||||
);
|
||||
if (!runUpdated) {
|
||||
const latest = await transaction.findRunById(currentRun.id);
|
||||
throw new RunVersionConflictError(
|
||||
currentRun.id,
|
||||
currentRun.version,
|
||||
latest?.version ?? currentRun.version,
|
||||
);
|
||||
}
|
||||
const attemptUpdated = await transaction.compareAndSetAttempt(
|
||||
decision.attempt,
|
||||
{
|
||||
status: currentAttempt.status,
|
||||
callbackSequence: currentAttempt.callbackSequence,
|
||||
},
|
||||
);
|
||||
if (!attemptUpdated) {
|
||||
throw new RunAttemptConcurrentWriteError(
|
||||
currentAttempt.id,
|
||||
currentAttempt.status,
|
||||
currentAttempt.callbackSequence,
|
||||
);
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
this.event(decision.run, decision.event, atMs, extraPayload),
|
||||
);
|
||||
}
|
||||
|
||||
private event(
|
||||
run: RunRecord,
|
||||
draft: RunDomainEventDraft,
|
||||
createdAtMs: number,
|
||||
extraPayload: Readonly<Record<string, unknown>> = {},
|
||||
): RunEventRecord {
|
||||
return {
|
||||
id: this.createId(),
|
||||
runId: run.id,
|
||||
sequence: draft.sequence,
|
||||
type: draft.type,
|
||||
dedupeKey: `shadow:${draft.sequence}:${draft.type}`,
|
||||
actorType: 'compatibility',
|
||||
payload: {
|
||||
...draft.payload,
|
||||
...extraPayload,
|
||||
shadow: true,
|
||||
},
|
||||
createdAtMs,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
import {
|
||||
assertArtifactReadProjectId,
|
||||
normalizeArtifactReadSubject,
|
||||
normalizeLocalArtifactReadMetadata,
|
||||
normalizeLocalArtifactReadRange,
|
||||
type ArtifactReadSubject,
|
||||
type LocalArtifactReadMetadata,
|
||||
type LocalArtifactReadRange,
|
||||
} from '../domain/artifactRead';
|
||||
import { assertCompletionReceiptId } from '../domain/completionReceipt';
|
||||
import type { LocalArtifactTruncationFact } from '../domain/localArtifactTruncation';
|
||||
import { assertLocalExecutionArtifactId } from '../domain/localExecutionArtifact';
|
||||
import type {
|
||||
ArtifactReadAuthorizationEffect,
|
||||
ArtifactReadAuthorizer,
|
||||
} from '../ports/artifactReadAuthorizer';
|
||||
import type { LocalArtifactByteRangeReader } from '../ports/localArtifactByteRangeReader';
|
||||
import type { LocalArtifactReadMetadataRepository } from '../ports/localArtifactReadMetadataRepository';
|
||||
import type { LocalArtifactTruncationFactStore } from '../ports/localArtifactTruncationFactStore';
|
||||
|
||||
export type LocalArtifactTruncationState = boolean | 'unknown';
|
||||
|
||||
export interface LocalArtifactTruncationView {
|
||||
truncated: LocalArtifactTruncationState;
|
||||
maximumBytes?: number;
|
||||
observedAtMs?: number;
|
||||
}
|
||||
|
||||
export interface LocalArtifactReadRequest {
|
||||
subject: ArtifactReadSubject;
|
||||
projectId: string;
|
||||
runId: string;
|
||||
logArtifactId: string;
|
||||
range: LocalArtifactReadRange;
|
||||
}
|
||||
|
||||
interface LocalArtifactReadIdentity {
|
||||
projectId: string;
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
logArtifactId: string;
|
||||
}
|
||||
|
||||
export type LocalArtifactReadResult =
|
||||
| { status: 'not_found' }
|
||||
| {
|
||||
status: 'forbidden';
|
||||
effect: Exclude<ArtifactReadAuthorizationEffect, 'allow'>;
|
||||
}
|
||||
| (LocalArtifactReadIdentity & {
|
||||
status: 'retained';
|
||||
retention: NonNullable<LocalArtifactReadMetadata['retention']>;
|
||||
truncation: { truncated: 'unknown' };
|
||||
})
|
||||
| (LocalArtifactReadIdentity & {
|
||||
status: 'missing';
|
||||
truncation: Readonly<LocalArtifactTruncationView>;
|
||||
})
|
||||
| (LocalArtifactReadIdentity & {
|
||||
status: 'available';
|
||||
content: Buffer;
|
||||
start: number;
|
||||
endExclusive: number;
|
||||
totalBytes: number;
|
||||
nextOffset?: number;
|
||||
truncation: Readonly<LocalArtifactTruncationView>;
|
||||
});
|
||||
|
||||
export class LocalArtifactReadEvidenceConflictError extends Error {
|
||||
constructor() {
|
||||
super('Local Artifact read evidence conflicts with database identity');
|
||||
this.name = 'LocalArtifactReadEvidenceConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
function identity(
|
||||
metadata: Readonly<LocalArtifactReadMetadata>,
|
||||
): LocalArtifactReadIdentity {
|
||||
return {
|
||||
projectId: metadata.projectId,
|
||||
runId: metadata.runId,
|
||||
attemptId: metadata.attemptId,
|
||||
logArtifactId: metadata.logArtifactId,
|
||||
};
|
||||
}
|
||||
|
||||
function sameIdentity(
|
||||
left: Readonly<LocalArtifactReadMetadata>,
|
||||
right: Readonly<LocalArtifactReadMetadata>,
|
||||
): boolean {
|
||||
return (
|
||||
left.projectId === right.projectId &&
|
||||
left.runId === right.runId &&
|
||||
left.attemptId === right.attemptId &&
|
||||
left.logArtifactId === right.logArtifactId
|
||||
);
|
||||
}
|
||||
|
||||
function truncationView(
|
||||
metadata: Readonly<LocalArtifactReadMetadata>,
|
||||
fact: Readonly<LocalArtifactTruncationFact> | null,
|
||||
): Readonly<LocalArtifactTruncationView> {
|
||||
if (!fact) return Object.freeze({ truncated: 'unknown' });
|
||||
if (
|
||||
fact.runId !== metadata.runId ||
|
||||
fact.attemptId !== metadata.attemptId ||
|
||||
fact.logArtifactId !== metadata.logArtifactId
|
||||
) {
|
||||
throw new LocalArtifactReadEvidenceConflictError();
|
||||
}
|
||||
return Object.freeze({
|
||||
truncated: fact.quotaReached,
|
||||
maximumBytes: fact.maximumBytes,
|
||||
observedAtMs: fact.observedAtMs,
|
||||
});
|
||||
}
|
||||
|
||||
export class LocalArtifactReadService {
|
||||
constructor(
|
||||
private readonly metadata: LocalArtifactReadMetadataRepository,
|
||||
private readonly authorizer: ArtifactReadAuthorizer,
|
||||
private readonly bytes: LocalArtifactByteRangeReader,
|
||||
private readonly truncationFacts: LocalArtifactTruncationFactStore,
|
||||
) {}
|
||||
|
||||
async read(
|
||||
request: LocalArtifactReadRequest,
|
||||
): Promise<LocalArtifactReadResult> {
|
||||
if (!request || typeof request !== 'object' || Array.isArray(request)) {
|
||||
throw new TypeError('Local Artifact read request must be an object');
|
||||
}
|
||||
const subject = normalizeArtifactReadSubject(request.subject);
|
||||
assertArtifactReadProjectId(request.projectId);
|
||||
assertCompletionReceiptId(request.runId, 'runId');
|
||||
assertLocalExecutionArtifactId(request.logArtifactId);
|
||||
const range = normalizeLocalArtifactReadRange(request.range);
|
||||
const lookup = Object.freeze({
|
||||
projectId: request.projectId,
|
||||
runId: request.runId,
|
||||
logArtifactId: request.logArtifactId,
|
||||
});
|
||||
const initial = await this.metadata.find(lookup);
|
||||
if (!initial) return Object.freeze({ status: 'not_found' });
|
||||
const artifact = normalizeLocalArtifactReadMetadata(initial);
|
||||
const effect = await this.authorizer.authorize(
|
||||
Object.freeze({
|
||||
action: 'artifact.read',
|
||||
subject,
|
||||
projectId: artifact.projectId,
|
||||
runId: artifact.runId,
|
||||
logArtifactId: artifact.logArtifactId,
|
||||
}),
|
||||
);
|
||||
if (effect !== 'allow') {
|
||||
if (effect !== 'deny' && effect !== 'require_approval') {
|
||||
throw new TypeError('Artifact read authorization effect is invalid');
|
||||
}
|
||||
return Object.freeze({ status: 'forbidden', effect });
|
||||
}
|
||||
if (artifact.retention) {
|
||||
return Object.freeze({
|
||||
status: 'retained',
|
||||
...identity(artifact),
|
||||
retention: artifact.retention,
|
||||
truncation: Object.freeze({ truncated: 'unknown' as const }),
|
||||
});
|
||||
}
|
||||
|
||||
const content = await this.bytes.read(artifact.logArtifactId, range);
|
||||
if (content.status === 'missing') {
|
||||
const refreshedValue = await this.metadata.find(lookup);
|
||||
if (!refreshedValue) throw new LocalArtifactReadEvidenceConflictError();
|
||||
const refreshed = normalizeLocalArtifactReadMetadata(refreshedValue);
|
||||
if (!sameIdentity(artifact, refreshed)) {
|
||||
throw new LocalArtifactReadEvidenceConflictError();
|
||||
}
|
||||
if (refreshed.retention) {
|
||||
return Object.freeze({
|
||||
status: 'retained',
|
||||
...identity(refreshed),
|
||||
retention: refreshed.retention,
|
||||
truncation: Object.freeze({ truncated: 'unknown' as const }),
|
||||
});
|
||||
}
|
||||
const fact = await this.truncationFacts.read(artifact.logArtifactId);
|
||||
return Object.freeze({
|
||||
status: 'missing',
|
||||
...identity(artifact),
|
||||
truncation: truncationView(artifact, fact),
|
||||
});
|
||||
}
|
||||
|
||||
const fact = await this.truncationFacts.read(artifact.logArtifactId);
|
||||
return Object.freeze({
|
||||
status: 'available',
|
||||
...identity(artifact),
|
||||
content: content.content,
|
||||
start: content.start,
|
||||
endExclusive: content.endExclusive,
|
||||
totalBytes: content.totalBytes,
|
||||
...(content.nextOffset === undefined
|
||||
? {}
|
||||
: { nextOffset: content.nextOffset }),
|
||||
truncation: truncationView(artifact, fact),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
import type { LocalArtifactRetentionCursor } from '../domain/localArtifactRetention';
|
||||
import type { LocalArtifactRetentionCheckpointStore } from '../ports/localArtifactRetentionCheckpointStore';
|
||||
import type {
|
||||
LocalArtifactRetentionService,
|
||||
LocalArtifactRetentionSweepResult,
|
||||
} from './localArtifactRetentionService';
|
||||
|
||||
export const MIN_LOCAL_ARTIFACT_RETENTION_INTERVAL_MS = 1_000;
|
||||
export const MAX_LOCAL_ARTIFACT_RETENTION_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_LOCAL_ARTIFACT_RETENTION_INITIAL_DELAY_MS =
|
||||
24 * 60 * 60 * 1_000;
|
||||
export const MAX_LOCAL_ARTIFACT_RETENTION_STOP_TIMEOUT_MS = 60_000;
|
||||
|
||||
interface ScheduledTimer {
|
||||
unref?: () => void;
|
||||
}
|
||||
|
||||
export interface LocalArtifactRetentionLifecycleScheduler {
|
||||
setTimeout(callback: () => void, delayMs: number): ScheduledTimer;
|
||||
clearTimeout(timer: ScheduledTimer): void;
|
||||
}
|
||||
|
||||
export interface LocalArtifactRetentionCycleSummary {
|
||||
pressure: boolean;
|
||||
observedAtMs: number;
|
||||
retentionMs: number;
|
||||
availableBytes: string;
|
||||
totalBytes: string;
|
||||
candidatesScanned: number;
|
||||
deletionsAttempted: number;
|
||||
recordsWritten: number;
|
||||
failedCandidates: number;
|
||||
bytesReclaimed: number;
|
||||
sweepStatus: LocalArtifactRetentionSweepResult['status'];
|
||||
cursorAction: 'unchanged' | 'advanced' | 'cleared' | 'fenced';
|
||||
}
|
||||
|
||||
export interface LocalArtifactRetentionLifecycleOptions {
|
||||
intervalMs: number;
|
||||
initialDelayMs?: number;
|
||||
stopTimeoutMs?: number;
|
||||
scheduler?: LocalArtifactRetentionLifecycleScheduler;
|
||||
onCycle?: (summary: Readonly<LocalArtifactRetentionCycleSummary>) => void;
|
||||
onError?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export type LocalArtifactRetentionStopResult = 'drained' | 'timed_out';
|
||||
|
||||
const defaultScheduler: LocalArtifactRetentionLifecycleScheduler = {
|
||||
setTimeout(callback, delayMs) {
|
||||
return setTimeout(callback, delayMs);
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
clearTimeout(timer as ReturnType<typeof setTimeout>);
|
||||
},
|
||||
};
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
function sameCursor(
|
||||
left: LocalArtifactRetentionCursor | undefined,
|
||||
right: LocalArtifactRetentionCursor | undefined,
|
||||
): boolean {
|
||||
return (
|
||||
left === right ||
|
||||
(left !== undefined &&
|
||||
right !== undefined &&
|
||||
left.finishedAtMs === right.finishedAtMs &&
|
||||
left.attemptId === right.attemptId)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Explicit one-page cadence with a durable CAS cursor. Idle complete cycles do
|
||||
* not write a checkpoint, keeping edge flash write amplification bounded.
|
||||
*/
|
||||
export class LocalArtifactRetentionLifecycle {
|
||||
private readonly intervalMs: number;
|
||||
private readonly initialDelayMs: number;
|
||||
private readonly stopTimeoutMs: number;
|
||||
private readonly scheduler: LocalArtifactRetentionLifecycleScheduler;
|
||||
private readonly onCycle?: LocalArtifactRetentionLifecycleOptions['onCycle'];
|
||||
private readonly onError?: LocalArtifactRetentionLifecycleOptions['onError'];
|
||||
private started = false;
|
||||
private timer?: ScheduledTimer;
|
||||
private inFlight?: Promise<void>;
|
||||
|
||||
constructor(
|
||||
private readonly service: Pick<LocalArtifactRetentionService, 'sweep'>,
|
||||
private readonly checkpoints: LocalArtifactRetentionCheckpointStore,
|
||||
options: LocalArtifactRetentionLifecycleOptions,
|
||||
) {
|
||||
this.intervalMs = options.intervalMs;
|
||||
this.initialDelayMs = options.initialDelayMs ?? 0;
|
||||
this.stopTimeoutMs = options.stopTimeoutMs ?? 5_000;
|
||||
this.scheduler = options.scheduler ?? defaultScheduler;
|
||||
this.onCycle = options.onCycle;
|
||||
this.onError = options.onError;
|
||||
assertIntegerBetween(
|
||||
'intervalMs',
|
||||
this.intervalMs,
|
||||
MIN_LOCAL_ARTIFACT_RETENTION_INTERVAL_MS,
|
||||
MAX_LOCAL_ARTIFACT_RETENTION_INTERVAL_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'initialDelayMs',
|
||||
this.initialDelayMs,
|
||||
0,
|
||||
MAX_LOCAL_ARTIFACT_RETENTION_INITIAL_DELAY_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'stopTimeoutMs',
|
||||
this.stopTimeoutMs,
|
||||
1,
|
||||
MAX_LOCAL_ARTIFACT_RETENTION_STOP_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
start(): boolean {
|
||||
if (this.started || this.inFlight) return false;
|
||||
this.started = true;
|
||||
this.schedule(this.initialDelayMs);
|
||||
return true;
|
||||
}
|
||||
|
||||
async stop(): Promise<LocalArtifactRetentionStopResult> {
|
||||
this.started = false;
|
||||
if (this.timer) {
|
||||
this.scheduler.clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
const inFlight = this.inFlight;
|
||||
if (!inFlight) return 'drained';
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const result = await Promise.race<LocalArtifactRetentionStopResult>([
|
||||
inFlight.then(() => 'drained' as const),
|
||||
new Promise<'timed_out'>((resolve) => {
|
||||
timeout = setTimeout(() => resolve('timed_out'), this.stopTimeoutMs);
|
||||
}),
|
||||
]);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
return result;
|
||||
}
|
||||
|
||||
private schedule(delayMs: number): void {
|
||||
if (!this.started || this.timer) return;
|
||||
const timer = this.scheduler.setTimeout(() => {
|
||||
if (this.timer === timer) this.timer = undefined;
|
||||
this.run();
|
||||
}, delayMs);
|
||||
this.timer = timer;
|
||||
timer.unref?.();
|
||||
}
|
||||
|
||||
private run(): void {
|
||||
if (!this.started || this.inFlight) return;
|
||||
const inFlight = this.runCycle()
|
||||
.then((summary) => this.notifyCycle(summary))
|
||||
.catch((error) => this.notifyError(error))
|
||||
.then(() => undefined)
|
||||
.finally(() => {
|
||||
if (this.inFlight === inFlight) this.inFlight = undefined;
|
||||
if (this.started) this.schedule(this.intervalMs);
|
||||
});
|
||||
this.inFlight = inFlight;
|
||||
}
|
||||
|
||||
private async runCycle(): Promise<LocalArtifactRetentionCycleSummary> {
|
||||
const checkpoint = await this.checkpoints.load();
|
||||
const sweep = await this.service.sweep(checkpoint.cursor);
|
||||
const nextCursor =
|
||||
sweep.status === 'complete' ? undefined : sweep.nextCursor;
|
||||
if (sweep.status !== 'complete' && !nextCursor) {
|
||||
throw new TypeError(
|
||||
'Incomplete Local Artifact retention sweep requires a resume cursor',
|
||||
);
|
||||
}
|
||||
let cursorAction: LocalArtifactRetentionCycleSummary['cursorAction'] =
|
||||
'unchanged';
|
||||
if (!sameCursor(checkpoint.cursor, nextCursor)) {
|
||||
const updated = await this.checkpoints.compareAndSet({
|
||||
expectedVersion: checkpoint.version,
|
||||
...(nextCursor ? { cursor: nextCursor } : {}),
|
||||
updatedAtMs: sweep.observedAtMs,
|
||||
});
|
||||
cursorAction = updated ? (nextCursor ? 'advanced' : 'cleared') : 'fenced';
|
||||
}
|
||||
return Object.freeze({
|
||||
pressure: sweep.pressure,
|
||||
observedAtMs: sweep.observedAtMs,
|
||||
retentionMs: sweep.retentionMs,
|
||||
availableBytes: sweep.availableBytes.toString(10),
|
||||
totalBytes: sweep.totalBytes.toString(10),
|
||||
candidatesScanned: sweep.candidatesScanned,
|
||||
deletionsAttempted: sweep.deletionsAttempted,
|
||||
recordsWritten: sweep.recordsWritten,
|
||||
failedCandidates: sweep.failedCandidates,
|
||||
bytesReclaimed: sweep.bytesReclaimed,
|
||||
sweepStatus: sweep.status,
|
||||
cursorAction,
|
||||
});
|
||||
}
|
||||
|
||||
private notifyCycle(summary: LocalArtifactRetentionCycleSummary): void {
|
||||
try {
|
||||
this.onCycle?.(summary);
|
||||
} catch (error) {
|
||||
this.notifyError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private notifyError(error: unknown): void {
|
||||
try {
|
||||
this.onError?.(error);
|
||||
} catch {
|
||||
// Diagnostics must never create another scheduler failure loop.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
import type {
|
||||
LocalArtifactRetentionCandidate,
|
||||
LocalArtifactRetentionCursor,
|
||||
} from '../domain/localArtifactRetention';
|
||||
import {
|
||||
normalizeLocalArtifactRetentionCandidate,
|
||||
normalizeLocalArtifactRetentionCursor,
|
||||
} from '../domain/localArtifactRetention';
|
||||
import type { LocalArtifactCapacitySource } from '../ports/localArtifactCapacityProbe';
|
||||
import type { LocalArtifactFileRetirementStore } from '../ports/localArtifactFileRetirementStore';
|
||||
import type {
|
||||
LocalArtifactRetentionPage,
|
||||
LocalArtifactRetentionRepository,
|
||||
} from '../ports/localArtifactRetentionRepository';
|
||||
import { MAX_LOCAL_ARTIFACT_RETENTION_PAGE_SIZE } from '../ports/localArtifactRetentionRepository';
|
||||
|
||||
export const MIN_LOCAL_ARTIFACT_RETENTION_MS = 60_000;
|
||||
export const MAX_LOCAL_ARTIFACT_RETENTION_MS = 365 * 24 * 60 * 60_000;
|
||||
|
||||
export interface LocalArtifactRetentionServiceOptions {
|
||||
normalRetentionMs: number;
|
||||
pressureRetentionMs: number;
|
||||
minimumFreeBytes: number;
|
||||
pageSize?: number;
|
||||
maximumDeletions?: number;
|
||||
clock?: { now(): number };
|
||||
}
|
||||
|
||||
export interface LocalArtifactRetentionEntry {
|
||||
attemptId: string;
|
||||
logArtifactId: string;
|
||||
outcome: 'deleted' | 'already_absent' | 'file_failed' | 'record_failed';
|
||||
bytesReclaimed: number;
|
||||
}
|
||||
|
||||
export interface LocalArtifactRetentionSweepResult {
|
||||
status: 'complete' | 'page_complete' | 'deletion_budget_exhausted';
|
||||
pressure: boolean;
|
||||
observedAtMs: number;
|
||||
retentionMs: number;
|
||||
availableBytes: bigint;
|
||||
totalBytes: bigint;
|
||||
candidatesScanned: number;
|
||||
deletionsAttempted: number;
|
||||
recordsWritten: number;
|
||||
failedCandidates: number;
|
||||
bytesReclaimed: number;
|
||||
entries: readonly LocalArtifactRetentionEntry[];
|
||||
nextCursor?: LocalArtifactRetentionCursor;
|
||||
}
|
||||
|
||||
export class InvalidLocalArtifactRetentionPageError extends Error {
|
||||
constructor(message: string) {
|
||||
super(`Local Artifact retention page is invalid: ${message}`);
|
||||
this.name = 'InvalidLocalArtifactRetentionPageError';
|
||||
}
|
||||
}
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalArtifactRetentionService {
|
||||
private readonly normalRetentionMs: number;
|
||||
private readonly pressureRetentionMs: number;
|
||||
private readonly minimumFreeBytes: number;
|
||||
private readonly pageSize: number;
|
||||
private readonly maximumDeletions: number;
|
||||
private readonly clock: { now(): number };
|
||||
|
||||
constructor(
|
||||
private readonly repository: LocalArtifactRetentionRepository,
|
||||
private readonly files: LocalArtifactFileRetirementStore,
|
||||
private readonly capacity: LocalArtifactCapacitySource,
|
||||
options: LocalArtifactRetentionServiceOptions,
|
||||
) {
|
||||
this.normalRetentionMs = options.normalRetentionMs;
|
||||
this.pressureRetentionMs = options.pressureRetentionMs;
|
||||
this.minimumFreeBytes = options.minimumFreeBytes;
|
||||
this.pageSize = options.pageSize ?? 16;
|
||||
this.maximumDeletions = options.maximumDeletions ?? 8;
|
||||
this.clock = options.clock ?? Date;
|
||||
assertIntegerBetween(
|
||||
'normalRetentionMs',
|
||||
this.normalRetentionMs,
|
||||
MIN_LOCAL_ARTIFACT_RETENTION_MS,
|
||||
MAX_LOCAL_ARTIFACT_RETENTION_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'pressureRetentionMs',
|
||||
this.pressureRetentionMs,
|
||||
MIN_LOCAL_ARTIFACT_RETENTION_MS,
|
||||
this.normalRetentionMs,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'minimumFreeBytes',
|
||||
this.minimumFreeBytes,
|
||||
0,
|
||||
Number.MAX_SAFE_INTEGER,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'pageSize',
|
||||
this.pageSize,
|
||||
1,
|
||||
MAX_LOCAL_ARTIFACT_RETENTION_PAGE_SIZE,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'maximumDeletions',
|
||||
this.maximumDeletions,
|
||||
1,
|
||||
this.pageSize,
|
||||
);
|
||||
}
|
||||
|
||||
async sweep(
|
||||
cursor?: LocalArtifactRetentionCursor,
|
||||
): Promise<LocalArtifactRetentionSweepResult> {
|
||||
const normalizedCursor = cursor
|
||||
? normalizeLocalArtifactRetentionCursor(cursor)
|
||||
: undefined;
|
||||
const observedAtMs = this.now();
|
||||
const capacity = await this.capacity.inspect();
|
||||
if (
|
||||
typeof capacity?.availableBytes !== 'bigint' ||
|
||||
typeof capacity.totalBytes !== 'bigint' ||
|
||||
capacity.availableBytes < BigInt(0) ||
|
||||
capacity.totalBytes < BigInt(1) ||
|
||||
capacity.availableBytes > capacity.totalBytes
|
||||
) {
|
||||
throw new TypeError('Local Artifact capacity snapshot is invalid');
|
||||
}
|
||||
const pressure = capacity.availableBytes < BigInt(this.minimumFreeBytes);
|
||||
const retentionMs = pressure
|
||||
? this.pressureRetentionMs
|
||||
: this.normalRetentionMs;
|
||||
const cutoffMs = Math.max(0, observedAtMs - retentionMs);
|
||||
const page = await this.repository.list({
|
||||
cutoffMs,
|
||||
...(normalizedCursor ? { cursor: normalizedCursor } : {}),
|
||||
limit: this.pageSize,
|
||||
});
|
||||
this.assertPage(page, normalizedCursor);
|
||||
|
||||
const entries: LocalArtifactRetentionEntry[] = [];
|
||||
let candidatesScanned = 0;
|
||||
let deletionsAttempted = 0;
|
||||
let recordsWritten = 0;
|
||||
let failedCandidates = 0;
|
||||
let bytesReclaimed = 0;
|
||||
let lastProcessed = normalizedCursor;
|
||||
|
||||
for (const candidate of page.candidates) {
|
||||
if (deletionsAttempted >= this.maximumDeletions) {
|
||||
return this.result({
|
||||
status: 'deletion_budget_exhausted',
|
||||
pressure,
|
||||
observedAtMs,
|
||||
retentionMs,
|
||||
availableBytes: capacity.availableBytes,
|
||||
totalBytes: capacity.totalBytes,
|
||||
candidatesScanned,
|
||||
deletionsAttempted,
|
||||
recordsWritten,
|
||||
failedCandidates,
|
||||
bytesReclaimed,
|
||||
entries,
|
||||
nextCursor: lastProcessed,
|
||||
});
|
||||
}
|
||||
candidatesScanned += 1;
|
||||
deletionsAttempted += 1;
|
||||
lastProcessed = this.cursor(candidate);
|
||||
let retired;
|
||||
try {
|
||||
retired = await this.files.retire(candidate.logArtifactId);
|
||||
} catch {
|
||||
failedCandidates += 1;
|
||||
entries.push({
|
||||
attemptId: candidate.attemptId,
|
||||
logArtifactId: candidate.logArtifactId,
|
||||
outcome: 'file_failed',
|
||||
bytesReclaimed: 0,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await this.repository.record({
|
||||
...candidate,
|
||||
eligibleAtMs: candidate.finishedAtMs + retentionMs,
|
||||
disposition: retired.disposition,
|
||||
bytesReclaimed: retired.bytesReclaimed,
|
||||
recordedAtMs: observedAtMs,
|
||||
});
|
||||
recordsWritten += 1;
|
||||
bytesReclaimed += retired.bytesReclaimed;
|
||||
entries.push({
|
||||
attemptId: candidate.attemptId,
|
||||
logArtifactId: candidate.logArtifactId,
|
||||
outcome: retired.disposition,
|
||||
bytesReclaimed: retired.bytesReclaimed,
|
||||
});
|
||||
} catch {
|
||||
failedCandidates += 1;
|
||||
entries.push({
|
||||
attemptId: candidate.attemptId,
|
||||
logArtifactId: candidate.logArtifactId,
|
||||
outcome: 'record_failed',
|
||||
bytesReclaimed: retired.bytesReclaimed,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return this.result({
|
||||
status: page.truncated ? 'page_complete' : 'complete',
|
||||
pressure,
|
||||
observedAtMs,
|
||||
retentionMs,
|
||||
availableBytes: capacity.availableBytes,
|
||||
totalBytes: capacity.totalBytes,
|
||||
candidatesScanned,
|
||||
deletionsAttempted,
|
||||
recordsWritten,
|
||||
failedCandidates,
|
||||
bytesReclaimed,
|
||||
entries,
|
||||
nextCursor: page.nextCursor,
|
||||
});
|
||||
}
|
||||
|
||||
private assertPage(
|
||||
page: LocalArtifactRetentionPage,
|
||||
cursor: Readonly<LocalArtifactRetentionCursor> | undefined,
|
||||
): void {
|
||||
if (
|
||||
!page ||
|
||||
!Array.isArray(page.candidates) ||
|
||||
page.candidates.length > this.pageSize ||
|
||||
typeof page.truncated !== 'boolean'
|
||||
) {
|
||||
throw new InvalidLocalArtifactRetentionPageError(
|
||||
'candidate count exceeds pageSize',
|
||||
);
|
||||
}
|
||||
let previous = cursor;
|
||||
for (const candidate of page.candidates) {
|
||||
normalizeLocalArtifactRetentionCandidate(candidate);
|
||||
if (
|
||||
previous &&
|
||||
(candidate.finishedAtMs < previous.finishedAtMs ||
|
||||
(candidate.finishedAtMs === previous.finishedAtMs &&
|
||||
candidate.attemptId <= previous.attemptId))
|
||||
) {
|
||||
throw new InvalidLocalArtifactRetentionPageError(
|
||||
'candidate cursor did not advance',
|
||||
);
|
||||
}
|
||||
previous = candidate;
|
||||
}
|
||||
const last = page.candidates[page.candidates.length - 1];
|
||||
if (
|
||||
page.truncated !== (page.nextCursor !== undefined) ||
|
||||
(page.nextCursor &&
|
||||
(!last ||
|
||||
page.nextCursor.finishedAtMs !== last.finishedAtMs ||
|
||||
page.nextCursor.attemptId !== last.attemptId))
|
||||
) {
|
||||
throw new InvalidLocalArtifactRetentionPageError(
|
||||
'resume cursor is inconsistent',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private cursor(
|
||||
candidate: LocalArtifactRetentionCandidate,
|
||||
): LocalArtifactRetentionCursor {
|
||||
return Object.freeze({
|
||||
finishedAtMs: candidate.finishedAtMs,
|
||||
attemptId: candidate.attemptId,
|
||||
});
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const value = this.clock.now();
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new TypeError('Local Artifact retention clock is invalid');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private result(
|
||||
value: Omit<LocalArtifactRetentionSweepResult, 'entries'> & {
|
||||
entries: LocalArtifactRetentionEntry[];
|
||||
},
|
||||
): LocalArtifactRetentionSweepResult {
|
||||
const { nextCursor, ...rest } = value;
|
||||
return Object.freeze({
|
||||
...rest,
|
||||
entries: Object.freeze([...value.entries]),
|
||||
...(nextCursor ? { nextCursor: Object.freeze({ ...nextCursor }) } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
import { EXECUTOR_TYPES, type ExecutorType } from '../domain/execution';
|
||||
import {
|
||||
assertRunDispatchCandidate,
|
||||
assertRunDispatchCandidatePageSize,
|
||||
type RunDispatchCandidate,
|
||||
type RunDispatchCandidateCursor,
|
||||
} from '../domain/runDispatchCandidate';
|
||||
import { assertRunDispatchLeaseVersion } from '../domain/runDispatchLease';
|
||||
import { executionSpecForRunDispatchCandidate } from '../domain/runDispatchPlan';
|
||||
import type { RunDispatchCandidateSource } from '../ports/runDispatchCandidateSource';
|
||||
import type { LocalRunDispatchPlanSource } from '../ports/localRunDispatchPlanSource';
|
||||
import type {
|
||||
ActivePrimaryRun,
|
||||
PrimaryClaimedRunStartCommand,
|
||||
} from './primaryRunOrchestrator';
|
||||
import {
|
||||
PrimaryClaimedRunRejectedError,
|
||||
PrimaryRunLaunchError,
|
||||
} from './primaryRunOrchestrator';
|
||||
|
||||
const DEFAULT_LOCAL_DISPATCH_PAGE_SIZE = 8;
|
||||
const DEFAULT_LOCAL_DISPATCH_MAX_PAGES = 1;
|
||||
const MAX_LOCAL_DISPATCH_PAGES = 16;
|
||||
|
||||
export interface LocalClaimedRunActivator {
|
||||
activateClaimed(
|
||||
command: PrimaryClaimedRunStartCommand,
|
||||
): Promise<ActivePrimaryRun>;
|
||||
}
|
||||
|
||||
export interface LocalRunDispatcherOptions {
|
||||
executorType: ExecutorType;
|
||||
pageSize?: number;
|
||||
maxPages?: number;
|
||||
clock?: { now(): number };
|
||||
onDisposeError?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export interface LocalRunDispatcherStats {
|
||||
pages: number;
|
||||
candidatesScanned: number;
|
||||
executorMismatches: number;
|
||||
plansUnavailable: number;
|
||||
activationRaces: number;
|
||||
}
|
||||
|
||||
export type LocalRunDispatcherIdleReason =
|
||||
| 'no_candidates'
|
||||
| 'no_matching_executor'
|
||||
| 'plans_unavailable'
|
||||
| 'activation_raced'
|
||||
| 'scan_budget_exhausted';
|
||||
|
||||
export type LocalRunDispatcherResult =
|
||||
| {
|
||||
status: 'activated';
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
completion: ActivePrimaryRun['completion'];
|
||||
stats: LocalRunDispatcherStats;
|
||||
truncated: boolean;
|
||||
}
|
||||
| {
|
||||
status: 'activation_failed';
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
stats: LocalRunDispatcherStats;
|
||||
truncated: boolean;
|
||||
}
|
||||
| {
|
||||
status: 'idle';
|
||||
reason: LocalRunDispatcherIdleReason;
|
||||
stats: LocalRunDispatcherStats;
|
||||
truncated: boolean;
|
||||
};
|
||||
|
||||
function cursorOf(candidate: RunDispatchCandidate): RunDispatchCandidateCursor {
|
||||
return {
|
||||
priority: candidate.priority,
|
||||
queuedAtMs: candidate.queuedAtMs,
|
||||
attemptCreatedAtMs: candidate.attemptCreatedAtMs,
|
||||
attemptId: candidate.attemptId,
|
||||
};
|
||||
}
|
||||
|
||||
function cursorAdvances(
|
||||
previous: RunDispatchCandidateCursor,
|
||||
next: RunDispatchCandidateCursor,
|
||||
): boolean {
|
||||
return (
|
||||
next.priority < previous.priority ||
|
||||
(next.priority === previous.priority &&
|
||||
(next.queuedAtMs > previous.queuedAtMs ||
|
||||
(next.queuedAtMs === previous.queuedAtMs &&
|
||||
(next.attemptCreatedAtMs > previous.attemptCreatedAtMs ||
|
||||
(next.attemptCreatedAtMs === previous.attemptCreatedAtMs &&
|
||||
next.attemptId > previous.attemptId)))))
|
||||
);
|
||||
}
|
||||
|
||||
function emptyStats(): LocalRunDispatcherStats {
|
||||
return {
|
||||
pages: 0,
|
||||
candidatesScanned: 0,
|
||||
executorMismatches: 0,
|
||||
plansUnavailable: 0,
|
||||
activationRaces: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/** One bounded local dispatch cycle. It owns neither a timer nor task storage. */
|
||||
export class LocalRunDispatcher {
|
||||
private readonly executorType: ExecutorType;
|
||||
private readonly pageSize: number;
|
||||
private readonly maxPages: number;
|
||||
private readonly clock: { now(): number };
|
||||
private readonly onDisposeError?: (error: unknown) => void;
|
||||
|
||||
constructor(
|
||||
private readonly candidates: RunDispatchCandidateSource,
|
||||
private readonly plans: LocalRunDispatchPlanSource,
|
||||
private readonly activator: LocalClaimedRunActivator,
|
||||
options: LocalRunDispatcherOptions,
|
||||
) {
|
||||
this.executorType = options.executorType;
|
||||
this.pageSize = options.pageSize ?? DEFAULT_LOCAL_DISPATCH_PAGE_SIZE;
|
||||
this.maxPages = options.maxPages ?? DEFAULT_LOCAL_DISPATCH_MAX_PAGES;
|
||||
this.clock = options.clock ?? Date;
|
||||
this.onDisposeError = options.onDisposeError;
|
||||
if (!EXECUTOR_TYPES.includes(this.executorType)) {
|
||||
throw new TypeError('Local Run Dispatcher executorType is invalid');
|
||||
}
|
||||
assertRunDispatchCandidatePageSize(this.pageSize);
|
||||
if (
|
||||
!Number.isSafeInteger(this.maxPages) ||
|
||||
this.maxPages < 1 ||
|
||||
this.maxPages > MAX_LOCAL_DISPATCH_PAGES
|
||||
) {
|
||||
throw new RangeError(
|
||||
`maxPages must be between 1 and ${MAX_LOCAL_DISPATCH_PAGES}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async dispatchOnce(): Promise<LocalRunDispatcherResult> {
|
||||
const observedAtMs = this.clock.now();
|
||||
assertRunDispatchLeaseVersion('observedAtMs', observedAtMs);
|
||||
const stats = emptyStats();
|
||||
const seen = new Set<string>();
|
||||
let after: RunDispatchCandidateCursor | undefined;
|
||||
|
||||
for (let pageIndex = 0; pageIndex < this.maxPages; pageIndex += 1) {
|
||||
const page = await this.candidates.listCandidates({
|
||||
observedAtMs,
|
||||
...(after === undefined ? {} : { after }),
|
||||
limit: this.pageSize,
|
||||
});
|
||||
if (page.length > this.pageSize) {
|
||||
throw new RangeError('Local Run candidate source exceeded page size');
|
||||
}
|
||||
stats.pages += 1;
|
||||
let previous = after;
|
||||
for (const candidate of page) {
|
||||
assertRunDispatchCandidate(candidate);
|
||||
const cursor = cursorOf(candidate);
|
||||
if (
|
||||
seen.has(candidate.attemptId) ||
|
||||
(previous !== undefined && !cursorAdvances(previous, cursor))
|
||||
) {
|
||||
throw new Error('Local Run candidate page is not strictly ordered');
|
||||
}
|
||||
seen.add(candidate.attemptId);
|
||||
previous = cursor;
|
||||
stats.candidatesScanned += 1;
|
||||
if (candidate.executorType !== this.executorType) {
|
||||
stats.executorMismatches += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const plan = await this.plans.prepare(Object.freeze({ ...candidate }));
|
||||
if (!plan) {
|
||||
stats.plansUnavailable += 1;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const spec = executionSpecForRunDispatchCandidate(
|
||||
candidate,
|
||||
plan.executionSpec,
|
||||
);
|
||||
const active = await this.activator.activateClaimed({
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
...(spec.timeoutMs === undefined
|
||||
? {}
|
||||
: { timeoutMs: spec.timeoutMs }),
|
||||
createSpec: () => spec,
|
||||
context: plan.context,
|
||||
...(plan.logArtifactId === undefined
|
||||
? {}
|
||||
: { logArtifactId: plan.logArtifactId }),
|
||||
});
|
||||
this.disposeAfterCompletion(active, plan.dispose);
|
||||
return {
|
||||
status: 'activated',
|
||||
runId: active.run.id,
|
||||
attemptId: active.attempt.id,
|
||||
completion: active.completion,
|
||||
stats,
|
||||
truncated: page.length === this.pageSize,
|
||||
};
|
||||
} catch (error) {
|
||||
await this.dispose(plan.dispose);
|
||||
if (error instanceof PrimaryClaimedRunRejectedError) {
|
||||
if (
|
||||
error.reason === 'aggregate_mismatch' ||
|
||||
error.reason === 'executor_mismatch'
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
stats.activationRaces += 1;
|
||||
continue;
|
||||
}
|
||||
if (error instanceof PrimaryRunLaunchError) {
|
||||
return {
|
||||
status: 'activation_failed',
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
stats,
|
||||
truncated: page.length === this.pageSize,
|
||||
};
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
if (page.length < this.pageSize) {
|
||||
return this.idle(stats, false);
|
||||
}
|
||||
after = cursorOf(page[page.length - 1]);
|
||||
}
|
||||
return {
|
||||
status: 'idle',
|
||||
reason: 'scan_budget_exhausted',
|
||||
stats,
|
||||
truncated: true,
|
||||
};
|
||||
}
|
||||
|
||||
private idle(
|
||||
stats: LocalRunDispatcherStats,
|
||||
truncated: boolean,
|
||||
): LocalRunDispatcherResult {
|
||||
const eligible = stats.candidatesScanned - stats.executorMismatches;
|
||||
const reason: LocalRunDispatcherIdleReason =
|
||||
stats.candidatesScanned === 0
|
||||
? 'no_candidates'
|
||||
: eligible === 0
|
||||
? 'no_matching_executor'
|
||||
: stats.plansUnavailable === eligible
|
||||
? 'plans_unavailable'
|
||||
: 'activation_raced';
|
||||
return { status: 'idle', reason, stats, truncated };
|
||||
}
|
||||
|
||||
private disposeAfterCompletion(
|
||||
active: ActivePrimaryRun,
|
||||
dispose: (() => void | Promise<void>) | undefined,
|
||||
): void {
|
||||
if (!dispose) return;
|
||||
void active.completion.then(
|
||||
() => this.dispose(dispose),
|
||||
() => this.dispose(dispose),
|
||||
);
|
||||
}
|
||||
|
||||
private async dispose(
|
||||
dispose: (() => void | Promise<void>) | undefined,
|
||||
): Promise<void> {
|
||||
if (!dispose) return;
|
||||
try {
|
||||
await dispose();
|
||||
} catch (error) {
|
||||
try {
|
||||
this.onDisposeError?.(error);
|
||||
} catch {
|
||||
// Diagnostics must not change activation ownership.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import type { PinnedTaskExecutionRevision } from '../domain/taskExecutionRevision';
|
||||
import type { LocalExecutionContextRecipe } from '../domain/localExecutionContextRecipe';
|
||||
import type { LocalExecutionContextRecipeRepository } from '../ports/localExecutionContextRecipeRepository';
|
||||
import type { TaskExecutionRevisionRepository } from '../ports/taskExecutionRevisionRepository';
|
||||
|
||||
export interface PublishLocalTaskExecutionRevisionCommand {
|
||||
revision: PinnedTaskExecutionRevision;
|
||||
contextRecipe: LocalExecutionContextRecipe;
|
||||
createdAtMs: number;
|
||||
}
|
||||
|
||||
export interface PublishLocalTaskExecutionRevisionResult {
|
||||
contextRecipe: 'inserted' | 'idempotent';
|
||||
revision: 'inserted' | 'idempotent';
|
||||
}
|
||||
|
||||
/** Publishes the dependency first so a revision never points at a missing recipe. */
|
||||
export class LocalTaskExecutionRevisionPublisher {
|
||||
constructor(
|
||||
private readonly recipes: LocalExecutionContextRecipeRepository,
|
||||
private readonly revisions: TaskExecutionRevisionRepository,
|
||||
) {}
|
||||
|
||||
async publish(
|
||||
command: PublishLocalTaskExecutionRevisionCommand,
|
||||
): Promise<PublishLocalTaskExecutionRevisionResult> {
|
||||
if (command.revision.contextRef !== command.contextRecipe.contextRef) {
|
||||
throw new TypeError('Task revision contextRef does not match its recipe');
|
||||
}
|
||||
const contextRecipe = await this.recipes.insert(
|
||||
command.contextRecipe,
|
||||
command.createdAtMs,
|
||||
);
|
||||
const revision = await this.revisions.insert(
|
||||
command.revision,
|
||||
command.createdAtMs,
|
||||
);
|
||||
return { contextRecipe, revision };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
import type {
|
||||
ExecutionOutcome,
|
||||
ExecutionOutputSink,
|
||||
} from '../domain/execution';
|
||||
import type { Executor } from '../ports/executor';
|
||||
import type { RunRepository } from '../ports/runRepository';
|
||||
import { buildLegacyCronExecutionSpec } from '../adapters/legacy/legacyCronExecutionSpec';
|
||||
import { createLegacyTaskRevision } from '../compatibility/legacyTaskRevision';
|
||||
import { createLegacyLogOutputRef } from '../compatibility/legacyLogOutputRef';
|
||||
import type {
|
||||
ManualPrimaryCompletion,
|
||||
ManualPrimaryExecutionRouter,
|
||||
ManualPrimaryStartInput,
|
||||
ManualPrimaryStartedExecution,
|
||||
ManualPrimaryStopResult,
|
||||
} from '../compatibility/manualPrimaryExecutionBridge';
|
||||
import type { RuntimeRolloutPolicy } from '../domain/runtimeRollout';
|
||||
import {
|
||||
PrimaryRunOrchestrator,
|
||||
type ActivePrimaryRun,
|
||||
type PrimaryRunClock,
|
||||
type PrimaryRunOrchestratorOptions,
|
||||
} from './primaryRunOrchestrator';
|
||||
|
||||
export interface PreparedManualPrimaryLog {
|
||||
logPath: string;
|
||||
output: ExecutionOutputSink;
|
||||
completionCommitted?(attemptId: string): Promise<void>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface ManualPrimaryLogFiles {
|
||||
prepare(input: ManualPrimaryStartInput): Promise<PreparedManualPrimaryLog>;
|
||||
}
|
||||
|
||||
export interface ManualPrimaryRuntimeOptions {
|
||||
clock?: PrimaryRunClock;
|
||||
orchestrator?: PrimaryRunOrchestratorOptions;
|
||||
}
|
||||
|
||||
interface ActiveManualExecution {
|
||||
cronId: number;
|
||||
attemptId: string;
|
||||
execution: ActivePrimaryRun;
|
||||
}
|
||||
|
||||
interface PendingManualExecution {
|
||||
cronId: number;
|
||||
controller: AbortController;
|
||||
}
|
||||
|
||||
export class ManualPrimaryOwnershipError extends Error {
|
||||
constructor() {
|
||||
super('Manual execution is not Runtime-owned');
|
||||
this.name = 'ManualPrimaryOwnershipError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process-local manual Primary owner. The active maps are bounded by actual
|
||||
* concurrent executions; durable restart/cross-worker ownership remains the
|
||||
* startup Reconciler and future supervisor's responsibility.
|
||||
*/
|
||||
export class ManualPrimaryRuntime implements ManualPrimaryExecutionRouter {
|
||||
private readonly orchestrator: PrimaryRunOrchestrator;
|
||||
private readonly clock: PrimaryRunClock;
|
||||
private readonly pending = new Map<symbol, PendingManualExecution>();
|
||||
private readonly active = new Map<string, ActiveManualExecution>();
|
||||
|
||||
constructor(
|
||||
repository: RunRepository,
|
||||
executor: Executor,
|
||||
private readonly rollout: RuntimeRolloutPolicy,
|
||||
private readonly logs: ManualPrimaryLogFiles,
|
||||
options: ManualPrimaryRuntimeOptions = {},
|
||||
) {
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.orchestrator = new PrimaryRunOrchestrator(repository, executor, {
|
||||
...options.orchestrator,
|
||||
clock: this.clock,
|
||||
});
|
||||
}
|
||||
|
||||
ownsNewRuns(): boolean {
|
||||
return this.rollout.decide('manual').owner === 'runtime';
|
||||
}
|
||||
|
||||
async start(
|
||||
input: ManualPrimaryStartInput,
|
||||
): Promise<ManualPrimaryStartedExecution> {
|
||||
if (!this.ownsNewRuns()) throw new ManualPrimaryOwnershipError();
|
||||
const pendingId = Symbol('manual-primary-pending');
|
||||
const controller = new AbortController();
|
||||
this.pending.set(pendingId, { cronId: input.cron.id, controller });
|
||||
|
||||
let prepared: PreparedManualPrimaryLog | undefined;
|
||||
try {
|
||||
prepared = await this.logs.prepare(input);
|
||||
const taskRevision = createLegacyTaskRevision({
|
||||
command: input.cron.command,
|
||||
...(input.cron.schedule === undefined
|
||||
? {}
|
||||
: { schedule: input.cron.schedule }),
|
||||
extraSchedules: input.cron.extraSchedules,
|
||||
...(input.cron.taskBefore === undefined
|
||||
? {}
|
||||
: { taskBefore: input.cron.taskBefore }),
|
||||
...(input.cron.taskAfter === undefined
|
||||
? {}
|
||||
: { taskAfter: input.cron.taskAfter }),
|
||||
...(input.cron.workDirectory === undefined
|
||||
? {}
|
||||
: { workDirectory: input.cron.workDirectory }),
|
||||
...(input.cron.logName === undefined
|
||||
? {}
|
||||
: { logName: input.cron.logName }),
|
||||
});
|
||||
const outputRef = createLegacyLogOutputRef(prepared.logPath);
|
||||
const active = await this.orchestrator.start({
|
||||
definition: {
|
||||
projectId: 'default',
|
||||
taskId: 'legacy-cron:' + input.cron.id,
|
||||
taskRevision,
|
||||
...(input.cron.name === undefined
|
||||
? {}
|
||||
: { taskName: input.cron.name }),
|
||||
legacyCronId: input.cron.id,
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
triggeredBy: 'legacy:manual-api',
|
||||
outputRef,
|
||||
acceptedAtMs: input.acceptedAtMs,
|
||||
actor: { type: 'compatibility', id: 'legacy:manual-api' },
|
||||
},
|
||||
createSpec: (reference) =>
|
||||
buildLegacyCronExecutionSpec({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
projectId: reference.run.projectId,
|
||||
taskRevision: reference.run.taskRevision,
|
||||
cron: {
|
||||
id: input.cron.id,
|
||||
command: input.cron.command,
|
||||
...(input.cron.taskBefore === undefined
|
||||
? {}
|
||||
: { taskBefore: input.cron.taskBefore }),
|
||||
...(input.cron.taskAfter === undefined
|
||||
? {}
|
||||
: { taskAfter: input.cron.taskAfter }),
|
||||
...(input.cron.workDirectory === undefined
|
||||
? {}
|
||||
: { workDirectory: input.cron.workDirectory }),
|
||||
...(input.cron.logName === undefined
|
||||
? {}
|
||||
: { logName: input.cron.logName }),
|
||||
},
|
||||
realTime: true,
|
||||
realLogPath: prepared!.logPath,
|
||||
noDelay: true,
|
||||
}),
|
||||
context: {
|
||||
environment: {},
|
||||
signal: controller.signal,
|
||||
output: prepared.output,
|
||||
},
|
||||
});
|
||||
|
||||
const entry: ActiveManualExecution = {
|
||||
cronId: input.cron.id,
|
||||
attemptId: active.attempt.id,
|
||||
execution: active,
|
||||
};
|
||||
this.active.set(active.run.id, entry);
|
||||
this.pending.delete(pendingId);
|
||||
const completion = this.completion(active, prepared).finally(() => {
|
||||
if (this.active.get(active.run.id) === entry) {
|
||||
this.active.delete(active.run.id);
|
||||
}
|
||||
});
|
||||
void completion.catch(() => undefined);
|
||||
return {
|
||||
runId: active.run.id,
|
||||
attemptId: active.attempt.id,
|
||||
...(active.handle.pid === undefined ? {} : { pid: active.handle.pid }),
|
||||
logPath: prepared.logPath,
|
||||
completion,
|
||||
};
|
||||
} catch (error) {
|
||||
if (prepared) await prepared.close().catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
this.pending.delete(pendingId);
|
||||
}
|
||||
}
|
||||
|
||||
async stopCron(
|
||||
cronId: number,
|
||||
requestedAtMs: number,
|
||||
): Promise<ManualPrimaryStopResult> {
|
||||
let matched = 0;
|
||||
for (const pending of this.pending.values()) {
|
||||
if (pending.cronId !== cronId) continue;
|
||||
matched += 1;
|
||||
pending.controller.abort();
|
||||
}
|
||||
const executions = [...this.active.values()].filter(
|
||||
(entry) => entry.cronId === cronId,
|
||||
);
|
||||
matched += executions.length;
|
||||
const failed = await this.stopActive(executions, requestedAtMs);
|
||||
return { matched, failed };
|
||||
}
|
||||
|
||||
async stopAttempt(
|
||||
attemptId: string,
|
||||
requestedAtMs: number,
|
||||
): Promise<ManualPrimaryStopResult> {
|
||||
const execution = [...this.active.values()].find(
|
||||
(entry) => entry.attemptId === attemptId,
|
||||
);
|
||||
if (!execution) return { matched: 0, failed: 0 };
|
||||
const failed = await this.stopActive([execution], requestedAtMs);
|
||||
return { matched: 1, failed };
|
||||
}
|
||||
|
||||
private async stopActive(
|
||||
executions: readonly ActiveManualExecution[],
|
||||
requestedAtMs: number,
|
||||
): Promise<number> {
|
||||
const results = await Promise.allSettled(
|
||||
executions.map((entry) =>
|
||||
entry.execution.cancel({ kind: 'user', requestedAtMs }),
|
||||
),
|
||||
);
|
||||
return results.filter((result) => result.status === 'rejected').length;
|
||||
}
|
||||
|
||||
private async completion(
|
||||
active: ActivePrimaryRun,
|
||||
prepared: PreparedManualPrimaryLog,
|
||||
): Promise<ManualPrimaryCompletion> {
|
||||
try {
|
||||
const completed = await active.completion;
|
||||
await prepared
|
||||
.completionCommitted?.(completed.attempt.id)
|
||||
.catch(() => undefined);
|
||||
const result = completed.result;
|
||||
return {
|
||||
runId: completed.run.id,
|
||||
attemptId: completed.attempt.id,
|
||||
outcome: result.outcome as ExecutionOutcome,
|
||||
...(result.exitCode === undefined ? {} : { exitCode: result.exitCode }),
|
||||
};
|
||||
} finally {
|
||||
await prepared.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
import type { ManualPrimaryExecutionRouter } from '../compatibility/manualPrimaryExecutionBridge';
|
||||
import type { RuntimeRolloutPolicy } from '../domain/runtimeRollout';
|
||||
import type {
|
||||
RuntimeRolloutLoadAudit,
|
||||
RuntimeRolloutLoadResult,
|
||||
} from '../ports/runtimeRolloutLoader';
|
||||
import type { PrimaryCancellationStopResult } from './primaryCancellationLifecycle';
|
||||
import type { PrimaryCompletionReceiptStopResult } from './primaryCompletionReceiptLifecycle';
|
||||
import type { PrimaryRunStartupSummary } from './primaryRunStartupSupervisor';
|
||||
import type { PrimaryTimeoutStopResult } from './primaryTimeoutLifecycle';
|
||||
|
||||
export type ManualPrimaryActivationState =
|
||||
| 'not_activated'
|
||||
| 'selected'
|
||||
| 'reconciled'
|
||||
| 'activated'
|
||||
| 'failed'
|
||||
| 'stopped';
|
||||
|
||||
export interface ManualPrimaryActivationAudit extends RuntimeRolloutLoadAudit {
|
||||
activation: ManualPrimaryActivationState;
|
||||
recovery?: {
|
||||
scanned: number;
|
||||
verifiedRunning: number;
|
||||
recoveredRunning: number;
|
||||
completedFromReceipt: number;
|
||||
quarantinedReceipts: number;
|
||||
publishGraceWaits: number;
|
||||
markedLost: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ManualPrimaryActivationStack {
|
||||
router: ManualPrimaryExecutionRouter;
|
||||
reconcile(): Promise<PrimaryRunStartupSummary>;
|
||||
startCompletion(): boolean;
|
||||
stopCompletion(): Promise<PrimaryCompletionReceiptStopResult>;
|
||||
startTimeout(): boolean;
|
||||
stopTimeout(): Promise<PrimaryTimeoutStopResult>;
|
||||
startCancellation(): boolean;
|
||||
stopCancellation(): Promise<PrimaryCancellationStopResult>;
|
||||
}
|
||||
|
||||
export interface ManualPrimaryRuntimeActivationOptions {
|
||||
load(): Promise<RuntimeRolloutLoadResult>;
|
||||
create(policy: RuntimeRolloutPolicy): ManualPrimaryActivationStack;
|
||||
install(router: ManualPrimaryExecutionRouter): () => void;
|
||||
audit(record: ManualPrimaryActivationAudit): void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface ManualPrimaryRuntimeActivationResult {
|
||||
load: RuntimeRolloutLoadResult;
|
||||
active: boolean;
|
||||
recovery?: PrimaryRunStartupSummary;
|
||||
stop(): Promise<PrimaryCancellationStopResult>;
|
||||
}
|
||||
|
||||
const NOOP_STOP = async (): Promise<PrimaryCancellationStopResult> => 'drained';
|
||||
|
||||
async function stopLifecycles(
|
||||
stack: ManualPrimaryActivationStack,
|
||||
started: { completion: boolean; timeout: boolean; cancellation: boolean },
|
||||
): Promise<PrimaryCancellationStopResult> {
|
||||
let result: PrimaryCancellationStopResult = 'drained';
|
||||
let firstError: unknown;
|
||||
if (started.timeout) {
|
||||
try {
|
||||
if ((await stack.stopTimeout()) === 'timed_out') result = 'timed_out';
|
||||
} catch (error) {
|
||||
firstError = error;
|
||||
}
|
||||
}
|
||||
if (started.cancellation) {
|
||||
try {
|
||||
if ((await stack.stopCancellation()) === 'timed_out') {
|
||||
result = 'timed_out';
|
||||
}
|
||||
} catch (error) {
|
||||
firstError ??= error;
|
||||
}
|
||||
}
|
||||
if (started.completion) {
|
||||
try {
|
||||
if ((await stack.stopCompletion()) === 'timed_out') {
|
||||
result = 'timed_out';
|
||||
}
|
||||
} catch (error) {
|
||||
firstError ??= error;
|
||||
}
|
||||
}
|
||||
if (firstError !== undefined) throw firstError;
|
||||
return result;
|
||||
}
|
||||
|
||||
function assertSafeRecovery(summary: PrimaryRunStartupSummary): void {
|
||||
if (
|
||||
summary.remaining ||
|
||||
summary.stopReason !== 'complete' ||
|
||||
summary.skipped > 0 ||
|
||||
summary.ambiguous > 0 ||
|
||||
summary.failed > 0
|
||||
) {
|
||||
throw new Error('Primary startup reconciliation did not converge safely');
|
||||
}
|
||||
}
|
||||
|
||||
function recoveryAudit(summary: PrimaryRunStartupSummary) {
|
||||
return {
|
||||
scanned: summary.scanned,
|
||||
verifiedRunning: summary.verifiedRunning,
|
||||
recoveredRunning: summary.recoveredRunning,
|
||||
completedFromReceipt: summary.completedFromReceipt,
|
||||
quarantinedReceipts: summary.quarantinedReceipts,
|
||||
publishGraceWaits: summary.publishGraceWaits,
|
||||
markedLost: summary.markedLost,
|
||||
};
|
||||
}
|
||||
|
||||
export async function activateManualPrimaryRuntime(
|
||||
options: ManualPrimaryRuntimeActivationOptions,
|
||||
): Promise<ManualPrimaryRuntimeActivationResult> {
|
||||
const load = await options.load();
|
||||
const shouldActivate =
|
||||
load.status === 'accepted' && load.policy.modeFor('manual') === 'primary';
|
||||
if (!shouldActivate) {
|
||||
await options.audit({ ...load.audit, activation: 'not_activated' });
|
||||
return { load, active: false, stop: NOOP_STOP };
|
||||
}
|
||||
|
||||
let stack: ManualPrimaryActivationStack | undefined;
|
||||
let dispose: (() => void) | undefined;
|
||||
let completionStarted = false;
|
||||
let timeoutStarted = false;
|
||||
let cancellationStarted = false;
|
||||
try {
|
||||
await options.audit({ ...load.audit, activation: 'selected' });
|
||||
stack = options.create(load.policy);
|
||||
const recovery = await stack.reconcile();
|
||||
assertSafeRecovery(recovery);
|
||||
await options.audit({
|
||||
...load.audit,
|
||||
activation: 'reconciled',
|
||||
recovery: recoveryAudit(recovery),
|
||||
});
|
||||
completionStarted = stack.startCompletion();
|
||||
if (!completionStarted) {
|
||||
throw new Error('Primary completion lifecycle did not start');
|
||||
}
|
||||
timeoutStarted = stack.startTimeout();
|
||||
if (!timeoutStarted) {
|
||||
throw new Error('Primary timeout lifecycle did not start');
|
||||
}
|
||||
cancellationStarted = stack.startCancellation();
|
||||
if (!cancellationStarted) {
|
||||
throw new Error('Primary cancellation lifecycle did not start');
|
||||
}
|
||||
dispose = options.install(stack.router);
|
||||
await options.audit({
|
||||
...load.audit,
|
||||
activation: 'activated',
|
||||
recovery: recoveryAudit(recovery),
|
||||
});
|
||||
|
||||
let stopped = false;
|
||||
return {
|
||||
load,
|
||||
active: true,
|
||||
recovery,
|
||||
async stop() {
|
||||
if (stopped) return 'drained';
|
||||
stopped = true;
|
||||
dispose?.();
|
||||
const result = await stopLifecycles(stack!, {
|
||||
completion: completionStarted,
|
||||
timeout: timeoutStarted,
|
||||
cancellation: cancellationStarted,
|
||||
});
|
||||
try {
|
||||
await options.audit({
|
||||
...load.audit,
|
||||
activation: 'stopped',
|
||||
recovery: recoveryAudit(recovery),
|
||||
});
|
||||
} catch {
|
||||
// Cleanup must not be reversed by a diagnostic failure.
|
||||
}
|
||||
return result;
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
dispose?.();
|
||||
if (stack && (completionStarted || timeoutStarted || cancellationStarted)) {
|
||||
try {
|
||||
await stopLifecycles(stack, {
|
||||
completion: completionStarted,
|
||||
timeout: timeoutStarted,
|
||||
cancellation: cancellationStarted,
|
||||
});
|
||||
} catch {
|
||||
// Preserve the activation error after best-effort cleanup.
|
||||
}
|
||||
}
|
||||
try {
|
||||
await options.audit({ ...load.audit, activation: 'failed' });
|
||||
} catch {
|
||||
// Preserve the activation error while ownership remains uninstalled.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import { normalizeExecutionContext } from '../domain/executionContext';
|
||||
import type { RunDispatchCandidate } from '../domain/runDispatchCandidate';
|
||||
import {
|
||||
executionSpecFromPinnedTaskRevision,
|
||||
type PinnedTaskExecutionRevision,
|
||||
} from '../domain/taskExecutionRevision';
|
||||
import type { LocalExecutionContextMaterializer } from '../ports/localExecutionContextMaterializer';
|
||||
import type {
|
||||
LocalRunDispatchPlan,
|
||||
LocalRunDispatchPlanSource,
|
||||
} from '../ports/localRunDispatchPlanSource';
|
||||
import type { TaskExecutionRevisionSource } from '../ports/taskExecutionRevisionSource';
|
||||
|
||||
/** Composes immutable Task facts with fresh Attempt-scoped local capabilities. */
|
||||
export class PinnedTaskLocalRunDispatchPlanSource
|
||||
implements LocalRunDispatchPlanSource
|
||||
{
|
||||
constructor(
|
||||
private readonly revisions: TaskExecutionRevisionSource,
|
||||
private readonly contexts: LocalExecutionContextMaterializer,
|
||||
) {}
|
||||
|
||||
async prepare(
|
||||
candidate: Readonly<RunDispatchCandidate>,
|
||||
): Promise<LocalRunDispatchPlan | null> {
|
||||
const revision = await this.revisions.resolve(
|
||||
Object.freeze({
|
||||
projectId: candidate.projectId,
|
||||
taskId: candidate.taskId,
|
||||
taskRevision: candidate.taskRevision,
|
||||
}),
|
||||
);
|
||||
if (!revision) return null;
|
||||
const executionSpec = executionSpecFromPinnedTaskRevision(
|
||||
candidate,
|
||||
this.knownRevision(revision),
|
||||
);
|
||||
const context = await this.contexts.prepare(
|
||||
Object.freeze({
|
||||
candidate: Object.freeze({ ...candidate }),
|
||||
contextRef: revision.contextRef,
|
||||
}),
|
||||
);
|
||||
if (!context) return null;
|
||||
let normalizedContext;
|
||||
try {
|
||||
normalizedContext = normalizeExecutionContext(context.context);
|
||||
} catch (error) {
|
||||
try {
|
||||
await context.dispose?.();
|
||||
} catch {
|
||||
// Cleanup failure must not replace the validation failure.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
executionSpec,
|
||||
context: normalizedContext,
|
||||
...(context.logArtifactId === undefined
|
||||
? {}
|
||||
: { logArtifactId: context.logArtifactId }),
|
||||
...(context.dispose === undefined ? {} : { dispose: context.dispose }),
|
||||
};
|
||||
}
|
||||
|
||||
private knownRevision(
|
||||
revision: PinnedTaskExecutionRevision,
|
||||
): PinnedTaskExecutionRevision {
|
||||
return {
|
||||
projectId: revision.projectId,
|
||||
taskId: revision.taskId,
|
||||
taskRevision: revision.taskRevision,
|
||||
executorType: revision.executorType,
|
||||
execution: revision.execution,
|
||||
contextRef: revision.contextRef,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import type { CancellationDispatchResult } from '../domain/cancellationDispatch';
|
||||
import { RUN_CANCELLATION_REASONS } from '../domain/run';
|
||||
import type { CancellationDispatchRepository } from '../ports/cancellationDispatchRepository';
|
||||
import type { PersistedExecutionController } from '../ports/persistedExecutionController';
|
||||
import type {
|
||||
PrimaryCancellationAttemptReference,
|
||||
PrimaryCancellationCursor,
|
||||
PrimaryCancellationSource,
|
||||
} from '../ports/primaryCancellationSource';
|
||||
|
||||
const DEFAULT_LEASE_DURATION_MS = 30_000;
|
||||
const DEFAULT_RETRY_BASE_MS = 1_000;
|
||||
const DEFAULT_RETRY_MAX_MS = 60_000;
|
||||
|
||||
export interface PrimaryCancellationDispatchSummary {
|
||||
scanned: number;
|
||||
claimed: number;
|
||||
terminationRequested: number;
|
||||
alreadyExited: number;
|
||||
pending: number;
|
||||
ambiguous: number;
|
||||
blocked: number;
|
||||
deferred: number;
|
||||
alreadyResolved: number;
|
||||
notEligible: number;
|
||||
failed: number;
|
||||
truncated: boolean;
|
||||
unsafeAttemptOverflow: boolean;
|
||||
nextCursor?: PrimaryCancellationCursor;
|
||||
}
|
||||
|
||||
export interface PrimaryCancellationDispatcherOptions {
|
||||
owner: string;
|
||||
leaseDurationMs?: number;
|
||||
retryBaseMs?: number;
|
||||
retryMaxMs?: number;
|
||||
clock?: () => number;
|
||||
createId?: () => string;
|
||||
}
|
||||
|
||||
function assertPositiveInteger(name: string, value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 1) {
|
||||
throw new RangeError(`${name} must be a positive safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
/** One bounded pass. The caller owns scheduling and pagination. */
|
||||
export class PrimaryCancellationDispatcher {
|
||||
private readonly controllers = new Map<
|
||||
PersistedExecutionController['executorType'],
|
||||
PersistedExecutionController
|
||||
>();
|
||||
private readonly owner: string;
|
||||
private readonly leaseDurationMs: number;
|
||||
private readonly retryBaseMs: number;
|
||||
private readonly retryMaxMs: number;
|
||||
private readonly clock: () => number;
|
||||
private readonly createId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly source: PrimaryCancellationSource,
|
||||
private readonly dispatches: CancellationDispatchRepository,
|
||||
controllers: readonly PersistedExecutionController[],
|
||||
options: PrimaryCancellationDispatcherOptions,
|
||||
) {
|
||||
if (!options.owner || options.owner.length > 128) {
|
||||
throw new RangeError('owner must be between 1 and 128 characters');
|
||||
}
|
||||
this.owner = options.owner;
|
||||
this.leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
|
||||
this.retryBaseMs = options.retryBaseMs ?? DEFAULT_RETRY_BASE_MS;
|
||||
this.retryMaxMs = options.retryMaxMs ?? DEFAULT_RETRY_MAX_MS;
|
||||
this.clock = options.clock ?? Date.now;
|
||||
this.createId = options.createId ?? uuidV7;
|
||||
assertPositiveInteger('leaseDurationMs', this.leaseDurationMs);
|
||||
assertPositiveInteger('retryBaseMs', this.retryBaseMs);
|
||||
assertPositiveInteger('retryMaxMs', this.retryMaxMs);
|
||||
if (this.retryMaxMs < this.retryBaseMs) {
|
||||
throw new RangeError(
|
||||
'retryMaxMs must be greater than or equal to retryBaseMs',
|
||||
);
|
||||
}
|
||||
|
||||
for (const controller of controllers) {
|
||||
if (this.controllers.has(controller.executorType)) {
|
||||
throw new Error(
|
||||
`Duplicate persisted Executor controller: ${controller.executorType}`,
|
||||
);
|
||||
}
|
||||
this.controllers.set(controller.executorType, controller);
|
||||
}
|
||||
}
|
||||
|
||||
async dispatchBatch(
|
||||
options: { cursor?: PrimaryCancellationCursor; limit?: number } = {},
|
||||
): Promise<PrimaryCancellationDispatchSummary> {
|
||||
const page = await this.source.listCandidates(options);
|
||||
const summary: PrimaryCancellationDispatchSummary = {
|
||||
scanned: page.candidates.length,
|
||||
claimed: 0,
|
||||
terminationRequested: 0,
|
||||
alreadyExited: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
blocked: 0,
|
||||
deferred: 0,
|
||||
alreadyResolved: 0,
|
||||
notEligible: 0,
|
||||
failed: 0,
|
||||
truncated: page.truncated,
|
||||
unsafeAttemptOverflow: page.unsafeAttemptOverflow,
|
||||
...(page.nextCursor === undefined ? {} : { nextCursor: page.nextCursor }),
|
||||
};
|
||||
if (page.unsafeAttemptOverflow) return summary;
|
||||
|
||||
for (const candidate of page.candidates) {
|
||||
if (!RUN_CANCELLATION_REASONS.includes(candidate.reason)) {
|
||||
summary.pending += 1;
|
||||
continue;
|
||||
}
|
||||
if (candidate.attempts.length === 0) {
|
||||
summary.pending += 1;
|
||||
continue;
|
||||
}
|
||||
if (candidate.attempts.length > 1) {
|
||||
summary.ambiguous += 1;
|
||||
summary.pending += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const attempt = candidate.attempts[0];
|
||||
const claimedAtMs = this.now();
|
||||
let claim;
|
||||
try {
|
||||
claim = await this.dispatches.claim({
|
||||
runId: candidate.runId,
|
||||
attemptId: attempt.attemptId,
|
||||
requestedAtMs: candidate.requestedAtMs,
|
||||
owner: this.owner,
|
||||
leaseToken: this.createId(),
|
||||
nowMs: claimedAtMs,
|
||||
leaseDurationMs: this.leaseDurationMs,
|
||||
});
|
||||
} catch {
|
||||
summary.failed += 1;
|
||||
summary.pending += 1;
|
||||
continue;
|
||||
}
|
||||
if (claim.status === 'not_eligible') {
|
||||
summary.notEligible += 1;
|
||||
continue;
|
||||
}
|
||||
if (claim.status === 'leased' || claim.status === 'not_due') {
|
||||
summary.deferred += 1;
|
||||
summary.pending += 1;
|
||||
continue;
|
||||
}
|
||||
if (claim.status === 'dispatched') {
|
||||
summary.alreadyResolved += 1;
|
||||
continue;
|
||||
}
|
||||
if (claim.status === 'blocked') {
|
||||
summary.alreadyResolved += 1;
|
||||
summary.blocked += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
summary.claimed += 1;
|
||||
await this.dispatchClaimed(
|
||||
candidate.reason,
|
||||
candidate.requestedAtMs,
|
||||
attempt,
|
||||
claim.dispatch,
|
||||
summary,
|
||||
);
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
private async dispatchClaimed(
|
||||
reason: (typeof RUN_CANCELLATION_REASONS)[number],
|
||||
requestedAtMs: number,
|
||||
attempt: PrimaryCancellationAttemptReference,
|
||||
dispatch: Extract<
|
||||
Awaited<ReturnType<CancellationDispatchRepository['claim']>>,
|
||||
{ status: 'claimed' }
|
||||
>['dispatch'],
|
||||
summary: PrimaryCancellationDispatchSummary,
|
||||
): Promise<void> {
|
||||
const controller = this.controllers.get(attempt.executorType);
|
||||
if (!controller) {
|
||||
await this.record(attempt, dispatch, 'controller_missing', summary);
|
||||
return;
|
||||
}
|
||||
if (!attempt.executorHandle) {
|
||||
await this.record(attempt, dispatch, 'handle_missing', summary);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await controller.stop({
|
||||
durableHandle: attempt.executorHandle,
|
||||
...(attempt.pid === undefined ? {} : { expectedPid: attempt.pid }),
|
||||
reason: {
|
||||
kind: reason,
|
||||
requestedAtMs,
|
||||
},
|
||||
});
|
||||
await this.record(attempt, dispatch, result.status, summary);
|
||||
if (result.status === 'termination_requested') {
|
||||
summary.terminationRequested += 1;
|
||||
} else if (result.status === 'already_exited') {
|
||||
summary.alreadyExited += 1;
|
||||
} else {
|
||||
summary.blocked += 1;
|
||||
}
|
||||
} catch {
|
||||
summary.failed += 1;
|
||||
await this.record(attempt, dispatch, 'dispatch_error', summary);
|
||||
}
|
||||
}
|
||||
|
||||
private async record(
|
||||
attempt: PrimaryCancellationAttemptReference,
|
||||
dispatch: Extract<
|
||||
Awaited<ReturnType<CancellationDispatchRepository['claim']>>,
|
||||
{ status: 'claimed' }
|
||||
>['dispatch'],
|
||||
result: CancellationDispatchResult,
|
||||
summary: PrimaryCancellationDispatchSummary,
|
||||
): Promise<void> {
|
||||
const atMs = this.now();
|
||||
const retryable = [
|
||||
'controller_missing',
|
||||
'handle_missing',
|
||||
'dispatch_error',
|
||||
].includes(result);
|
||||
try {
|
||||
await this.dispatches.recordResult({
|
||||
runId: dispatch.runId,
|
||||
attemptId: attempt.attemptId,
|
||||
owner: this.owner,
|
||||
leaseToken: dispatch.leaseToken!,
|
||||
expectedVersion: dispatch.version,
|
||||
result,
|
||||
atMs,
|
||||
...(retryable
|
||||
? { nextAttemptAtMs: this.nextRetryAt(atMs, dispatch.dispatchCount) }
|
||||
: {}),
|
||||
eventId: this.createId(),
|
||||
});
|
||||
if (retryable) summary.pending += 1;
|
||||
} catch {
|
||||
summary.failed += 1;
|
||||
summary.pending += 1;
|
||||
}
|
||||
}
|
||||
|
||||
private nextRetryAt(atMs: number, dispatchCount: number): number {
|
||||
const exponent = Math.max(0, Math.min(dispatchCount - 1, 30));
|
||||
const delay = Math.min(this.retryMaxMs, this.retryBaseMs * 2 ** exponent);
|
||||
return Math.min(Number.MAX_SAFE_INTEGER, atMs + delay);
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new RangeError('clock must return a non-negative safe integer');
|
||||
}
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
import type {
|
||||
PrimaryCancellationCycleOptions,
|
||||
PrimaryCancellationCycleSummary,
|
||||
PrimaryCancellationSupervisor,
|
||||
} from './primaryCancellationSupervisor';
|
||||
|
||||
export const MIN_CANCELLATION_CYCLE_INTERVAL_MS = 250;
|
||||
export const MAX_CANCELLATION_CYCLE_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_CANCELLATION_INITIAL_DELAY_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_CANCELLATION_STOP_TIMEOUT_MS = 60_000;
|
||||
|
||||
interface ScheduledTimer {
|
||||
unref?: () => void;
|
||||
}
|
||||
|
||||
export interface CancellationLifecycleScheduler {
|
||||
setTimeout(callback: () => void, delayMs: number): ScheduledTimer;
|
||||
clearTimeout(timer: ScheduledTimer): void;
|
||||
}
|
||||
|
||||
export interface PrimaryCancellationLifecycleOptions {
|
||||
intervalMs: number;
|
||||
initialDelayMs?: number;
|
||||
stopTimeoutMs?: number;
|
||||
cycle?: PrimaryCancellationCycleOptions;
|
||||
scheduler?: CancellationLifecycleScheduler;
|
||||
onCycle?: (summary: PrimaryCancellationCycleSummary) => void;
|
||||
onError?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export type PrimaryCancellationStopResult = 'drained' | 'timed_out';
|
||||
|
||||
const defaultScheduler: CancellationLifecycleScheduler = {
|
||||
setTimeout(callback, delayMs) {
|
||||
return setTimeout(callback, delayMs);
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
clearTimeout(timer as ReturnType<typeof setTimeout>);
|
||||
},
|
||||
};
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Explicit lifecycle wrapper for a bounded supervisor cycle. It is inert until
|
||||
* start() is called and schedules the next cycle only after the current one
|
||||
* settles, so slow edge devices cannot accumulate overlapping scans.
|
||||
*/
|
||||
export class PrimaryCancellationLifecycle {
|
||||
private readonly intervalMs: number;
|
||||
private readonly initialDelayMs: number;
|
||||
private readonly stopTimeoutMs: number;
|
||||
private readonly cycleOptions: PrimaryCancellationCycleOptions;
|
||||
private readonly scheduler: CancellationLifecycleScheduler;
|
||||
private readonly onCycle?: (summary: PrimaryCancellationCycleSummary) => void;
|
||||
private readonly onError?: (error: unknown) => void;
|
||||
private started = false;
|
||||
private timer?: ScheduledTimer;
|
||||
private inFlight?: Promise<void>;
|
||||
|
||||
constructor(
|
||||
private readonly supervisor: Pick<
|
||||
PrimaryCancellationSupervisor,
|
||||
'runCycle'
|
||||
>,
|
||||
options: PrimaryCancellationLifecycleOptions,
|
||||
) {
|
||||
this.intervalMs = options.intervalMs;
|
||||
this.initialDelayMs = options.initialDelayMs ?? 0;
|
||||
this.stopTimeoutMs = options.stopTimeoutMs ?? 5_000;
|
||||
this.cycleOptions = {
|
||||
...(options.cycle?.cursor === undefined
|
||||
? {}
|
||||
: { cursor: { ...options.cycle.cursor } }),
|
||||
...(options.cycle?.pageSize === undefined
|
||||
? {}
|
||||
: { pageSize: options.cycle.pageSize }),
|
||||
...(options.cycle?.maxPages === undefined
|
||||
? {}
|
||||
: { maxPages: options.cycle.maxPages }),
|
||||
};
|
||||
this.scheduler = options.scheduler ?? defaultScheduler;
|
||||
this.onCycle = options.onCycle;
|
||||
this.onError = options.onError;
|
||||
assertIntegerBetween(
|
||||
'intervalMs',
|
||||
this.intervalMs,
|
||||
MIN_CANCELLATION_CYCLE_INTERVAL_MS,
|
||||
MAX_CANCELLATION_CYCLE_INTERVAL_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'initialDelayMs',
|
||||
this.initialDelayMs,
|
||||
0,
|
||||
MAX_CANCELLATION_INITIAL_DELAY_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'stopTimeoutMs',
|
||||
this.stopTimeoutMs,
|
||||
1,
|
||||
MAX_CANCELLATION_STOP_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
start(): boolean {
|
||||
if (this.started || this.inFlight) return false;
|
||||
this.started = true;
|
||||
this.schedule(this.initialDelayMs);
|
||||
return true;
|
||||
}
|
||||
|
||||
async stop(): Promise<PrimaryCancellationStopResult> {
|
||||
this.started = false;
|
||||
if (this.timer) {
|
||||
this.scheduler.clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
const inFlight = this.inFlight;
|
||||
if (!inFlight) return 'drained';
|
||||
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const result = await Promise.race<PrimaryCancellationStopResult>([
|
||||
inFlight.then(() => 'drained' as const),
|
||||
new Promise<'timed_out'>((resolve) => {
|
||||
timeout = setTimeout(() => resolve('timed_out'), this.stopTimeoutMs);
|
||||
}),
|
||||
]);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
return result;
|
||||
}
|
||||
|
||||
private schedule(delayMs: number): void {
|
||||
if (!this.started || this.timer) return;
|
||||
const timer = this.scheduler.setTimeout(() => {
|
||||
if (this.timer === timer) this.timer = undefined;
|
||||
this.run();
|
||||
}, delayMs);
|
||||
this.timer = timer;
|
||||
timer.unref?.();
|
||||
}
|
||||
|
||||
private run(): void {
|
||||
if (!this.started || this.inFlight) return;
|
||||
const inFlight = this.supervisor
|
||||
.runCycle(this.cycleOptions)
|
||||
.then((summary) => this.notifyCycle(summary))
|
||||
.catch((error) => this.notifyError(error))
|
||||
.then(() => undefined)
|
||||
.finally(() => {
|
||||
if (this.inFlight === inFlight) this.inFlight = undefined;
|
||||
if (this.started) this.schedule(this.intervalMs);
|
||||
});
|
||||
this.inFlight = inFlight;
|
||||
}
|
||||
|
||||
private notifyCycle(summary: PrimaryCancellationCycleSummary): void {
|
||||
try {
|
||||
this.onCycle?.(summary);
|
||||
} catch (error) {
|
||||
this.notifyError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private notifyError(error: unknown): void {
|
||||
try {
|
||||
this.onError?.(error);
|
||||
} catch {
|
||||
// Diagnostics must never create another scheduler failure loop.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
import { MAX_PRIMARY_CANCELLATION_BATCH_SIZE } from '../ports/primaryCancellationSource';
|
||||
import type { PrimaryCancellationCursor } from '../ports/primaryCancellationSource';
|
||||
import type {
|
||||
PrimaryCancellationDispatcher,
|
||||
PrimaryCancellationDispatchSummary,
|
||||
} from './primaryCancellationDispatcher';
|
||||
|
||||
export const MAX_PRIMARY_CANCELLATION_PAGES_PER_CYCLE = 64;
|
||||
|
||||
export type PrimaryCancellationCycleStopReason =
|
||||
| 'complete'
|
||||
| 'page_limit'
|
||||
| 'unsafe_attempt_overflow'
|
||||
| 'cursor_stalled';
|
||||
|
||||
export interface PrimaryCancellationCycleSummary
|
||||
extends Omit<
|
||||
PrimaryCancellationDispatchSummary,
|
||||
'truncated' | 'unsafeAttemptOverflow' | 'nextCursor'
|
||||
> {
|
||||
pages: number;
|
||||
stopReason: PrimaryCancellationCycleStopReason;
|
||||
remaining: boolean;
|
||||
nextCursor?: PrimaryCancellationCursor;
|
||||
}
|
||||
|
||||
export interface PrimaryCancellationCycleOptions {
|
||||
cursor?: PrimaryCancellationCursor;
|
||||
pageSize?: number;
|
||||
maxPages?: number;
|
||||
}
|
||||
|
||||
function sameCursor(
|
||||
left: PrimaryCancellationCursor | undefined,
|
||||
right: PrimaryCancellationCursor,
|
||||
): boolean {
|
||||
return (
|
||||
left !== undefined &&
|
||||
left.requestedAtMs === right.requestedAtMs &&
|
||||
left.runId === right.runId
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs a bounded recovery cycle. It deliberately owns no timer or process hook;
|
||||
* edge and cluster deployments choose their own cadence and lifecycle.
|
||||
*/
|
||||
export class PrimaryCancellationSupervisor {
|
||||
constructor(
|
||||
private readonly dispatcher: Pick<
|
||||
PrimaryCancellationDispatcher,
|
||||
'dispatchBatch'
|
||||
>,
|
||||
) {}
|
||||
|
||||
async runCycle(
|
||||
options: PrimaryCancellationCycleOptions = {},
|
||||
): Promise<PrimaryCancellationCycleSummary> {
|
||||
const pageSize = options.pageSize ?? 32;
|
||||
const maxPages = options.maxPages ?? 4;
|
||||
if (
|
||||
!Number.isSafeInteger(pageSize) ||
|
||||
pageSize < 1 ||
|
||||
pageSize > MAX_PRIMARY_CANCELLATION_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'pageSize must be between 1 and MAX_PRIMARY_CANCELLATION_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(maxPages) ||
|
||||
maxPages < 1 ||
|
||||
maxPages > MAX_PRIMARY_CANCELLATION_PAGES_PER_CYCLE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'maxPages must be between 1 and MAX_PRIMARY_CANCELLATION_PAGES_PER_CYCLE',
|
||||
);
|
||||
}
|
||||
|
||||
const total: PrimaryCancellationCycleSummary = {
|
||||
pages: 0,
|
||||
scanned: 0,
|
||||
claimed: 0,
|
||||
terminationRequested: 0,
|
||||
alreadyExited: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
blocked: 0,
|
||||
deferred: 0,
|
||||
alreadyResolved: 0,
|
||||
notEligible: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
let cursor = options.cursor;
|
||||
|
||||
for (let pageNumber = 0; pageNumber < maxPages; pageNumber += 1) {
|
||||
const page = await this.dispatcher.dispatchBatch({
|
||||
...(cursor === undefined ? {} : { cursor }),
|
||||
limit: pageSize,
|
||||
});
|
||||
total.pages += 1;
|
||||
total.scanned += page.scanned;
|
||||
total.claimed += page.claimed;
|
||||
total.terminationRequested += page.terminationRequested;
|
||||
total.alreadyExited += page.alreadyExited;
|
||||
total.pending += page.pending;
|
||||
total.ambiguous += page.ambiguous;
|
||||
total.blocked += page.blocked;
|
||||
total.deferred += page.deferred;
|
||||
total.alreadyResolved += page.alreadyResolved;
|
||||
total.notEligible += page.notEligible;
|
||||
total.failed += page.failed;
|
||||
|
||||
if (page.unsafeAttemptOverflow) {
|
||||
total.stopReason = 'unsafe_attempt_overflow';
|
||||
total.remaining = true;
|
||||
return total;
|
||||
}
|
||||
if (!page.truncated) return total;
|
||||
if (!page.nextCursor || sameCursor(cursor, page.nextCursor)) {
|
||||
total.stopReason = 'cursor_stalled';
|
||||
total.remaining = true;
|
||||
return total;
|
||||
}
|
||||
cursor = page.nextCursor;
|
||||
if (pageNumber === maxPages - 1) {
|
||||
total.stopReason = 'page_limit';
|
||||
total.remaining = true;
|
||||
total.nextCursor = cursor;
|
||||
return total;
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
import type { CompletionReceipt } from '../domain/completionReceipt';
|
||||
import { InvalidCompletionReceiptError } from '../domain/completionReceipt';
|
||||
import type { CompletionReceiptStore } from '../ports/completionReceiptStore';
|
||||
import type { CompletionReceiptJournal } from '../ports/completionReceiptJournal';
|
||||
import type {
|
||||
PrimaryRunCompletionResult,
|
||||
PrimaryRunCompletionService,
|
||||
} from './primaryRunCompletionService';
|
||||
import {
|
||||
PrimaryCompletionNotFoundError,
|
||||
PrimaryCompletionSequenceError,
|
||||
PrimaryCompletionStateError,
|
||||
PrimaryCompletionUnauthorizedError,
|
||||
} from './primaryRunCompletionService';
|
||||
|
||||
export interface PrimaryCompletionReceiptConsumeResult {
|
||||
status: 'missing' | 'quarantined' | PrimaryRunCompletionResult['status'];
|
||||
cleaned: boolean;
|
||||
quarantineRef?: string;
|
||||
completion?: PrimaryRunCompletionResult;
|
||||
}
|
||||
|
||||
export interface PrimaryCompletionReceiptConsumerOptions {
|
||||
journal?: Pick<CompletionReceiptJournal, 'markQuarantined' | 'resolve'>;
|
||||
quarantineRetentionMs?: number;
|
||||
clock?: { now(): number };
|
||||
}
|
||||
|
||||
function mustQuarantine(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof InvalidCompletionReceiptError ||
|
||||
error instanceof PrimaryCompletionNotFoundError ||
|
||||
error instanceof PrimaryCompletionUnauthorizedError ||
|
||||
error instanceof PrimaryCompletionSequenceError ||
|
||||
error instanceof PrimaryCompletionStateError
|
||||
);
|
||||
}
|
||||
|
||||
function receiptResult(receipt: CompletionReceipt) {
|
||||
return {
|
||||
outcome:
|
||||
receipt.exitCode === 0 ? ('succeeded' as const) : ('failed' as const),
|
||||
startedAtMs: receipt.startedAtMs,
|
||||
finishedAtMs: receipt.finishedAtMs,
|
||||
exitCode: receipt.exitCode,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads only a database-discovered Attempt receipt. Cleanup happens after the
|
||||
* terminal transaction; failures leave the immutable receipt replayable.
|
||||
*/
|
||||
export class PrimaryCompletionReceiptConsumer {
|
||||
private readonly journal?: Pick<
|
||||
CompletionReceiptJournal,
|
||||
'markQuarantined' | 'resolve'
|
||||
>;
|
||||
private readonly quarantineRetentionMs: number;
|
||||
private readonly clock: { now(): number };
|
||||
|
||||
constructor(
|
||||
private readonly store: CompletionReceiptStore,
|
||||
private readonly completions: Pick<PrimaryRunCompletionService, 'complete'>,
|
||||
options: PrimaryCompletionReceiptConsumerOptions = {},
|
||||
) {
|
||||
this.journal = options.journal;
|
||||
this.quarantineRetentionMs = options.quarantineRetentionMs ?? 60 * 60_000;
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
if (
|
||||
!Number.isSafeInteger(this.quarantineRetentionMs) ||
|
||||
this.quarantineRetentionMs < 1 ||
|
||||
this.quarantineRetentionMs > 30 * 24 * 60 * 60_000
|
||||
) {
|
||||
throw new RangeError(
|
||||
'quarantineRetentionMs must be between 1 and 30 days',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async consume(
|
||||
attemptId: string,
|
||||
): Promise<PrimaryCompletionReceiptConsumeResult> {
|
||||
let completion: PrimaryRunCompletionResult;
|
||||
try {
|
||||
const receipt = await this.store.read(attemptId);
|
||||
if (!receipt) return { status: 'missing', cleaned: false };
|
||||
completion = await this.completions.complete({
|
||||
runId: receipt.runId,
|
||||
attemptId: receipt.attemptId,
|
||||
callbackSequence: receipt.callbackSequence,
|
||||
result: receiptResult(receipt),
|
||||
source: { kind: 'receipt', token: receipt.token },
|
||||
});
|
||||
} catch (error) {
|
||||
if (!mustQuarantine(error)) throw error;
|
||||
const quarantineRef = this.store.quarantineReference(attemptId);
|
||||
if (this.journal) {
|
||||
const updatedAtMs = this.clock.now();
|
||||
const purgeAfterMs = updatedAtMs + this.quarantineRetentionMs;
|
||||
if (
|
||||
!Number.isSafeInteger(updatedAtMs) ||
|
||||
updatedAtMs < 0 ||
|
||||
!Number.isSafeInteger(purgeAfterMs)
|
||||
) {
|
||||
throw new RangeError('Completion receipt quarantine time is invalid');
|
||||
}
|
||||
await this.journal.markQuarantined({
|
||||
attemptId,
|
||||
quarantineRef,
|
||||
updatedAtMs,
|
||||
purgeAfterMs,
|
||||
});
|
||||
}
|
||||
const quarantined = await this.store.quarantine(attemptId);
|
||||
if (!quarantined) return { status: 'missing', cleaned: false };
|
||||
return {
|
||||
status: 'quarantined',
|
||||
cleaned: true,
|
||||
quarantineRef: quarantined,
|
||||
};
|
||||
}
|
||||
const cleaned = await this.store.remove(attemptId);
|
||||
if (cleaned) await this.journal?.resolve(attemptId);
|
||||
return { status: completion.status, cleaned, completion };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
import type { CompletionReceiptJournal } from '../ports/completionReceiptJournal';
|
||||
import type { CompletionReceiptStore } from '../ports/completionReceiptStore';
|
||||
import type { PrimaryRunRecoveryCursor } from '../ports/primaryRunRecoverySource';
|
||||
import { isTerminalRunAttemptStatus } from '../domain/runStateMachine';
|
||||
import type { PrimaryCompletionReceiptConsumer } from './primaryCompletionReceiptConsumer';
|
||||
import type { PrimaryCompletionReceiptScanSummary } from './primaryCompletionReceiptScanner';
|
||||
|
||||
export interface PrimaryCompletionReceiptJournalScannerOptions {
|
||||
terminalMissingRetentionMs?: number;
|
||||
clock?: { now(): number };
|
||||
}
|
||||
|
||||
/**
|
||||
* Database-indexed receipt retention. The supervisor cursor is only a transport
|
||||
* shape here: createdAtMs carries journal.updatedAtMs and runId carries Attempt
|
||||
* id. No directory enumeration is performed.
|
||||
*/
|
||||
export class PrimaryCompletionReceiptJournalScanner {
|
||||
private readonly terminalMissingRetentionMs: number;
|
||||
private readonly clock: { now(): number };
|
||||
|
||||
constructor(
|
||||
private readonly journal: CompletionReceiptJournal,
|
||||
private readonly store: CompletionReceiptStore,
|
||||
private readonly consumer: Pick<
|
||||
PrimaryCompletionReceiptConsumer,
|
||||
'consume'
|
||||
>,
|
||||
options: PrimaryCompletionReceiptJournalScannerOptions = {},
|
||||
) {
|
||||
this.terminalMissingRetentionMs =
|
||||
options.terminalMissingRetentionMs ?? 60_000;
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
if (
|
||||
!Number.isSafeInteger(this.terminalMissingRetentionMs) ||
|
||||
this.terminalMissingRetentionMs < 0 ||
|
||||
this.terminalMissingRetentionMs > 24 * 60 * 60_000
|
||||
) {
|
||||
throw new RangeError(
|
||||
'terminalMissingRetentionMs must be between 0 and 24 hours',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async scanBatch(
|
||||
options: { cursor?: PrimaryRunRecoveryCursor; limit?: number } = {},
|
||||
): Promise<PrimaryCompletionReceiptScanSummary> {
|
||||
const observedAtMs = this.clock.now();
|
||||
if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0) {
|
||||
throw new RangeError('Completion receipt observation time is invalid');
|
||||
}
|
||||
const page = await this.journal.listCandidates({
|
||||
observedAtMs,
|
||||
...(options.limit === undefined ? {} : { limit: options.limit }),
|
||||
...(options.cursor === undefined
|
||||
? {}
|
||||
: {
|
||||
cursor: {
|
||||
updatedAtMs: options.cursor.createdAtMs,
|
||||
attemptId: options.cursor.runId,
|
||||
},
|
||||
}),
|
||||
});
|
||||
const summary: PrimaryCompletionReceiptScanSummary = {
|
||||
scanned: page.candidates.length,
|
||||
applied: 0,
|
||||
alreadyTerminal: 0,
|
||||
quarantined: 0,
|
||||
purgedQuarantines: 0,
|
||||
expiredMissing: 0,
|
||||
missing: 0,
|
||||
cleanupPending: 0,
|
||||
skipped: 0,
|
||||
ambiguous: 0,
|
||||
failed: 0,
|
||||
truncated: page.truncated,
|
||||
unsafeAttemptOverflow: false,
|
||||
...(page.nextCursor === undefined
|
||||
? {}
|
||||
: {
|
||||
nextCursor: {
|
||||
createdAtMs: page.nextCursor.updatedAtMs,
|
||||
runId: page.nextCursor.attemptId,
|
||||
},
|
||||
}),
|
||||
};
|
||||
|
||||
for (const candidate of page.candidates) {
|
||||
if (candidate.executorType !== 'local_process') {
|
||||
summary.skipped += 1;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
if (candidate.state === 'quarantined') {
|
||||
await this.store.quarantine(candidate.attemptId);
|
||||
await this.store.purgeQuarantine(candidate.attemptId);
|
||||
await this.journal.resolve(candidate.attemptId);
|
||||
summary.purgedQuarantines += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const result = await this.consumer.consume(candidate.attemptId);
|
||||
if (result.status === 'quarantined') {
|
||||
summary.quarantined += 1;
|
||||
continue;
|
||||
}
|
||||
if (result.status === 'missing') {
|
||||
if (
|
||||
isTerminalRunAttemptStatus(candidate.attemptStatus) &&
|
||||
candidate.finishedAtMs !== undefined &&
|
||||
candidate.finishedAtMs + this.terminalMissingRetentionMs <=
|
||||
observedAtMs
|
||||
) {
|
||||
await this.journal.resolve(candidate.attemptId);
|
||||
summary.expiredMissing += 1;
|
||||
} else {
|
||||
summary.missing += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (result.status === 'applied') summary.applied += 1;
|
||||
else summary.alreadyTerminal += 1;
|
||||
if (!result.cleaned) summary.cleanupPending += 1;
|
||||
} catch {
|
||||
summary.failed += 1;
|
||||
}
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
import type {
|
||||
PrimaryCompletionReceiptSupervisor,
|
||||
PrimaryCompletionReceiptSupervisorOptions,
|
||||
PrimaryCompletionReceiptSupervisorSummary,
|
||||
} from './primaryCompletionReceiptSupervisor';
|
||||
import type { PrimaryRunRecoveryCursor } from '../ports/primaryRunRecoverySource';
|
||||
|
||||
export const MIN_COMPLETION_RECEIPT_INTERVAL_MS = 250;
|
||||
export const MAX_COMPLETION_RECEIPT_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_COMPLETION_RECEIPT_INITIAL_DELAY_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_COMPLETION_RECEIPT_STOP_TIMEOUT_MS = 60_000;
|
||||
|
||||
interface ScheduledTimer {
|
||||
unref?: () => void;
|
||||
}
|
||||
|
||||
export interface CompletionReceiptLifecycleScheduler {
|
||||
setTimeout(callback: () => void, delayMs: number): ScheduledTimer;
|
||||
clearTimeout(timer: ScheduledTimer): void;
|
||||
}
|
||||
|
||||
export interface PrimaryCompletionReceiptLifecycleOptions {
|
||||
intervalMs: number;
|
||||
initialDelayMs?: number;
|
||||
stopTimeoutMs?: number;
|
||||
cycle?: PrimaryCompletionReceiptSupervisorOptions;
|
||||
scheduler?: CompletionReceiptLifecycleScheduler;
|
||||
onCycle?: (summary: PrimaryCompletionReceiptSupervisorSummary) => void;
|
||||
onError?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export type PrimaryCompletionReceiptStopResult = 'drained' | 'timed_out';
|
||||
|
||||
const defaultScheduler: CompletionReceiptLifecycleScheduler = {
|
||||
setTimeout(callback, delayMs) {
|
||||
return setTimeout(callback, delayMs);
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
clearTimeout(timer as ReturnType<typeof setTimeout>);
|
||||
},
|
||||
};
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Explicit, non-overlapping receipt polling for small edge deployments. */
|
||||
export class PrimaryCompletionReceiptLifecycle {
|
||||
private readonly intervalMs: number;
|
||||
private readonly initialDelayMs: number;
|
||||
private readonly stopTimeoutMs: number;
|
||||
private readonly cycleOptions: Omit<
|
||||
PrimaryCompletionReceiptSupervisorOptions,
|
||||
'cursor'
|
||||
>;
|
||||
private readonly scheduler: CompletionReceiptLifecycleScheduler;
|
||||
private readonly onCycle?: (
|
||||
summary: PrimaryCompletionReceiptSupervisorSummary,
|
||||
) => void;
|
||||
private readonly onError?: (error: unknown) => void;
|
||||
private started = false;
|
||||
private timer?: ScheduledTimer;
|
||||
private inFlight?: Promise<void>;
|
||||
private resumeCursor?: PrimaryRunRecoveryCursor;
|
||||
|
||||
constructor(
|
||||
private readonly supervisor: Pick<
|
||||
PrimaryCompletionReceiptSupervisor,
|
||||
'run'
|
||||
>,
|
||||
options: PrimaryCompletionReceiptLifecycleOptions,
|
||||
) {
|
||||
this.intervalMs = options.intervalMs;
|
||||
this.initialDelayMs = options.initialDelayMs ?? 0;
|
||||
this.stopTimeoutMs = options.stopTimeoutMs ?? 5_000;
|
||||
this.cycleOptions = {
|
||||
...(options.cycle?.pageSize === undefined
|
||||
? {}
|
||||
: { pageSize: options.cycle.pageSize }),
|
||||
...(options.cycle?.maxPages === undefined
|
||||
? {}
|
||||
: { maxPages: options.cycle.maxPages }),
|
||||
};
|
||||
this.resumeCursor =
|
||||
options.cycle?.cursor === undefined
|
||||
? undefined
|
||||
: { ...options.cycle.cursor };
|
||||
this.scheduler = options.scheduler ?? defaultScheduler;
|
||||
this.onCycle = options.onCycle;
|
||||
this.onError = options.onError;
|
||||
assertIntegerBetween(
|
||||
'intervalMs',
|
||||
this.intervalMs,
|
||||
MIN_COMPLETION_RECEIPT_INTERVAL_MS,
|
||||
MAX_COMPLETION_RECEIPT_INTERVAL_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'initialDelayMs',
|
||||
this.initialDelayMs,
|
||||
0,
|
||||
MAX_COMPLETION_RECEIPT_INITIAL_DELAY_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'stopTimeoutMs',
|
||||
this.stopTimeoutMs,
|
||||
1,
|
||||
MAX_COMPLETION_RECEIPT_STOP_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
start(): boolean {
|
||||
if (this.started || this.inFlight) return false;
|
||||
this.started = true;
|
||||
this.schedule(this.initialDelayMs);
|
||||
return true;
|
||||
}
|
||||
|
||||
async stop(): Promise<PrimaryCompletionReceiptStopResult> {
|
||||
this.started = false;
|
||||
if (this.timer) {
|
||||
this.scheduler.clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
const inFlight = this.inFlight;
|
||||
if (!inFlight) return 'drained';
|
||||
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const result = await Promise.race<PrimaryCompletionReceiptStopResult>([
|
||||
inFlight.then(() => 'drained' as const),
|
||||
new Promise<'timed_out'>((resolve) => {
|
||||
timeout = setTimeout(() => resolve('timed_out'), this.stopTimeoutMs);
|
||||
}),
|
||||
]);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
return result;
|
||||
}
|
||||
|
||||
private schedule(delayMs: number): void {
|
||||
if (!this.started || this.timer) return;
|
||||
const timer = this.scheduler.setTimeout(() => {
|
||||
if (this.timer === timer) this.timer = undefined;
|
||||
this.run();
|
||||
}, delayMs);
|
||||
this.timer = timer;
|
||||
timer.unref?.();
|
||||
}
|
||||
|
||||
private run(): void {
|
||||
if (!this.started || this.inFlight) return;
|
||||
const inFlight = this.supervisor
|
||||
.run({
|
||||
...this.cycleOptions,
|
||||
...(this.resumeCursor === undefined
|
||||
? {}
|
||||
: { cursor: { ...this.resumeCursor } }),
|
||||
})
|
||||
.then((summary) => {
|
||||
this.resumeCursor =
|
||||
summary.remaining && summary.nextCursor
|
||||
? { ...summary.nextCursor }
|
||||
: undefined;
|
||||
this.notifyCycle(summary);
|
||||
})
|
||||
.catch((error) => this.notifyError(error))
|
||||
.then(() => undefined)
|
||||
.finally(() => {
|
||||
if (this.inFlight === inFlight) this.inFlight = undefined;
|
||||
if (this.started) this.schedule(this.intervalMs);
|
||||
});
|
||||
this.inFlight = inFlight;
|
||||
}
|
||||
|
||||
private notifyCycle(
|
||||
summary: PrimaryCompletionReceiptSupervisorSummary,
|
||||
): void {
|
||||
try {
|
||||
this.onCycle?.(summary);
|
||||
} catch (error) {
|
||||
this.notifyError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private notifyError(error: unknown): void {
|
||||
try {
|
||||
this.onError?.(error);
|
||||
} catch {
|
||||
// Diagnostics must never create another scheduler failure loop.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import type {
|
||||
PrimaryRunRecoveryCursor,
|
||||
PrimaryRunRecoverySource,
|
||||
} from '../ports/primaryRunRecoverySource';
|
||||
import type { PrimaryCompletionReceiptConsumer } from './primaryCompletionReceiptConsumer';
|
||||
|
||||
export interface PrimaryCompletionReceiptScanSummary {
|
||||
scanned: number;
|
||||
applied: number;
|
||||
alreadyTerminal: number;
|
||||
quarantined: number;
|
||||
purgedQuarantines: number;
|
||||
expiredMissing: number;
|
||||
missing: number;
|
||||
cleanupPending: number;
|
||||
skipped: number;
|
||||
ambiguous: number;
|
||||
failed: number;
|
||||
truncated: boolean;
|
||||
unsafeAttemptOverflow: boolean;
|
||||
nextCursor?: PrimaryRunRecoveryCursor;
|
||||
}
|
||||
|
||||
/**
|
||||
* One bounded database-driven receipt pass. The database is the index: this
|
||||
* scanner never watches or enumerates the receipt directory.
|
||||
*/
|
||||
export class PrimaryCompletionReceiptScanner {
|
||||
constructor(
|
||||
private readonly source: PrimaryRunRecoverySource,
|
||||
private readonly consumer: Pick<
|
||||
PrimaryCompletionReceiptConsumer,
|
||||
'consume'
|
||||
>,
|
||||
) {}
|
||||
|
||||
async scanBatch(
|
||||
options: {
|
||||
cursor?: PrimaryRunRecoveryCursor;
|
||||
limit?: number;
|
||||
} = {},
|
||||
): Promise<PrimaryCompletionReceiptScanSummary> {
|
||||
const page = await this.source.listCandidates(options);
|
||||
const summary: PrimaryCompletionReceiptScanSummary = {
|
||||
scanned: page.candidates.length,
|
||||
applied: 0,
|
||||
alreadyTerminal: 0,
|
||||
quarantined: 0,
|
||||
purgedQuarantines: 0,
|
||||
expiredMissing: 0,
|
||||
missing: 0,
|
||||
cleanupPending: 0,
|
||||
skipped: 0,
|
||||
ambiguous: 0,
|
||||
failed: 0,
|
||||
truncated: page.truncated,
|
||||
unsafeAttemptOverflow: page.unsafeAttemptOverflow,
|
||||
...(page.nextCursor === undefined ? {} : { nextCursor: page.nextCursor }),
|
||||
};
|
||||
if (page.unsafeAttemptOverflow) return summary;
|
||||
|
||||
for (const candidate of page.candidates) {
|
||||
if (candidate.attempts.length !== 1) {
|
||||
summary.ambiguous += 1;
|
||||
continue;
|
||||
}
|
||||
const attempt = candidate.attempts[0];
|
||||
if (attempt.executorType !== 'local_process') {
|
||||
summary.skipped += 1;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const result = await this.consumer.consume(attempt.attemptId);
|
||||
if (result.status === 'missing') {
|
||||
summary.missing += 1;
|
||||
continue;
|
||||
}
|
||||
if (result.status === 'quarantined') {
|
||||
summary.quarantined += 1;
|
||||
continue;
|
||||
}
|
||||
if (result.status === 'applied') summary.applied += 1;
|
||||
else summary.alreadyTerminal += 1;
|
||||
if (!result.cleaned) summary.cleanupPending += 1;
|
||||
} catch {
|
||||
summary.failed += 1;
|
||||
}
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import {
|
||||
MAX_PRIMARY_RECOVERY_BATCH_SIZE,
|
||||
type PrimaryRunRecoveryCursor,
|
||||
} from '../ports/primaryRunRecoverySource';
|
||||
import type {
|
||||
PrimaryCompletionReceiptScanner,
|
||||
PrimaryCompletionReceiptScanSummary,
|
||||
} from './primaryCompletionReceiptScanner';
|
||||
|
||||
export const MAX_PRIMARY_COMPLETION_RECEIPT_PAGES = 64;
|
||||
|
||||
export type PrimaryCompletionReceiptStopReason =
|
||||
| 'complete'
|
||||
| 'page_limit'
|
||||
| 'unsafe_attempt_overflow'
|
||||
| 'cursor_stalled';
|
||||
|
||||
export interface PrimaryCompletionReceiptSupervisorOptions {
|
||||
cursor?: PrimaryRunRecoveryCursor;
|
||||
pageSize?: number;
|
||||
maxPages?: number;
|
||||
}
|
||||
|
||||
export interface PrimaryCompletionReceiptSupervisorSummary
|
||||
extends Omit<
|
||||
PrimaryCompletionReceiptScanSummary,
|
||||
'truncated' | 'unsafeAttemptOverflow' | 'nextCursor'
|
||||
> {
|
||||
pages: number;
|
||||
stopReason: PrimaryCompletionReceiptStopReason;
|
||||
remaining: boolean;
|
||||
nextCursor?: PrimaryRunRecoveryCursor;
|
||||
}
|
||||
|
||||
function sameCursor(
|
||||
left: PrimaryRunRecoveryCursor | undefined,
|
||||
right: PrimaryRunRecoveryCursor,
|
||||
): boolean {
|
||||
return (
|
||||
left !== undefined &&
|
||||
left.createdAtMs === right.createdAtMs &&
|
||||
left.runId === right.runId
|
||||
);
|
||||
}
|
||||
|
||||
export class PrimaryCompletionReceiptSupervisor {
|
||||
constructor(
|
||||
private readonly scanner: Pick<
|
||||
PrimaryCompletionReceiptScanner,
|
||||
'scanBatch'
|
||||
>,
|
||||
) {}
|
||||
|
||||
async run(
|
||||
options: PrimaryCompletionReceiptSupervisorOptions = {},
|
||||
): Promise<PrimaryCompletionReceiptSupervisorSummary> {
|
||||
const pageSize = options.pageSize ?? 32;
|
||||
const maxPages = options.maxPages ?? 4;
|
||||
if (
|
||||
!Number.isSafeInteger(pageSize) ||
|
||||
pageSize < 1 ||
|
||||
pageSize > MAX_PRIMARY_RECOVERY_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'pageSize must be between 1 and MAX_PRIMARY_RECOVERY_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(maxPages) ||
|
||||
maxPages < 1 ||
|
||||
maxPages > MAX_PRIMARY_COMPLETION_RECEIPT_PAGES
|
||||
) {
|
||||
throw new RangeError(
|
||||
'maxPages must be between 1 and MAX_PRIMARY_COMPLETION_RECEIPT_PAGES',
|
||||
);
|
||||
}
|
||||
|
||||
const total: PrimaryCompletionReceiptSupervisorSummary = {
|
||||
pages: 0,
|
||||
scanned: 0,
|
||||
applied: 0,
|
||||
alreadyTerminal: 0,
|
||||
quarantined: 0,
|
||||
purgedQuarantines: 0,
|
||||
expiredMissing: 0,
|
||||
missing: 0,
|
||||
cleanupPending: 0,
|
||||
skipped: 0,
|
||||
ambiguous: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
let cursor = options.cursor;
|
||||
|
||||
for (let pageNumber = 0; pageNumber < maxPages; pageNumber += 1) {
|
||||
const page = await this.scanner.scanBatch({
|
||||
...(cursor === undefined ? {} : { cursor }),
|
||||
limit: pageSize,
|
||||
});
|
||||
total.pages += 1;
|
||||
total.scanned += page.scanned;
|
||||
total.applied += page.applied;
|
||||
total.alreadyTerminal += page.alreadyTerminal;
|
||||
total.quarantined += page.quarantined;
|
||||
total.purgedQuarantines += page.purgedQuarantines;
|
||||
total.expiredMissing += page.expiredMissing;
|
||||
total.missing += page.missing;
|
||||
total.cleanupPending += page.cleanupPending;
|
||||
total.skipped += page.skipped;
|
||||
total.ambiguous += page.ambiguous;
|
||||
total.failed += page.failed;
|
||||
|
||||
if (page.unsafeAttemptOverflow) {
|
||||
total.stopReason = 'unsafe_attempt_overflow';
|
||||
total.remaining = true;
|
||||
return total;
|
||||
}
|
||||
if (!page.truncated) return total;
|
||||
if (!page.nextCursor || sameCursor(cursor, page.nextCursor)) {
|
||||
total.stopReason = 'cursor_stalled';
|
||||
total.remaining = true;
|
||||
if (page.nextCursor) total.nextCursor = page.nextCursor;
|
||||
return total;
|
||||
}
|
||||
cursor = page.nextCursor;
|
||||
if (pageNumber === maxPages - 1) {
|
||||
total.stopReason = 'page_limit';
|
||||
total.remaining = true;
|
||||
total.nextCursor = cursor;
|
||||
return total;
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
import { createHash, timingSafeEqual } from 'crypto';
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import type { ExecutionOutcome, ExecutionResult } from '../domain/execution';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
RunStatus,
|
||||
} from '../domain/run';
|
||||
import {
|
||||
isTerminalRunAttemptStatus,
|
||||
transitionRun,
|
||||
transitionRunAttempt,
|
||||
type RunDomainEventDraft,
|
||||
} from '../domain/runStateMachine';
|
||||
import type { RunRepository } from '../ports/runRepository';
|
||||
|
||||
interface TerminalMapping {
|
||||
attemptStatus: Exclude<
|
||||
RunAttemptRecord['status'],
|
||||
'claimed' | 'starting' | 'running'
|
||||
>;
|
||||
runStatus: Exclude<
|
||||
RunStatus,
|
||||
| 'created'
|
||||
| 'queued'
|
||||
| 'dispatching'
|
||||
| 'running'
|
||||
| 'waiting_approval'
|
||||
| 'retry_wait'
|
||||
>;
|
||||
errorCode?: string;
|
||||
errorSummary?: string;
|
||||
}
|
||||
|
||||
const TERMINAL_MAPPING: Readonly<Record<ExecutionOutcome, TerminalMapping>> = {
|
||||
succeeded: {
|
||||
attemptStatus: 'succeeded',
|
||||
runStatus: 'succeeded',
|
||||
},
|
||||
failed: {
|
||||
attemptStatus: 'failed',
|
||||
runStatus: 'failed',
|
||||
errorCode: 'EXECUTION_FAILED',
|
||||
errorSummary: 'Execution completed without success',
|
||||
},
|
||||
cancelled: {
|
||||
attemptStatus: 'cancelled',
|
||||
runStatus: 'cancelled',
|
||||
errorCode: 'EXECUTION_CANCELLED',
|
||||
errorSummary: 'Execution was cancelled',
|
||||
},
|
||||
timed_out: {
|
||||
attemptStatus: 'timed_out',
|
||||
runStatus: 'timed_out',
|
||||
errorCode: 'EXECUTION_TIMED_OUT',
|
||||
errorSummary: 'Execution exceeded its configured timeout',
|
||||
},
|
||||
lost: {
|
||||
attemptStatus: 'lost',
|
||||
runStatus: 'lost',
|
||||
errorCode: 'EXECUTION_LOST',
|
||||
errorSummary: 'Execution ownership was lost',
|
||||
},
|
||||
};
|
||||
|
||||
export const MAX_PRIMARY_COMPLETION_RETRIES = 4;
|
||||
const TOKEN_PATTERN = /^[A-Za-z0-9_-]{32,128}$/;
|
||||
|
||||
export type PrimaryCompletionSource =
|
||||
| { kind: 'executor'; executorType: string }
|
||||
| { kind: 'receipt'; token: string };
|
||||
|
||||
export interface PrimaryRunCompletionCommand {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
callbackSequence: number;
|
||||
result: ExecutionResult;
|
||||
source: PrimaryCompletionSource;
|
||||
}
|
||||
|
||||
export interface PrimaryRunCompletionResult {
|
||||
status: 'applied' | 'already_terminal';
|
||||
run: RunRecord;
|
||||
attempt: RunAttemptRecord;
|
||||
result: ExecutionResult;
|
||||
}
|
||||
|
||||
export type PrimaryCompletionEventIdFactory = () => string;
|
||||
|
||||
export class PrimaryCompletionNotFoundError extends Error {
|
||||
constructor() {
|
||||
super('Primary completion target was not found');
|
||||
this.name = 'PrimaryCompletionNotFoundError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryCompletionUnauthorizedError extends Error {
|
||||
constructor() {
|
||||
super('Primary completion source is not authorized');
|
||||
this.name = 'PrimaryCompletionUnauthorizedError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryCompletionSequenceError extends Error {
|
||||
constructor() {
|
||||
super('Primary completion callback sequence is invalid');
|
||||
this.name = 'PrimaryCompletionSequenceError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryCompletionStateError extends Error {
|
||||
constructor() {
|
||||
super('Primary completion target state is inconsistent');
|
||||
this.name = 'PrimaryCompletionStateError';
|
||||
}
|
||||
}
|
||||
|
||||
class PrimaryCompletionConcurrentWriteError extends Error {}
|
||||
|
||||
export function hashPrimaryCompletionToken(token: string): string {
|
||||
if (!TOKEN_PATTERN.test(token)) {
|
||||
throw new TypeError('Primary completion token is invalid');
|
||||
}
|
||||
return createHash('sha256').update(token, 'utf8').digest('hex');
|
||||
}
|
||||
|
||||
function validateResult(result: ExecutionResult): void {
|
||||
if (!Object.hasOwn(TERMINAL_MAPPING, result.outcome)) {
|
||||
throw new TypeError('Primary completion outcome is invalid');
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(result.startedAtMs) ||
|
||||
result.startedAtMs < 0 ||
|
||||
!Number.isSafeInteger(result.finishedAtMs) ||
|
||||
result.finishedAtMs < result.startedAtMs
|
||||
) {
|
||||
throw new TypeError('Primary completion timestamps are invalid');
|
||||
}
|
||||
if (
|
||||
result.exitCode !== undefined &&
|
||||
(!Number.isInteger(result.exitCode) ||
|
||||
result.exitCode < 0 ||
|
||||
result.exitCode > 255)
|
||||
) {
|
||||
throw new TypeError('Primary completion exitCode is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function mappingFor(run: RunRecord, result: ExecutionResult): TerminalMapping {
|
||||
if (run.cancelRequestedAtMs !== undefined) {
|
||||
return run.cancelReason === 'timeout'
|
||||
? TERMINAL_MAPPING.timed_out
|
||||
: TERMINAL_MAPPING.cancelled;
|
||||
}
|
||||
return TERMINAL_MAPPING[result.outcome];
|
||||
}
|
||||
|
||||
function sameTerminalState(
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
mapping: TerminalMapping,
|
||||
): boolean {
|
||||
return (
|
||||
attempt.status === mapping.attemptStatus && run.status === mapping.runStatus
|
||||
);
|
||||
}
|
||||
|
||||
function authorize(
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
source: PrimaryCompletionSource,
|
||||
): void {
|
||||
if (run.executionOwner !== 'runtime') {
|
||||
throw new PrimaryCompletionUnauthorizedError();
|
||||
}
|
||||
if (source.kind === 'executor') {
|
||||
if (!source.executorType || attempt.executorType !== source.executorType) {
|
||||
throw new PrimaryCompletionUnauthorizedError();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let actualHash: string;
|
||||
try {
|
||||
actualHash = hashPrimaryCompletionToken(source.token);
|
||||
} catch {
|
||||
throw new PrimaryCompletionUnauthorizedError();
|
||||
}
|
||||
const expectedHash = attempt.callbackTokenHash;
|
||||
if (!expectedHash || !/^[a-f0-9]{64}$/.test(expectedHash)) {
|
||||
throw new PrimaryCompletionUnauthorizedError();
|
||||
}
|
||||
const expected = Buffer.from(expectedHash, 'hex');
|
||||
const actual = Buffer.from(actualHash, 'hex');
|
||||
if (!timingSafeEqual(expected, actual)) {
|
||||
throw new PrimaryCompletionUnauthorizedError();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The only terminal completion transaction for both live Executor callbacks
|
||||
* and durable receipt replay. Attempt, Run and both events commit atomically.
|
||||
*/
|
||||
export class PrimaryRunCompletionService {
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly createEventId: PrimaryCompletionEventIdFactory = uuidV7,
|
||||
) {}
|
||||
|
||||
async complete(
|
||||
command: PrimaryRunCompletionCommand,
|
||||
): Promise<PrimaryRunCompletionResult> {
|
||||
if (
|
||||
!Number.isSafeInteger(command.callbackSequence) ||
|
||||
command.callbackSequence < 1
|
||||
) {
|
||||
throw new PrimaryCompletionSequenceError();
|
||||
}
|
||||
validateResult(command.result);
|
||||
|
||||
for (let retry = 0; retry <= MAX_PRIMARY_COMPLETION_RETRIES; retry += 1) {
|
||||
try {
|
||||
return await this.repository.transaction(async (transaction) => {
|
||||
const run = await transaction.findRunById(command.runId);
|
||||
const attempt = await transaction.findAttemptById(command.attemptId);
|
||||
if (!run || !attempt || attempt.runId !== run.id) {
|
||||
throw new PrimaryCompletionNotFoundError();
|
||||
}
|
||||
authorize(run, attempt, command.source);
|
||||
const mapping = mappingFor(run, command.result);
|
||||
|
||||
if (isTerminalRunAttemptStatus(attempt.status)) {
|
||||
if (attempt.callbackSequence !== command.callbackSequence) {
|
||||
throw new PrimaryCompletionSequenceError();
|
||||
}
|
||||
if (!sameTerminalState(run, attempt, mapping)) {
|
||||
throw new PrimaryCompletionStateError();
|
||||
}
|
||||
return {
|
||||
status: 'already_terminal',
|
||||
run,
|
||||
attempt,
|
||||
result: command.result,
|
||||
};
|
||||
}
|
||||
if (command.callbackSequence !== attempt.callbackSequence + 1) {
|
||||
throw new PrimaryCompletionSequenceError();
|
||||
}
|
||||
if (
|
||||
run.status === 'succeeded' ||
|
||||
run.status === 'failed' ||
|
||||
run.status === 'cancelled' ||
|
||||
run.status === 'timed_out'
|
||||
) {
|
||||
throw new PrimaryCompletionStateError();
|
||||
}
|
||||
|
||||
const atMs = Math.max(
|
||||
run.createdAtMs,
|
||||
run.startedAtMs ?? 0,
|
||||
attempt.createdAtMs,
|
||||
attempt.startedAtMs ?? 0,
|
||||
command.result.finishedAtMs,
|
||||
);
|
||||
const attemptDecision = transitionRunAttempt(run, attempt, {
|
||||
to: mapping.attemptStatus,
|
||||
expectedRunVersion: run.version,
|
||||
atMs,
|
||||
callbackSequence: command.callbackSequence,
|
||||
...(command.result.exitCode === undefined
|
||||
? {}
|
||||
: { exitCode: command.result.exitCode }),
|
||||
...(mapping.errorCode === undefined
|
||||
? {}
|
||||
: { errorCode: mapping.errorCode }),
|
||||
...(mapping.errorSummary === undefined
|
||||
? {}
|
||||
: { errorSummary: mapping.errorSummary }),
|
||||
});
|
||||
const runDecision = transitionRun(attemptDecision.run, {
|
||||
to: mapping.runStatus,
|
||||
expectedVersion: attemptDecision.run.version,
|
||||
atMs,
|
||||
...(mapping.errorCode === undefined
|
||||
? {}
|
||||
: { errorCode: mapping.errorCode }),
|
||||
...(mapping.errorSummary === undefined
|
||||
? {}
|
||||
: { errorSummary: mapping.errorSummary }),
|
||||
});
|
||||
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
attemptDecision.run,
|
||||
run.version,
|
||||
))
|
||||
) {
|
||||
throw new PrimaryCompletionConcurrentWriteError();
|
||||
}
|
||||
if (
|
||||
!(await transaction.compareAndSetAttempt(attemptDecision.attempt, {
|
||||
status: attempt.status,
|
||||
callbackSequence: attempt.callbackSequence,
|
||||
}))
|
||||
) {
|
||||
throw new PrimaryCompletionConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
this.event(
|
||||
attemptDecision.run,
|
||||
attemptDecision.event,
|
||||
attempt.id,
|
||||
command.source,
|
||||
`primary-completion:${attempt.id}:${command.callbackSequence}:attempt`,
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
runDecision.run,
|
||||
attemptDecision.run.version,
|
||||
))
|
||||
) {
|
||||
throw new PrimaryCompletionConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
this.event(
|
||||
runDecision.run,
|
||||
runDecision.event,
|
||||
attempt.id,
|
||||
command.source,
|
||||
`primary-completion:${attempt.id}:${command.callbackSequence}:run`,
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
|
||||
return {
|
||||
status: 'applied',
|
||||
run: runDecision.run,
|
||||
attempt: attemptDecision.attempt,
|
||||
result: command.result,
|
||||
};
|
||||
});
|
||||
} catch (error) {
|
||||
if (
|
||||
!(error instanceof PrimaryCompletionConcurrentWriteError) ||
|
||||
retry === MAX_PRIMARY_COMPLETION_RETRIES
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new Error('Primary completion retry budget was exhausted');
|
||||
}
|
||||
|
||||
private event(
|
||||
run: RunRecord,
|
||||
draft: RunDomainEventDraft,
|
||||
attemptId: string,
|
||||
source: PrimaryCompletionSource,
|
||||
dedupeKey: string,
|
||||
createdAtMs: number,
|
||||
): RunEventRecord {
|
||||
return {
|
||||
id: this.createEventId(),
|
||||
runId: run.id,
|
||||
sequence: draft.sequence,
|
||||
type: draft.type,
|
||||
dedupeKey,
|
||||
actorType: 'executor',
|
||||
actorId:
|
||||
source.kind === 'executor' ? source.executorType : 'completion-receipt',
|
||||
attemptId,
|
||||
payload: draft.payload,
|
||||
createdAtMs,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import type {
|
||||
ExecutionOrigin,
|
||||
RunAttemptRecord,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
} from '../domain/run';
|
||||
import {
|
||||
reserveRunEvent,
|
||||
transitionRun,
|
||||
type RunDomainEventDraft,
|
||||
} from '../domain/runStateMachine';
|
||||
import { RunVersionConflictError } from '../domain/stateMachineErrors';
|
||||
import type { ExecutorType } from '../domain/execution';
|
||||
import {
|
||||
assertAdmittedRunRetryPolicy,
|
||||
type RunRetryPolicyDefinition,
|
||||
type RunRetryPolicyRecord,
|
||||
} from '../domain/runRetryPolicy';
|
||||
import type { RunRepository } from '../ports/runRepository';
|
||||
import type { RunCommandActor } from './runCommandService';
|
||||
|
||||
export interface PrimaryRunDefinition {
|
||||
projectId: string;
|
||||
taskId: string;
|
||||
taskRevision: string;
|
||||
taskName?: string;
|
||||
taskSnapshotRef?: string;
|
||||
legacyCronId?: number;
|
||||
triggerId?: string;
|
||||
triggerType: string;
|
||||
executionOrigin: ExecutionOrigin;
|
||||
triggeredBy?: string;
|
||||
requestId?: string;
|
||||
scheduledForMs?: number;
|
||||
priority?: number;
|
||||
idempotencyKey?: string;
|
||||
inputRef?: string;
|
||||
outputRef?: string;
|
||||
acceptedAtMs: number;
|
||||
actor: RunCommandActor;
|
||||
retryPolicy?: RunRetryPolicyDefinition;
|
||||
}
|
||||
|
||||
export interface PrimaryRunReference {
|
||||
run: RunRecord;
|
||||
attempt: RunAttemptRecord;
|
||||
}
|
||||
|
||||
export type PrimaryRunIdFactory = () => string;
|
||||
|
||||
/**
|
||||
* Creates the durable runtime-owned aggregate before an Executor can observe it.
|
||||
*/
|
||||
export class PrimaryRunCreator {
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly createId: PrimaryRunIdFactory = uuidV7,
|
||||
) {}
|
||||
|
||||
async create(
|
||||
definition: PrimaryRunDefinition,
|
||||
executorType: ExecutorType,
|
||||
): Promise<PrimaryRunReference> {
|
||||
const initialRun: RunRecord = {
|
||||
id: this.createId(),
|
||||
projectId: definition.projectId,
|
||||
taskId: definition.taskId,
|
||||
taskRevision: definition.taskRevision,
|
||||
...(definition.taskName === undefined
|
||||
? {}
|
||||
: { taskName: definition.taskName }),
|
||||
...(definition.taskSnapshotRef === undefined
|
||||
? {}
|
||||
: { taskSnapshotRef: definition.taskSnapshotRef }),
|
||||
...(definition.legacyCronId === undefined
|
||||
? {}
|
||||
: { legacyCronId: definition.legacyCronId }),
|
||||
...(definition.triggerId === undefined
|
||||
? {}
|
||||
: { triggerId: definition.triggerId }),
|
||||
triggerType: definition.triggerType,
|
||||
executionOrigin: definition.executionOrigin,
|
||||
executionOwner: 'runtime',
|
||||
...(definition.triggeredBy === undefined
|
||||
? {}
|
||||
: { triggeredBy: definition.triggeredBy }),
|
||||
...(definition.requestId === undefined
|
||||
? {}
|
||||
: { requestId: definition.requestId }),
|
||||
...(definition.scheduledForMs === undefined
|
||||
? {}
|
||||
: { scheduledForMs: definition.scheduledForMs }),
|
||||
status: 'created',
|
||||
version: 0,
|
||||
eventSequence: 0,
|
||||
priority: definition.priority ?? 0,
|
||||
...(definition.idempotencyKey === undefined
|
||||
? {}
|
||||
: { idempotencyKey: definition.idempotencyKey }),
|
||||
...(definition.inputRef === undefined
|
||||
? {}
|
||||
: { inputRef: definition.inputRef }),
|
||||
...(definition.outputRef === undefined
|
||||
? {}
|
||||
: { outputRef: definition.outputRef }),
|
||||
createdAtMs: definition.acceptedAtMs,
|
||||
};
|
||||
const initialAttempt: RunAttemptRecord = {
|
||||
id: this.createId(),
|
||||
runId: initialRun.id,
|
||||
attempt: 1,
|
||||
status: 'claimed',
|
||||
executorType,
|
||||
callbackSequence: 0,
|
||||
createdAtMs: definition.acceptedAtMs,
|
||||
};
|
||||
let retryPolicy: RunRetryPolicyRecord | undefined;
|
||||
if (definition.retryPolicy !== undefined) {
|
||||
assertAdmittedRunRetryPolicy(definition.retryPolicy);
|
||||
retryPolicy = {
|
||||
runId: initialRun.id,
|
||||
...definition.retryPolicy,
|
||||
version: 0,
|
||||
createdAtMs: definition.acceptedAtMs,
|
||||
updatedAtMs: definition.acceptedAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
const run = await this.repository.transaction(async (transaction) => {
|
||||
await transaction.insertRun(initialRun);
|
||||
await transaction.insertAttempt(initialAttempt);
|
||||
if (retryPolicy !== undefined) {
|
||||
await transaction.insertRetryPolicy(retryPolicy);
|
||||
}
|
||||
|
||||
const created = reserveRunEvent(initialRun, 0);
|
||||
const createdUpdated = await transaction.compareAndSetRun(created.run, 0);
|
||||
if (!createdUpdated) {
|
||||
throw new RunVersionConflictError(initialRun.id, 0, initialRun.version);
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
this.event(
|
||||
created.run,
|
||||
{
|
||||
sequence: created.sequence,
|
||||
type: 'run.created',
|
||||
payload: {
|
||||
status: 'created',
|
||||
version: created.run.version,
|
||||
execution_owner: 'runtime',
|
||||
},
|
||||
},
|
||||
definition.actor,
|
||||
`primary-run-created:${initialRun.id}`,
|
||||
definition.acceptedAtMs,
|
||||
),
|
||||
);
|
||||
|
||||
const queued = transitionRun(created.run, {
|
||||
to: 'queued',
|
||||
expectedVersion: created.run.version,
|
||||
atMs: definition.acceptedAtMs,
|
||||
});
|
||||
const queuedUpdated = await transaction.compareAndSetRun(
|
||||
queued.run,
|
||||
created.run.version,
|
||||
);
|
||||
if (!queuedUpdated) {
|
||||
throw new RunVersionConflictError(
|
||||
initialRun.id,
|
||||
created.run.version,
|
||||
created.run.version,
|
||||
);
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
this.event(
|
||||
queued.run,
|
||||
queued.event,
|
||||
definition.actor,
|
||||
`primary-run-queued:${initialRun.id}`,
|
||||
definition.acceptedAtMs,
|
||||
),
|
||||
);
|
||||
return queued.run;
|
||||
});
|
||||
|
||||
return { run, attempt: initialAttempt };
|
||||
}
|
||||
|
||||
private event(
|
||||
run: RunRecord,
|
||||
draft: RunDomainEventDraft,
|
||||
actor: RunCommandActor,
|
||||
dedupeKey: string,
|
||||
createdAtMs: number,
|
||||
): RunEventRecord {
|
||||
return {
|
||||
id: this.createId(),
|
||||
runId: run.id,
|
||||
sequence: draft.sequence,
|
||||
type: draft.type,
|
||||
dedupeKey,
|
||||
actorType: actor.type,
|
||||
...(actor.id === undefined ? {} : { actorId: actor.id }),
|
||||
payload: draft.payload,
|
||||
createdAtMs,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,706 @@
|
||||
import { randomBytes } from 'crypto';
|
||||
import type {
|
||||
ExecutionContext,
|
||||
ExecutionHandle,
|
||||
ExecutionResult,
|
||||
ExecutionSpec,
|
||||
ExecutionStopReason,
|
||||
ExecutionStopResult,
|
||||
} from '../domain/execution';
|
||||
import type { RunAttemptRecord, RunRecord } from '../domain/run';
|
||||
import {
|
||||
assertAdmittedRunRetryPolicy,
|
||||
type RunRetryPolicyDefinition,
|
||||
} from '../domain/runRetryPolicy';
|
||||
import {
|
||||
MAX_LOG_ARTIFACT_ID_LENGTH,
|
||||
isTerminalRunAttemptStatus,
|
||||
isTerminalRunStatus,
|
||||
} from '../domain/runStateMachine';
|
||||
import type { Executor } from '../ports/executor';
|
||||
import type { CompletionReceiptJournal } from '../ports/completionReceiptJournal';
|
||||
import type { PrimaryRunIdempotencyLookup } from '../ports/primaryRunIdempotencyLookup';
|
||||
import type { RunRepository } from '../ports/runRepository';
|
||||
import type { RunRetryPolicyAdmission } from '../ports/runRetryPolicyAdmission';
|
||||
import { DuplicateIdempotencyKeyError } from '../domain/repositoryErrors';
|
||||
import {
|
||||
PrimaryRunCreator,
|
||||
type PrimaryRunDefinition,
|
||||
type PrimaryRunIdFactory,
|
||||
type PrimaryRunReference,
|
||||
} from './primaryRunCreator';
|
||||
import { RunCommandService } from './runCommandService';
|
||||
import {
|
||||
hashPrimaryCompletionToken,
|
||||
PrimaryRunCompletionService,
|
||||
} from './primaryRunCompletionService';
|
||||
|
||||
export interface PrimaryRunClock {
|
||||
now(): number;
|
||||
}
|
||||
|
||||
export interface PrimaryRunOrchestratorOptions {
|
||||
clock?: PrimaryRunClock;
|
||||
createId?: PrimaryRunIdFactory;
|
||||
idempotencyLookup?: PrimaryRunIdempotencyLookup;
|
||||
createCallbackToken?: () => string;
|
||||
completionReceiptJournal?: Pick<CompletionReceiptJournal, 'register'>;
|
||||
retryPolicyAdmission?: RunRetryPolicyAdmission;
|
||||
}
|
||||
|
||||
export interface PrimaryRunStartCommand {
|
||||
definition: Omit<PrimaryRunDefinition, 'acceptedAtMs' | 'retryPolicy'> & {
|
||||
acceptedAtMs?: number;
|
||||
};
|
||||
timeoutMs?: number;
|
||||
createSpec(reference: PrimaryRunReference): ExecutionSpec;
|
||||
context: ExecutionContext;
|
||||
}
|
||||
|
||||
/**
|
||||
* Trusted local-dispatch input for an aggregate that already owns a claimed
|
||||
* Attempt. Callers must materialize the spec from the persisted Task revision.
|
||||
*/
|
||||
export interface PrimaryClaimedRunStartCommand {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
timeoutMs?: number;
|
||||
createSpec(reference: PrimaryRunReference): ExecutionSpec;
|
||||
context: ExecutionContext;
|
||||
logArtifactId?: string;
|
||||
}
|
||||
|
||||
export interface PrimaryRunCompletion {
|
||||
run: RunRecord;
|
||||
attempt: RunAttemptRecord;
|
||||
result: ExecutionResult;
|
||||
}
|
||||
|
||||
export interface ActivePrimaryRun extends PrimaryRunReference {
|
||||
handle: ExecutionHandle;
|
||||
completion: Promise<PrimaryRunCompletion>;
|
||||
cancel(reason: ExecutionStopReason): Promise<ExecutionStopResult>;
|
||||
}
|
||||
|
||||
export class PrimaryRunLaunchError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly reference: PrimaryRunReference,
|
||||
options?: ErrorOptions,
|
||||
) {
|
||||
super(message, options);
|
||||
this.name = 'PrimaryRunLaunchError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryRunNotActiveError extends Error {
|
||||
constructor(readonly runId: string) {
|
||||
super(`Primary Run is not active: ${runId}`);
|
||||
this.name = 'PrimaryRunNotActiveError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryRunDuplicateRequestError extends Error {
|
||||
constructor(
|
||||
readonly projectId: string,
|
||||
readonly idempotencyKey: string,
|
||||
readonly existingRunId: string,
|
||||
) {
|
||||
super('A Primary Run already exists for this idempotent request');
|
||||
this.name = 'PrimaryRunDuplicateRequestError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryRunIdempotencyUnavailableError extends Error {
|
||||
constructor() {
|
||||
super('Primary Run idempotency lookup is not configured');
|
||||
this.name = 'PrimaryRunIdempotencyUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryRunRetryPolicyAuthorityError extends Error {
|
||||
readonly code = 'PRIMARY_RUN_RETRY_POLICY_AUTHORITY_REQUIRED';
|
||||
|
||||
constructor() {
|
||||
super('Run requests cannot self-assert automatic retry safety');
|
||||
this.name = 'PrimaryRunRetryPolicyAuthorityError';
|
||||
}
|
||||
}
|
||||
|
||||
export type PrimaryClaimedRunRejectionReason =
|
||||
| 'run_not_found'
|
||||
| 'attempt_not_found'
|
||||
| 'aggregate_mismatch'
|
||||
| 'not_latest_attempt'
|
||||
| 'not_queued'
|
||||
| 'not_claimed'
|
||||
| 'stale_execution_authority'
|
||||
| 'executor_mismatch'
|
||||
| 'cancellation_requested'
|
||||
| 'already_active';
|
||||
|
||||
export class PrimaryClaimedRunRejectedError extends Error {
|
||||
readonly code = 'PRIMARY_CLAIMED_RUN_REJECTED';
|
||||
|
||||
constructor(readonly reason: PrimaryClaimedRunRejectionReason) {
|
||||
super(`Claimed Primary Run activation was rejected: ${reason}`);
|
||||
this.name = 'PrimaryClaimedRunRejectedError';
|
||||
}
|
||||
}
|
||||
|
||||
interface ActiveExecution {
|
||||
handle: ExecutionHandle;
|
||||
completion: Promise<PrimaryRunCompletion>;
|
||||
}
|
||||
|
||||
const ALREADY_EXITED_STOP_RESULT: ExecutionStopResult = {
|
||||
status: 'already_exited',
|
||||
termSignalSent: false,
|
||||
killSignalSent: false,
|
||||
};
|
||||
|
||||
/**
|
||||
* Serializes durable Run transitions around a single Executor side effect.
|
||||
* It deliberately owns no scheduler or HTTP routing policy.
|
||||
*/
|
||||
export class PrimaryRunOrchestrator {
|
||||
private readonly clock: PrimaryRunClock;
|
||||
private readonly creator: PrimaryRunCreator;
|
||||
private readonly commands: RunCommandService;
|
||||
private readonly completions: PrimaryRunCompletionService;
|
||||
private readonly createCallbackToken: () => string;
|
||||
private readonly idempotencyLookup?: PrimaryRunIdempotencyLookup;
|
||||
private readonly completionReceiptJournal?: Pick<
|
||||
CompletionReceiptJournal,
|
||||
'register'
|
||||
>;
|
||||
private readonly retryPolicyAdmission?: RunRetryPolicyAdmission;
|
||||
private readonly active = new Map<string, ActiveExecution>();
|
||||
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly executor: Executor,
|
||||
options: PrimaryRunOrchestratorOptions = {},
|
||||
) {
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.creator = new PrimaryRunCreator(repository, options.createId);
|
||||
this.commands = new RunCommandService(repository, options.createId);
|
||||
this.completions = new PrimaryRunCompletionService(
|
||||
repository,
|
||||
options.createId,
|
||||
);
|
||||
this.createCallbackToken =
|
||||
options.createCallbackToken ??
|
||||
(() => randomBytes(32).toString('base64url'));
|
||||
this.idempotencyLookup = options.idempotencyLookup;
|
||||
this.completionReceiptJournal = options.completionReceiptJournal;
|
||||
this.retryPolicyAdmission = options.retryPolicyAdmission;
|
||||
}
|
||||
|
||||
async start(command: PrimaryRunStartCommand): Promise<ActivePrimaryRun> {
|
||||
this.assertTimeout(command.timeoutMs);
|
||||
if (
|
||||
Object.prototype.hasOwnProperty.call(command.definition, 'retryPolicy')
|
||||
) {
|
||||
throw new PrimaryRunRetryPolicyAuthorityError();
|
||||
}
|
||||
const retryPolicy = await this.admitRetryPolicy(command.definition);
|
||||
const acceptedAtMs = command.definition.acceptedAtMs ?? this.clock.now();
|
||||
const reference = await this.createPrimaryRun(
|
||||
{
|
||||
...command.definition,
|
||||
acceptedAtMs,
|
||||
...(retryPolicy === undefined ? {} : { retryPolicy }),
|
||||
},
|
||||
this.executor.type,
|
||||
);
|
||||
return this.activateReference(reference, command);
|
||||
}
|
||||
|
||||
async activateClaimed(
|
||||
command: PrimaryClaimedRunStartCommand,
|
||||
): Promise<ActivePrimaryRun> {
|
||||
this.assertTimeout(command.timeoutMs);
|
||||
this.assertLogArtifactId(command.logArtifactId);
|
||||
const reference = await this.loadClaimedReference(
|
||||
command.runId,
|
||||
command.attemptId,
|
||||
);
|
||||
return this.activateReference(reference, command);
|
||||
}
|
||||
|
||||
private async activateReference(
|
||||
initialReference: PrimaryRunReference,
|
||||
command: Pick<
|
||||
PrimaryClaimedRunStartCommand,
|
||||
'timeoutMs' | 'createSpec' | 'context' | 'logArtifactId'
|
||||
>,
|
||||
): Promise<ActivePrimaryRun> {
|
||||
const callbackToken = this.createCallbackToken();
|
||||
const callbackTokenHash = hashPrimaryCompletionToken(callbackToken);
|
||||
const callbackSequence = initialReference.attempt.callbackSequence + 1;
|
||||
let reference = initialReference;
|
||||
|
||||
reference = await this.prepareForSpawn(
|
||||
reference,
|
||||
command.timeoutMs,
|
||||
callbackTokenHash,
|
||||
command.logArtifactId,
|
||||
);
|
||||
|
||||
let spec: ExecutionSpec;
|
||||
let handle: ExecutionHandle;
|
||||
try {
|
||||
await this.completionReceiptJournal?.register({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
registeredAtMs: reference.attempt.createdAtMs,
|
||||
});
|
||||
spec = command.createSpec(reference);
|
||||
this.assertSpecMatches(reference, spec, command.timeoutMs);
|
||||
handle = await this.executor.start(spec, {
|
||||
...command.context,
|
||||
completionCallback: {
|
||||
token: callbackToken,
|
||||
callbackSequence,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
reference = await this.recordStartFailure(reference);
|
||||
throw new PrimaryRunLaunchError(
|
||||
'Primary Run could not start its Executor',
|
||||
reference,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
this.assertHandleMatches(reference, handle);
|
||||
reference = await this.recordRunning(reference, handle);
|
||||
} catch (error) {
|
||||
void handle.completion.catch(() => undefined);
|
||||
await this.compensateActivationFailure(reference, handle);
|
||||
const latest = await this.loadReference(reference);
|
||||
throw new PrimaryRunLaunchError(
|
||||
'Primary Run could not persist Executor ownership',
|
||||
latest,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
|
||||
const completion = handle.completion.then(
|
||||
(result) =>
|
||||
this.completions.complete({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
callbackSequence,
|
||||
result,
|
||||
source: { kind: 'executor', executorType: this.executor.type },
|
||||
}),
|
||||
() =>
|
||||
this.completions.complete({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
callbackSequence,
|
||||
result: {
|
||||
outcome: 'lost',
|
||||
startedAtMs: handle.startedAtMs,
|
||||
finishedAtMs: this.atOrAfter(handle.startedAtMs),
|
||||
errorCode: 'EXECUTOR_COMPLETION_REJECTED',
|
||||
errorSummary: 'Executor completion channel rejected',
|
||||
},
|
||||
source: { kind: 'executor', executorType: this.executor.type },
|
||||
}),
|
||||
);
|
||||
this.active.set(reference.run.id, { handle, completion });
|
||||
void completion.then(
|
||||
() => this.deleteActive(reference.run.id, handle),
|
||||
() => this.deleteActive(reference.run.id, handle),
|
||||
);
|
||||
|
||||
return {
|
||||
...reference,
|
||||
handle,
|
||||
completion,
|
||||
cancel: (reason) => this.cancel(reference.run.id, reason),
|
||||
};
|
||||
}
|
||||
|
||||
private async loadClaimedReference(
|
||||
runId: string,
|
||||
attemptId: string,
|
||||
): Promise<PrimaryRunReference> {
|
||||
const [run, attempt, latestAttempt] = await Promise.all([
|
||||
this.repository.findRunById(runId),
|
||||
this.repository.findAttemptById(attemptId),
|
||||
this.repository.findLatestAttemptByRunId(runId),
|
||||
]);
|
||||
if (!run) throw new PrimaryClaimedRunRejectedError('run_not_found');
|
||||
if (!attempt) {
|
||||
throw new PrimaryClaimedRunRejectedError('attempt_not_found');
|
||||
}
|
||||
if (
|
||||
attempt.runId !== run.id ||
|
||||
run.executionOwner !== 'runtime' ||
|
||||
latestAttempt?.runId !== run.id
|
||||
) {
|
||||
throw new PrimaryClaimedRunRejectedError('aggregate_mismatch');
|
||||
}
|
||||
if (latestAttempt.id !== attempt.id) {
|
||||
throw new PrimaryClaimedRunRejectedError('not_latest_attempt');
|
||||
}
|
||||
if (run.status !== 'queued') {
|
||||
throw new PrimaryClaimedRunRejectedError('not_queued');
|
||||
}
|
||||
if (attempt.status !== 'claimed') {
|
||||
throw new PrimaryClaimedRunRejectedError('not_claimed');
|
||||
}
|
||||
if (
|
||||
attempt.callbackSequence !== 0 ||
|
||||
attempt.callbackTokenHash !== undefined ||
|
||||
attempt.workerId !== undefined ||
|
||||
attempt.executorHandle !== undefined ||
|
||||
attempt.pid !== undefined ||
|
||||
attempt.logArtifactId !== undefined ||
|
||||
attempt.leaseToken !== undefined ||
|
||||
attempt.leaseExpiresAtMs !== undefined ||
|
||||
attempt.startedAtMs !== undefined ||
|
||||
attempt.finishedAtMs !== undefined ||
|
||||
attempt.exitCode !== undefined ||
|
||||
attempt.errorCode !== undefined ||
|
||||
attempt.errorSummary !== undefined
|
||||
) {
|
||||
throw new PrimaryClaimedRunRejectedError('stale_execution_authority');
|
||||
}
|
||||
if (attempt.executorType !== this.executor.type) {
|
||||
throw new PrimaryClaimedRunRejectedError('executor_mismatch');
|
||||
}
|
||||
if (run.cancelRequestedAtMs !== undefined) {
|
||||
throw new PrimaryClaimedRunRejectedError('cancellation_requested');
|
||||
}
|
||||
if (this.active.has(run.id)) {
|
||||
throw new PrimaryClaimedRunRejectedError('already_active');
|
||||
}
|
||||
return { run, attempt };
|
||||
}
|
||||
|
||||
private async admitRetryPolicy(
|
||||
definition: PrimaryRunStartCommand['definition'],
|
||||
): Promise<RunRetryPolicyDefinition | undefined> {
|
||||
if (!this.retryPolicyAdmission) return undefined;
|
||||
const admitted = await this.retryPolicyAdmission.resolve(
|
||||
Object.freeze({
|
||||
projectId: definition.projectId,
|
||||
taskId: definition.taskId,
|
||||
taskRevision: definition.taskRevision,
|
||||
triggerType: definition.triggerType,
|
||||
executionOrigin: definition.executionOrigin,
|
||||
}),
|
||||
);
|
||||
if (admitted === undefined) return undefined;
|
||||
const policy: RunRetryPolicyDefinition = {
|
||||
maxAttempts: admitted.maxAttempts,
|
||||
retryOnLost: admitted.retryOnLost,
|
||||
safety: admitted.safety,
|
||||
backoffBaseMs: admitted.backoffBaseMs,
|
||||
backoffMaxMs: admitted.backoffMaxMs,
|
||||
};
|
||||
assertAdmittedRunRetryPolicy(policy);
|
||||
return policy;
|
||||
}
|
||||
|
||||
async cancel(
|
||||
runId: string,
|
||||
reason: ExecutionStopReason,
|
||||
): Promise<ExecutionStopResult> {
|
||||
const active = this.active.get(runId);
|
||||
if (!active) throw new PrimaryRunNotActiveError(runId);
|
||||
const request = await this.commands.requestCancellation({
|
||||
runId,
|
||||
attemptId: active.handle.attemptId,
|
||||
atMs: this.atOrAfter(reason.requestedAtMs, active.handle.startedAtMs),
|
||||
reason: reason.kind,
|
||||
actor: this.cancellationActor(reason),
|
||||
});
|
||||
if (request.status === 'already_terminal') {
|
||||
return ALREADY_EXITED_STOP_RESULT;
|
||||
}
|
||||
return this.executor.stop(active.handle, reason);
|
||||
}
|
||||
|
||||
isActive(runId: string): boolean {
|
||||
return this.active.has(runId);
|
||||
}
|
||||
|
||||
private async createPrimaryRun(
|
||||
definition: PrimaryRunDefinition,
|
||||
executorType: Executor['type'],
|
||||
): Promise<PrimaryRunReference> {
|
||||
const key = definition.idempotencyKey;
|
||||
if (key === undefined) {
|
||||
return this.creator.create(definition, executorType);
|
||||
}
|
||||
if (!this.idempotencyLookup) {
|
||||
throw new PrimaryRunIdempotencyUnavailableError();
|
||||
}
|
||||
|
||||
const existingRunId = await this.idempotencyLookup.findRunId(
|
||||
definition.projectId,
|
||||
key,
|
||||
);
|
||||
if (existingRunId) {
|
||||
throw new PrimaryRunDuplicateRequestError(
|
||||
definition.projectId,
|
||||
key,
|
||||
existingRunId,
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
return await this.creator.create(definition, executorType);
|
||||
} catch (error) {
|
||||
if (!(error instanceof DuplicateIdempotencyKeyError)) throw error;
|
||||
const racedRunId = await this.idempotencyLookup.findRunId(
|
||||
definition.projectId,
|
||||
key,
|
||||
);
|
||||
if (!racedRunId) throw error;
|
||||
throw new PrimaryRunDuplicateRequestError(
|
||||
definition.projectId,
|
||||
key,
|
||||
racedRunId,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async prepareForSpawn(
|
||||
reference: PrimaryRunReference,
|
||||
timeoutMs: number | undefined,
|
||||
callbackTokenHash: string,
|
||||
logArtifactId?: string,
|
||||
): Promise<PrimaryRunReference> {
|
||||
const startingAtMs = this.atOrAfter(
|
||||
reference.run.createdAtMs,
|
||||
reference.attempt.createdAtMs,
|
||||
);
|
||||
const deadlineAtMs =
|
||||
timeoutMs === undefined ? undefined : startingAtMs + timeoutMs;
|
||||
if (deadlineAtMs !== undefined && !Number.isSafeInteger(deadlineAtMs)) {
|
||||
throw new RangeError('Primary Run deadline exceeds the supported range');
|
||||
}
|
||||
const dispatching = await this.commands.transitionRun({
|
||||
runId: reference.run.id,
|
||||
to: 'dispatching',
|
||||
expectedVersion: reference.run.version,
|
||||
atMs: startingAtMs,
|
||||
actor: { type: 'scheduler' },
|
||||
});
|
||||
const starting = await this.commands.transitionRunAttempt({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
to: 'starting',
|
||||
expectedRunVersion: dispatching.run.version,
|
||||
atMs: startingAtMs,
|
||||
...(deadlineAtMs === undefined ? {} : { deadlineAtMs }),
|
||||
callbackTokenHash,
|
||||
...(logArtifactId === undefined ? {} : { logArtifactId }),
|
||||
actor: { type: 'worker', id: this.executor.type },
|
||||
});
|
||||
return { run: starting.run, attempt: starting.attempt };
|
||||
}
|
||||
|
||||
private assertLogArtifactId(value: string | undefined): void {
|
||||
if (value === undefined) return;
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
value.length > MAX_LOG_ARTIFACT_ID_LENGTH ||
|
||||
!/^[A-Za-z0-9._:-]+$/.test(value)
|
||||
) {
|
||||
throw new TypeError('Primary Run logArtifactId is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
private async recordRunning(
|
||||
reference: PrimaryRunReference,
|
||||
handle: ExecutionHandle,
|
||||
): Promise<PrimaryRunReference> {
|
||||
const runningAttempt = await this.commands.transitionRunAttempt({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
to: 'running',
|
||||
expectedRunVersion: reference.run.version,
|
||||
atMs: this.atOrAfter(
|
||||
reference.run.createdAtMs,
|
||||
reference.attempt.createdAtMs,
|
||||
handle.startedAtMs,
|
||||
),
|
||||
executorHandle: handle.durableHandle ?? handle.id,
|
||||
...(handle.pid === undefined ? {} : { pid: handle.pid }),
|
||||
actor: { type: 'executor', id: this.executor.type },
|
||||
});
|
||||
const runningRun = await this.commands.transitionRun({
|
||||
runId: reference.run.id,
|
||||
to: 'running',
|
||||
expectedVersion: runningAttempt.run.version,
|
||||
atMs: this.atOrAfter(
|
||||
runningAttempt.run.createdAtMs,
|
||||
runningAttempt.attempt.startedAtMs,
|
||||
),
|
||||
actor: { type: 'executor', id: this.executor.type },
|
||||
});
|
||||
return { run: runningRun.run, attempt: runningAttempt.attempt };
|
||||
}
|
||||
|
||||
private async recordStartFailure(
|
||||
reference: PrimaryRunReference,
|
||||
): Promise<PrimaryRunReference> {
|
||||
const atMs = this.atOrAfter(
|
||||
reference.run.createdAtMs,
|
||||
reference.attempt.createdAtMs,
|
||||
);
|
||||
const failedAttempt = await this.commands.transitionRunAttempt({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
to: 'failed',
|
||||
expectedRunVersion: reference.run.version,
|
||||
atMs,
|
||||
errorCode: 'EXECUTOR_START_FAILED',
|
||||
errorSummary: 'Executor failed before ownership was established',
|
||||
actor: { type: 'executor', id: this.executor.type },
|
||||
});
|
||||
const failedRun = await this.commands.transitionRun({
|
||||
runId: reference.run.id,
|
||||
to: 'failed',
|
||||
expectedVersion: failedAttempt.run.version,
|
||||
atMs,
|
||||
errorCode: 'EXECUTOR_START_FAILED',
|
||||
errorSummary: 'Executor failed before ownership was established',
|
||||
actor: { type: 'executor', id: this.executor.type },
|
||||
});
|
||||
return { run: failedRun.run, attempt: failedAttempt.attempt };
|
||||
}
|
||||
|
||||
private async compensateActivationFailure(
|
||||
reference: PrimaryRunReference,
|
||||
handle: ExecutionHandle,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await this.executor.stop(handle, {
|
||||
kind: 'reconcile',
|
||||
requestedAtMs: this.atOrAfter(reference.run.createdAtMs),
|
||||
});
|
||||
} catch {
|
||||
// The durable state below remains lost when process ownership is unknown.
|
||||
}
|
||||
|
||||
try {
|
||||
let latest = await this.loadReference(reference);
|
||||
const atMs = this.atOrAfter(
|
||||
latest.run.createdAtMs,
|
||||
latest.run.startedAtMs,
|
||||
latest.attempt.createdAtMs,
|
||||
latest.attempt.startedAtMs,
|
||||
);
|
||||
if (!isTerminalRunAttemptStatus(latest.attempt.status)) {
|
||||
const attempt = await this.commands.transitionRunAttempt({
|
||||
runId: latest.run.id,
|
||||
attemptId: latest.attempt.id,
|
||||
to: 'lost',
|
||||
expectedRunVersion: latest.run.version,
|
||||
atMs,
|
||||
errorCode: 'EXECUTION_ACTIVATION_PERSISTENCE_FAILED',
|
||||
errorSummary: 'Executor ownership could not be persisted',
|
||||
actor: { type: 'reconciler' },
|
||||
});
|
||||
latest = { run: attempt.run, attempt: attempt.attempt };
|
||||
}
|
||||
if (!isTerminalRunStatus(latest.run.status)) {
|
||||
await this.commands.transitionRun({
|
||||
runId: latest.run.id,
|
||||
to: 'lost',
|
||||
expectedVersion: latest.run.version,
|
||||
atMs,
|
||||
errorCode: 'EXECUTION_ACTIVATION_PERSISTENCE_FAILED',
|
||||
errorSummary: 'Executor ownership could not be persisted',
|
||||
actor: { type: 'reconciler' },
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// Original persistence failure is reported to the caller.
|
||||
}
|
||||
}
|
||||
|
||||
private async loadReference(
|
||||
fallback: PrimaryRunReference,
|
||||
): Promise<PrimaryRunReference> {
|
||||
const [run, attempt] = await Promise.all([
|
||||
this.repository.findRunById(fallback.run.id),
|
||||
this.repository.findAttemptById(fallback.attempt.id),
|
||||
]);
|
||||
return {
|
||||
run: run ?? fallback.run,
|
||||
attempt: attempt ?? fallback.attempt,
|
||||
};
|
||||
}
|
||||
|
||||
private assertSpecMatches(
|
||||
reference: PrimaryRunReference,
|
||||
spec: ExecutionSpec,
|
||||
timeoutMs?: number,
|
||||
): void {
|
||||
if (
|
||||
spec.runId !== reference.run.id ||
|
||||
spec.attemptId !== reference.attempt.id ||
|
||||
spec.projectId !== reference.run.projectId ||
|
||||
spec.taskId !== reference.run.taskId ||
|
||||
spec.taskRevision !== reference.run.taskRevision ||
|
||||
spec.timeoutMs !== timeoutMs
|
||||
) {
|
||||
throw new Error('ExecutionSpec does not match its persisted Primary Run');
|
||||
}
|
||||
}
|
||||
|
||||
private assertTimeout(timeoutMs: number | undefined): void {
|
||||
if (
|
||||
timeoutMs !== undefined &&
|
||||
(!Number.isSafeInteger(timeoutMs) || timeoutMs < 1)
|
||||
) {
|
||||
throw new RangeError('Primary Run timeoutMs must be a positive integer');
|
||||
}
|
||||
}
|
||||
|
||||
private assertHandleMatches(
|
||||
reference: PrimaryRunReference,
|
||||
handle: ExecutionHandle,
|
||||
): void {
|
||||
if (
|
||||
handle.runId !== reference.run.id ||
|
||||
handle.attemptId !== reference.attempt.id ||
|
||||
handle.executorType !== this.executor.type
|
||||
) {
|
||||
throw new Error(
|
||||
'Executor handle does not match its persisted Primary Run',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private atOrAfter(...timestamps: Array<number | undefined>): number {
|
||||
return Math.max(
|
||||
this.clock.now(),
|
||||
...timestamps.filter((value): value is number => value !== undefined),
|
||||
);
|
||||
}
|
||||
|
||||
private cancellationActor(reason: ExecutionStopReason): {
|
||||
type: 'user' | 'reconciler' | 'system';
|
||||
} {
|
||||
if (reason.kind === 'user') return { type: 'user' };
|
||||
if (reason.kind === 'reconcile') return { type: 'reconciler' };
|
||||
return { type: 'system' };
|
||||
}
|
||||
|
||||
private deleteActive(runId: string, handle: ExecutionHandle): void {
|
||||
if (this.active.get(runId)?.handle === handle) this.active.delete(runId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,421 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
} from '../domain/run';
|
||||
import {
|
||||
isTerminalRunAttemptStatus,
|
||||
reserveRunEvent,
|
||||
} from '../domain/runStateMachine';
|
||||
import { RunVersionConflictError } from '../domain/stateMachineErrors';
|
||||
import type { PersistedExecutionInspector } from '../ports/persistedExecutionInspector';
|
||||
import type { CompletionReceiptJournal } from '../ports/completionReceiptJournal';
|
||||
import type {
|
||||
PrimaryRunRecoveryCandidate,
|
||||
PrimaryRunRecoveryCursor,
|
||||
PrimaryRunRecoverySource,
|
||||
} from '../ports/primaryRunRecoverySource';
|
||||
import type { RunRepository } from '../ports/runRepository';
|
||||
import type { PrimaryCompletionReceiptConsumer } from './primaryCompletionReceiptConsumer';
|
||||
import { RunCommandService } from './runCommandService';
|
||||
|
||||
export interface PrimaryRunStartupReconcilerClock {
|
||||
now(): number;
|
||||
}
|
||||
|
||||
export interface PrimaryRunStartupReconcilerOptions {
|
||||
clock?: PrimaryRunStartupReconcilerClock;
|
||||
createEventId?: () => string;
|
||||
completionReceipts?: Pick<PrimaryCompletionReceiptConsumer, 'consume'>;
|
||||
completionReceiptJournal?: Pick<CompletionReceiptJournal, 'register'>;
|
||||
receiptPublishGraceMs?: number;
|
||||
wait?: (delayMs: number) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface PrimaryRunStartupReconcileSummary {
|
||||
scanned: number;
|
||||
verifiedRunning: number;
|
||||
recoveredRunning: number;
|
||||
completedFromReceipt: number;
|
||||
quarantinedReceipts: number;
|
||||
publishGraceWaits: number;
|
||||
markedLost: number;
|
||||
skipped: number;
|
||||
ambiguous: number;
|
||||
failed: number;
|
||||
truncated: boolean;
|
||||
unsafeAttemptOverflow: boolean;
|
||||
nextCursor?: PrimaryRunRecoveryCursor;
|
||||
}
|
||||
|
||||
type RecoveryLossReason =
|
||||
| 'attempt_missing'
|
||||
| 'attempt_incomplete'
|
||||
| 'handle_missing'
|
||||
| 'handle_invalid'
|
||||
| 'identity_mismatch'
|
||||
| 'identity_pid_mismatch'
|
||||
| 'identity_unsupported'
|
||||
| 'process_exited_unobserved';
|
||||
|
||||
const LOSS_METADATA: Readonly<
|
||||
Record<RecoveryLossReason, { errorCode: string; errorSummary: string }>
|
||||
> = {
|
||||
attempt_missing: {
|
||||
errorCode: 'RECOVERY_ATTEMPT_MISSING',
|
||||
errorSummary: 'Active Run has no recoverable Attempt',
|
||||
},
|
||||
attempt_incomplete: {
|
||||
errorCode: 'RECOVERY_ATTEMPT_INCOMPLETE',
|
||||
errorSummary: 'Attempt did not persist executable ownership',
|
||||
},
|
||||
handle_missing: {
|
||||
errorCode: 'RECOVERY_HANDLE_MISSING',
|
||||
errorSummary: 'Attempt has no durable Executor handle',
|
||||
},
|
||||
handle_invalid: {
|
||||
errorCode: 'RECOVERY_HANDLE_INVALID',
|
||||
errorSummary: 'Attempt durable Executor handle is invalid',
|
||||
},
|
||||
identity_mismatch: {
|
||||
errorCode: 'RECOVERY_IDENTITY_MISMATCH',
|
||||
errorSummary: 'Operating system process identity does not match',
|
||||
},
|
||||
identity_pid_mismatch: {
|
||||
errorCode: 'RECOVERY_IDENTITY_PID_MISMATCH',
|
||||
errorSummary: 'Persisted PID does not match the durable handle',
|
||||
},
|
||||
identity_unsupported: {
|
||||
errorCode: 'RECOVERY_IDENTITY_UNSUPPORTED',
|
||||
errorSummary: 'Process identity cannot be verified on this platform',
|
||||
},
|
||||
process_exited_unobserved: {
|
||||
errorCode: 'RECOVERY_PROCESS_EXITED_UNOBSERVED',
|
||||
errorSummary: 'Process exited without a durable completion result',
|
||||
},
|
||||
};
|
||||
|
||||
/** One bounded startup pass. The caller owns scheduling and pagination. */
|
||||
export class PrimaryRunStartupReconciler {
|
||||
private readonly clock: PrimaryRunStartupReconcilerClock;
|
||||
private readonly createEventId: () => string;
|
||||
private readonly commands: RunCommandService;
|
||||
private readonly completionReceipts?: Pick<
|
||||
PrimaryCompletionReceiptConsumer,
|
||||
'consume'
|
||||
>;
|
||||
private readonly receiptPublishGraceMs: number;
|
||||
private readonly wait: (delayMs: number) => Promise<void>;
|
||||
private readonly completionReceiptJournal?: Pick<
|
||||
CompletionReceiptJournal,
|
||||
'register'
|
||||
>;
|
||||
private readonly inspectors = new Map<
|
||||
PersistedExecutionInspector['executorType'],
|
||||
PersistedExecutionInspector
|
||||
>();
|
||||
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly source: PrimaryRunRecoverySource,
|
||||
inspectors: readonly PersistedExecutionInspector[],
|
||||
options: PrimaryRunStartupReconcilerOptions = {},
|
||||
) {
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.createEventId = options.createEventId ?? uuidV7;
|
||||
this.commands = new RunCommandService(repository, this.createEventId);
|
||||
this.completionReceipts = options.completionReceipts;
|
||||
this.completionReceiptJournal = options.completionReceiptJournal;
|
||||
this.receiptPublishGraceMs = options.receiptPublishGraceMs ?? 0;
|
||||
if (
|
||||
!Number.isSafeInteger(this.receiptPublishGraceMs) ||
|
||||
this.receiptPublishGraceMs < 0 ||
|
||||
this.receiptPublishGraceMs > 5_000
|
||||
) {
|
||||
throw new RangeError('receiptPublishGraceMs must be between 0 and 5000');
|
||||
}
|
||||
this.wait =
|
||||
options.wait ??
|
||||
((delayMs) =>
|
||||
new Promise((resolve) => {
|
||||
setTimeout(resolve, delayMs);
|
||||
}));
|
||||
for (const inspector of inspectors) {
|
||||
if (this.inspectors.has(inspector.executorType)) {
|
||||
throw new Error(
|
||||
`Duplicate persisted Executor inspector: ${inspector.executorType}`,
|
||||
);
|
||||
}
|
||||
this.inspectors.set(inspector.executorType, inspector);
|
||||
}
|
||||
}
|
||||
|
||||
async reconcileBatch(
|
||||
options: {
|
||||
cursor?: PrimaryRunRecoveryCursor;
|
||||
limit?: number;
|
||||
} = {},
|
||||
): Promise<PrimaryRunStartupReconcileSummary> {
|
||||
const page = await this.source.listCandidates(options);
|
||||
const summary: PrimaryRunStartupReconcileSummary = {
|
||||
scanned: page.candidates.length,
|
||||
verifiedRunning: 0,
|
||||
recoveredRunning: 0,
|
||||
completedFromReceipt: 0,
|
||||
quarantinedReceipts: 0,
|
||||
publishGraceWaits: 0,
|
||||
markedLost: 0,
|
||||
skipped: 0,
|
||||
ambiguous: 0,
|
||||
failed: 0,
|
||||
truncated: page.truncated,
|
||||
unsafeAttemptOverflow: page.unsafeAttemptOverflow,
|
||||
...(page.nextCursor === undefined ? {} : { nextCursor: page.nextCursor }),
|
||||
};
|
||||
if (page.unsafeAttemptOverflow) return summary;
|
||||
|
||||
for (const candidate of page.candidates) {
|
||||
try {
|
||||
await this.reconcileCandidate(candidate, summary);
|
||||
} catch {
|
||||
summary.failed += 1;
|
||||
}
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
private async reconcileCandidate(
|
||||
candidate: PrimaryRunRecoveryCandidate,
|
||||
summary: PrimaryRunStartupReconcileSummary,
|
||||
): Promise<void> {
|
||||
const run = await this.repository.findRunById(candidate.runId);
|
||||
if (
|
||||
!run ||
|
||||
run.executionOwner !== 'runtime' ||
|
||||
!['dispatching', 'running'].includes(run.status)
|
||||
) {
|
||||
summary.skipped += 1;
|
||||
return;
|
||||
}
|
||||
if (candidate.attempts.length > 1) {
|
||||
summary.ambiguous += 1;
|
||||
return;
|
||||
}
|
||||
if (candidate.attempts.length === 0) {
|
||||
await this.markLost(run, undefined, 'attempt_missing');
|
||||
summary.markedLost += 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const attemptReference = candidate.attempts[0];
|
||||
const attempt = await this.repository.findAttemptById(
|
||||
attemptReference.attemptId,
|
||||
);
|
||||
if (
|
||||
!attempt ||
|
||||
attempt.runId !== run.id ||
|
||||
!['claimed', 'starting', 'running'].includes(attempt.status)
|
||||
) {
|
||||
summary.skipped += 1;
|
||||
return;
|
||||
}
|
||||
if (
|
||||
attempt.executorType === 'local_process' &&
|
||||
this.completionReceiptJournal
|
||||
) {
|
||||
await this.completionReceiptJournal.register({
|
||||
runId: run.id,
|
||||
attemptId: attempt.id,
|
||||
registeredAtMs: attempt.createdAtMs,
|
||||
});
|
||||
}
|
||||
if (await this.consumeCompletionReceipt(attempt, summary)) return;
|
||||
const inspector = this.inspectors.get(attemptReference.executorType);
|
||||
if (!inspector || attempt.executorType !== inspector.executorType) {
|
||||
summary.skipped += 1;
|
||||
return;
|
||||
}
|
||||
if (attempt.status !== 'running') {
|
||||
await this.markLost(run, attempt, 'attempt_incomplete');
|
||||
summary.markedLost += 1;
|
||||
return;
|
||||
}
|
||||
if (!attempt.executorHandle) {
|
||||
await this.markLost(run, attempt, 'handle_missing');
|
||||
summary.markedLost += 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const inspection = await inspector.inspect(attempt.executorHandle);
|
||||
if (
|
||||
inspection.status !== 'running' ||
|
||||
(inspection.identityPid !== undefined &&
|
||||
inspection.identityPid !== attempt.pid)
|
||||
) {
|
||||
if (await this.consumeCompletionReceipt(attempt, summary)) return;
|
||||
if (
|
||||
inspection.status === 'exited' &&
|
||||
(inspection.identityPid === undefined ||
|
||||
inspection.identityPid === attempt.pid) &&
|
||||
this.receiptPublishGraceMs > 0
|
||||
) {
|
||||
summary.publishGraceWaits += 1;
|
||||
await this.wait(this.receiptPublishGraceMs);
|
||||
if (await this.consumeCompletionReceipt(attempt, summary)) return;
|
||||
}
|
||||
}
|
||||
if (
|
||||
inspection.identityPid !== undefined &&
|
||||
inspection.identityPid !== attempt.pid
|
||||
) {
|
||||
await this.markLost(run, attempt, 'identity_pid_mismatch');
|
||||
summary.markedLost += 1;
|
||||
return;
|
||||
}
|
||||
if (inspection.status === 'running') {
|
||||
if (run.status === 'dispatching') {
|
||||
await this.commands.transitionRun({
|
||||
runId: run.id,
|
||||
to: 'running',
|
||||
expectedVersion: run.version,
|
||||
atMs: this.atOrAfter(
|
||||
run.createdAtMs,
|
||||
attempt.createdAtMs,
|
||||
attempt.startedAtMs,
|
||||
),
|
||||
actor: { type: 'reconciler' },
|
||||
});
|
||||
summary.recoveredRunning += 1;
|
||||
} else {
|
||||
await this.appendVerifiedRunningEvent(run, attempt);
|
||||
summary.verifiedRunning += 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const reason: RecoveryLossReason =
|
||||
inspection.status === 'invalid'
|
||||
? 'handle_invalid'
|
||||
: inspection.status === 'identity_mismatch'
|
||||
? 'identity_mismatch'
|
||||
: inspection.status === 'unsupported'
|
||||
? 'identity_unsupported'
|
||||
: 'process_exited_unobserved';
|
||||
await this.markLost(run, attempt, reason);
|
||||
summary.markedLost += 1;
|
||||
}
|
||||
|
||||
private async consumeCompletionReceipt(
|
||||
attempt: RunAttemptRecord,
|
||||
summary: PrimaryRunStartupReconcileSummary,
|
||||
): Promise<boolean> {
|
||||
if (!this.completionReceipts || attempt.executorType !== 'local_process') {
|
||||
return false;
|
||||
}
|
||||
const result = await this.completionReceipts.consume(attempt.id);
|
||||
if (result.status === 'missing') return false;
|
||||
if (result.status === 'quarantined') {
|
||||
summary.quarantinedReceipts += 1;
|
||||
return false;
|
||||
}
|
||||
summary.completedFromReceipt += 1;
|
||||
return true;
|
||||
}
|
||||
|
||||
private async markLost(
|
||||
initialRun: RunRecord,
|
||||
initialAttempt: RunAttemptRecord | undefined,
|
||||
reason: RecoveryLossReason,
|
||||
): Promise<void> {
|
||||
const metadata = LOSS_METADATA[reason];
|
||||
let run = initialRun;
|
||||
const atMs = this.atOrAfter(
|
||||
run.createdAtMs,
|
||||
run.startedAtMs,
|
||||
initialAttempt?.createdAtMs,
|
||||
initialAttempt?.startedAtMs,
|
||||
);
|
||||
if (initialAttempt && !isTerminalRunAttemptStatus(initialAttempt.status)) {
|
||||
const attempt = await this.commands.transitionRunAttempt({
|
||||
runId: run.id,
|
||||
attemptId: initialAttempt.id,
|
||||
to: 'lost',
|
||||
expectedRunVersion: run.version,
|
||||
atMs,
|
||||
errorCode: metadata.errorCode,
|
||||
errorSummary: metadata.errorSummary,
|
||||
actor: { type: 'reconciler' },
|
||||
});
|
||||
run = attempt.run;
|
||||
}
|
||||
await this.commands.transitionRun({
|
||||
runId: run.id,
|
||||
to: 'lost',
|
||||
expectedVersion: run.version,
|
||||
atMs,
|
||||
errorCode: metadata.errorCode,
|
||||
errorSummary: metadata.errorSummary,
|
||||
actor: { type: 'reconciler' },
|
||||
});
|
||||
}
|
||||
|
||||
private async appendVerifiedRunningEvent(
|
||||
expectedRun: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
): Promise<void> {
|
||||
const atMs = this.atOrAfter(
|
||||
expectedRun.createdAtMs,
|
||||
expectedRun.startedAtMs,
|
||||
attempt.createdAtMs,
|
||||
attempt.startedAtMs,
|
||||
);
|
||||
await this.repository.transaction(async (transaction) => {
|
||||
const current = await transaction.findRunById(expectedRun.id);
|
||||
if (!current) throw new Error('Primary Run disappeared during recovery');
|
||||
if (current.version !== expectedRun.version) {
|
||||
throw new RunVersionConflictError(
|
||||
current.id,
|
||||
expectedRun.version,
|
||||
current.version,
|
||||
);
|
||||
}
|
||||
const reserved = reserveRunEvent(current, current.version);
|
||||
const updated = await transaction.compareAndSetRun(
|
||||
reserved.run,
|
||||
current.version,
|
||||
);
|
||||
if (!updated) {
|
||||
throw new RunVersionConflictError(
|
||||
current.id,
|
||||
current.version,
|
||||
current.version,
|
||||
);
|
||||
}
|
||||
const event: RunEventRecord = {
|
||||
id: this.createEventId(),
|
||||
runId: current.id,
|
||||
attemptId: attempt.id,
|
||||
sequence: reserved.sequence,
|
||||
type: 'run.reconciled',
|
||||
dedupeKey: `primary-running-reconciled:${attempt.id}:${current.version}`,
|
||||
actorType: 'reconciler',
|
||||
payload: {
|
||||
status: 'running',
|
||||
executor_type: attempt.executorType,
|
||||
evidence: 'durable_handle',
|
||||
version: reserved.run.version,
|
||||
},
|
||||
createdAtMs: atMs,
|
||||
};
|
||||
await transaction.appendEvent(event);
|
||||
});
|
||||
}
|
||||
|
||||
private atOrAfter(...timestamps: Array<number | undefined>): number {
|
||||
return Math.max(
|
||||
this.clock.now(),
|
||||
...timestamps.filter((value): value is number => value !== undefined),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
import { MAX_PRIMARY_RECOVERY_BATCH_SIZE } from '../ports/primaryRunRecoverySource';
|
||||
import type { PrimaryRunRecoveryCursor } from '../ports/primaryRunRecoverySource';
|
||||
import type {
|
||||
PrimaryRunStartupReconcileSummary,
|
||||
PrimaryRunStartupReconciler,
|
||||
} from './primaryRunStartupReconciler';
|
||||
|
||||
export const MAX_PRIMARY_RECOVERY_PAGES_PER_STARTUP = 64;
|
||||
|
||||
export type PrimaryRunStartupStopReason =
|
||||
| 'complete'
|
||||
| 'page_limit'
|
||||
| 'unsafe_attempt_overflow'
|
||||
| 'cursor_stalled';
|
||||
|
||||
export interface PrimaryRunStartupSummary
|
||||
extends Omit<
|
||||
PrimaryRunStartupReconcileSummary,
|
||||
'truncated' | 'unsafeAttemptOverflow' | 'nextCursor'
|
||||
> {
|
||||
pages: number;
|
||||
stopReason: PrimaryRunStartupStopReason;
|
||||
remaining: boolean;
|
||||
nextCursor?: PrimaryRunRecoveryCursor;
|
||||
}
|
||||
|
||||
export interface PrimaryRunStartupOptions {
|
||||
cursor?: PrimaryRunRecoveryCursor;
|
||||
pageSize?: number;
|
||||
maxPages?: number;
|
||||
}
|
||||
|
||||
function sameCursor(
|
||||
left: PrimaryRunRecoveryCursor | undefined,
|
||||
right: PrimaryRunRecoveryCursor,
|
||||
): boolean {
|
||||
return (
|
||||
left !== undefined &&
|
||||
left.createdAtMs === right.createdAtMs &&
|
||||
left.runId === right.runId
|
||||
);
|
||||
}
|
||||
|
||||
/** Runs a complete but bounded startup reconciliation before Primary activates. */
|
||||
export class PrimaryRunStartupSupervisor {
|
||||
constructor(
|
||||
private readonly reconciler: Pick<
|
||||
PrimaryRunStartupReconciler,
|
||||
'reconcileBatch'
|
||||
>,
|
||||
) {}
|
||||
|
||||
async run(
|
||||
options: PrimaryRunStartupOptions = {},
|
||||
): Promise<PrimaryRunStartupSummary> {
|
||||
const pageSize = options.pageSize ?? 32;
|
||||
const maxPages = options.maxPages ?? 4;
|
||||
if (
|
||||
!Number.isSafeInteger(pageSize) ||
|
||||
pageSize < 1 ||
|
||||
pageSize > MAX_PRIMARY_RECOVERY_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'pageSize must be between 1 and MAX_PRIMARY_RECOVERY_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(maxPages) ||
|
||||
maxPages < 1 ||
|
||||
maxPages > MAX_PRIMARY_RECOVERY_PAGES_PER_STARTUP
|
||||
) {
|
||||
throw new RangeError(
|
||||
'maxPages must be between 1 and MAX_PRIMARY_RECOVERY_PAGES_PER_STARTUP',
|
||||
);
|
||||
}
|
||||
|
||||
const total: PrimaryRunStartupSummary = {
|
||||
pages: 0,
|
||||
scanned: 0,
|
||||
verifiedRunning: 0,
|
||||
recoveredRunning: 0,
|
||||
completedFromReceipt: 0,
|
||||
quarantinedReceipts: 0,
|
||||
publishGraceWaits: 0,
|
||||
markedLost: 0,
|
||||
skipped: 0,
|
||||
ambiguous: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
let cursor = options.cursor;
|
||||
|
||||
for (let pageNumber = 0; pageNumber < maxPages; pageNumber += 1) {
|
||||
const page = await this.reconciler.reconcileBatch({
|
||||
...(cursor === undefined ? {} : { cursor }),
|
||||
limit: pageSize,
|
||||
});
|
||||
total.pages += 1;
|
||||
total.scanned += page.scanned;
|
||||
total.verifiedRunning += page.verifiedRunning;
|
||||
total.recoveredRunning += page.recoveredRunning;
|
||||
total.completedFromReceipt += page.completedFromReceipt;
|
||||
total.quarantinedReceipts += page.quarantinedReceipts;
|
||||
total.publishGraceWaits += page.publishGraceWaits;
|
||||
total.markedLost += page.markedLost;
|
||||
total.skipped += page.skipped;
|
||||
total.ambiguous += page.ambiguous;
|
||||
total.failed += page.failed;
|
||||
|
||||
if (page.unsafeAttemptOverflow) {
|
||||
total.stopReason = 'unsafe_attempt_overflow';
|
||||
total.remaining = true;
|
||||
return total;
|
||||
}
|
||||
if (!page.truncated) return total;
|
||||
if (!page.nextCursor || sameCursor(cursor, page.nextCursor)) {
|
||||
total.stopReason = 'cursor_stalled';
|
||||
total.remaining = true;
|
||||
return total;
|
||||
}
|
||||
cursor = page.nextCursor;
|
||||
if (pageNumber === maxPages - 1) {
|
||||
total.stopReason = 'page_limit';
|
||||
total.remaining = true;
|
||||
total.nextCursor = cursor;
|
||||
return total;
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
import type {
|
||||
PrimaryTimeoutSupervisor,
|
||||
PrimaryTimeoutSupervisorOptions,
|
||||
PrimaryTimeoutSupervisorSummary,
|
||||
} from './primaryTimeoutSupervisor';
|
||||
|
||||
export const MIN_TIMEOUT_CYCLE_INTERVAL_MS = 250;
|
||||
export const MAX_TIMEOUT_CYCLE_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_TIMEOUT_INITIAL_DELAY_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_TIMEOUT_STOP_TIMEOUT_MS = 60_000;
|
||||
|
||||
interface ScheduledTimer {
|
||||
unref?: () => void;
|
||||
}
|
||||
|
||||
export interface TimeoutLifecycleScheduler {
|
||||
setTimeout(callback: () => void, delayMs: number): ScheduledTimer;
|
||||
clearTimeout(timer: ScheduledTimer): void;
|
||||
}
|
||||
|
||||
export interface PrimaryTimeoutLifecycleOptions {
|
||||
intervalMs: number;
|
||||
initialDelayMs?: number;
|
||||
stopTimeoutMs?: number;
|
||||
cycle?: Omit<PrimaryTimeoutSupervisorOptions, 'nowMs'>;
|
||||
scheduler?: TimeoutLifecycleScheduler;
|
||||
onCycle?: (summary: PrimaryTimeoutSupervisorSummary) => void;
|
||||
onError?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export type PrimaryTimeoutStopResult = 'drained' | 'timed_out';
|
||||
|
||||
const defaultScheduler: TimeoutLifecycleScheduler = {
|
||||
setTimeout(callback, delayMs) {
|
||||
return setTimeout(callback, delayMs);
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
clearTimeout(timer as ReturnType<typeof setTimeout>);
|
||||
},
|
||||
};
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Explicit, non-overlapping timeout-intent lifecycle. It owns no process
|
||||
* signal path: each cycle only asks the supervisor to persist timeout intent.
|
||||
*/
|
||||
export class PrimaryTimeoutLifecycle {
|
||||
private readonly intervalMs: number;
|
||||
private readonly initialDelayMs: number;
|
||||
private readonly stopTimeoutMs: number;
|
||||
private readonly cycleOptions: Omit<PrimaryTimeoutSupervisorOptions, 'nowMs'>;
|
||||
private readonly scheduler: TimeoutLifecycleScheduler;
|
||||
private readonly onCycle?: (summary: PrimaryTimeoutSupervisorSummary) => void;
|
||||
private readonly onError?: (error: unknown) => void;
|
||||
private started = false;
|
||||
private timer?: ScheduledTimer;
|
||||
private inFlight?: Promise<void>;
|
||||
|
||||
constructor(
|
||||
private readonly supervisor: Pick<PrimaryTimeoutSupervisor, 'run'>,
|
||||
options: PrimaryTimeoutLifecycleOptions,
|
||||
) {
|
||||
this.intervalMs = options.intervalMs;
|
||||
this.initialDelayMs = options.initialDelayMs ?? 0;
|
||||
this.stopTimeoutMs = options.stopTimeoutMs ?? 5_000;
|
||||
this.cycleOptions = {
|
||||
...(options.cycle?.cursor === undefined
|
||||
? {}
|
||||
: { cursor: { ...options.cycle.cursor } }),
|
||||
...(options.cycle?.pageSize === undefined
|
||||
? {}
|
||||
: { pageSize: options.cycle.pageSize }),
|
||||
...(options.cycle?.maxPages === undefined
|
||||
? {}
|
||||
: { maxPages: options.cycle.maxPages }),
|
||||
};
|
||||
this.scheduler = options.scheduler ?? defaultScheduler;
|
||||
this.onCycle = options.onCycle;
|
||||
this.onError = options.onError;
|
||||
assertIntegerBetween(
|
||||
'intervalMs',
|
||||
this.intervalMs,
|
||||
MIN_TIMEOUT_CYCLE_INTERVAL_MS,
|
||||
MAX_TIMEOUT_CYCLE_INTERVAL_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'initialDelayMs',
|
||||
this.initialDelayMs,
|
||||
0,
|
||||
MAX_TIMEOUT_INITIAL_DELAY_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'stopTimeoutMs',
|
||||
this.stopTimeoutMs,
|
||||
1,
|
||||
MAX_TIMEOUT_STOP_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
start(): boolean {
|
||||
if (this.started || this.inFlight) return false;
|
||||
this.started = true;
|
||||
this.schedule(this.initialDelayMs);
|
||||
return true;
|
||||
}
|
||||
|
||||
async stop(): Promise<PrimaryTimeoutStopResult> {
|
||||
this.started = false;
|
||||
if (this.timer) {
|
||||
this.scheduler.clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
const inFlight = this.inFlight;
|
||||
if (!inFlight) return 'drained';
|
||||
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const result = await Promise.race<PrimaryTimeoutStopResult>([
|
||||
inFlight.then(() => 'drained' as const),
|
||||
new Promise<'timed_out'>((resolve) => {
|
||||
timeout = setTimeout(() => resolve('timed_out'), this.stopTimeoutMs);
|
||||
}),
|
||||
]);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
return result;
|
||||
}
|
||||
|
||||
private schedule(delayMs: number): void {
|
||||
if (!this.started || this.timer) return;
|
||||
const timer = this.scheduler.setTimeout(() => {
|
||||
if (this.timer === timer) this.timer = undefined;
|
||||
this.run();
|
||||
}, delayMs);
|
||||
this.timer = timer;
|
||||
timer.unref?.();
|
||||
}
|
||||
|
||||
private run(): void {
|
||||
if (!this.started || this.inFlight) return;
|
||||
const inFlight = this.supervisor
|
||||
.run(this.cycleOptions)
|
||||
.then((summary) => this.notifyCycle(summary))
|
||||
.catch((error) => this.notifyError(error))
|
||||
.then(() => undefined)
|
||||
.finally(() => {
|
||||
if (this.inFlight === inFlight) this.inFlight = undefined;
|
||||
if (this.started) this.schedule(this.intervalMs);
|
||||
});
|
||||
this.inFlight = inFlight;
|
||||
}
|
||||
|
||||
private notifyCycle(summary: PrimaryTimeoutSupervisorSummary): void {
|
||||
try {
|
||||
this.onCycle?.(summary);
|
||||
} catch (error) {
|
||||
this.notifyError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private notifyError(error: unknown): void {
|
||||
try {
|
||||
this.onError?.(error);
|
||||
} catch {
|
||||
// Diagnostics must never create another scheduler failure loop.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import type { RunCancellationReason } from '../domain/run';
|
||||
import type {
|
||||
PrimaryTimeoutCursor,
|
||||
PrimaryTimeoutSource,
|
||||
} from '../ports/primaryTimeoutSource';
|
||||
import type {
|
||||
RequestRunCancellationCommand,
|
||||
RequestRunCancellationResult,
|
||||
} from './runCommandService';
|
||||
|
||||
export interface PrimaryTimeoutClock {
|
||||
now(): number;
|
||||
}
|
||||
|
||||
export interface PrimaryTimeoutCommandPort {
|
||||
requestCancellation(
|
||||
command: RequestRunCancellationCommand,
|
||||
): Promise<RequestRunCancellationResult>;
|
||||
}
|
||||
|
||||
export interface PrimaryTimeoutRequestSummary {
|
||||
scanned: number;
|
||||
accepted: number;
|
||||
alreadyRequested: number;
|
||||
alreadyTerminal: number;
|
||||
failed: number;
|
||||
truncated: boolean;
|
||||
nextCursor?: PrimaryTimeoutCursor;
|
||||
}
|
||||
|
||||
/** One bounded timeout-intent pass. It never calls an Executor or sends signal. */
|
||||
export class PrimaryTimeoutRequester {
|
||||
private readonly clock: PrimaryTimeoutClock;
|
||||
|
||||
constructor(
|
||||
private readonly source: PrimaryTimeoutSource,
|
||||
private readonly commands: PrimaryTimeoutCommandPort,
|
||||
clock: PrimaryTimeoutClock = { now: Date.now },
|
||||
) {
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
async requestBatch(
|
||||
options: {
|
||||
nowMs?: number;
|
||||
cursor?: PrimaryTimeoutCursor;
|
||||
limit?: number;
|
||||
} = {},
|
||||
): Promise<PrimaryTimeoutRequestSummary> {
|
||||
const nowMs = options.nowMs ?? this.clock.now();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new RangeError('nowMs must be a non-negative safe integer');
|
||||
}
|
||||
const page = await this.source.listOverdue({
|
||||
nowMs,
|
||||
...(options.cursor === undefined ? {} : { cursor: options.cursor }),
|
||||
...(options.limit === undefined ? {} : { limit: options.limit }),
|
||||
});
|
||||
const summary: PrimaryTimeoutRequestSummary = {
|
||||
scanned: page.candidates.length,
|
||||
accepted: 0,
|
||||
alreadyRequested: 0,
|
||||
alreadyTerminal: 0,
|
||||
failed: 0,
|
||||
truncated: page.truncated,
|
||||
...(page.nextCursor === undefined ? {} : { nextCursor: page.nextCursor }),
|
||||
};
|
||||
|
||||
for (const candidate of page.candidates) {
|
||||
if (candidate.deadlineAtMs > nowMs) {
|
||||
summary.failed += 1;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const result = await this.commands.requestCancellation({
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
atMs: nowMs,
|
||||
reason: 'timeout' satisfies RunCancellationReason,
|
||||
actor: { type: 'system', id: 'runtime:timeout' },
|
||||
});
|
||||
if (result.status === 'accepted') summary.accepted += 1;
|
||||
else if (result.status === 'already_requested') {
|
||||
summary.alreadyRequested += 1;
|
||||
} else {
|
||||
summary.alreadyTerminal += 1;
|
||||
}
|
||||
} catch {
|
||||
summary.failed += 1;
|
||||
}
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
import {
|
||||
MAX_PRIMARY_TIMEOUT_BATCH_SIZE,
|
||||
type PrimaryTimeoutCursor,
|
||||
} from '../ports/primaryTimeoutSource';
|
||||
import type {
|
||||
PrimaryTimeoutRequester,
|
||||
PrimaryTimeoutRequestSummary,
|
||||
} from './primaryTimeoutRequester';
|
||||
|
||||
export const MAX_PRIMARY_TIMEOUT_SUPERVISOR_PAGES = 64;
|
||||
|
||||
export type PrimaryTimeoutStopReason =
|
||||
| 'complete'
|
||||
| 'page_limit'
|
||||
| 'cursor_stalled';
|
||||
|
||||
export interface PrimaryTimeoutSupervisorSummary {
|
||||
pages: number;
|
||||
scanned: number;
|
||||
accepted: number;
|
||||
alreadyRequested: number;
|
||||
alreadyTerminal: number;
|
||||
failed: number;
|
||||
stopReason: PrimaryTimeoutStopReason;
|
||||
remaining: boolean;
|
||||
nextCursor?: PrimaryTimeoutCursor;
|
||||
}
|
||||
|
||||
export interface PrimaryTimeoutSupervisorOptions {
|
||||
nowMs?: number;
|
||||
pageSize?: number;
|
||||
maxPages?: number;
|
||||
cursor?: PrimaryTimeoutCursor;
|
||||
}
|
||||
|
||||
export interface PrimaryTimeoutSupervisorClock {
|
||||
now(): number;
|
||||
}
|
||||
|
||||
function sameCursor(
|
||||
left: PrimaryTimeoutCursor | undefined,
|
||||
right: PrimaryTimeoutCursor | undefined,
|
||||
): boolean {
|
||||
return (
|
||||
left !== undefined &&
|
||||
right !== undefined &&
|
||||
left.deadlineAtMs === right.deadlineAtMs &&
|
||||
left.attemptId === right.attemptId
|
||||
);
|
||||
}
|
||||
|
||||
export class PrimaryTimeoutSupervisor {
|
||||
constructor(
|
||||
private readonly requester: Pick<PrimaryTimeoutRequester, 'requestBatch'>,
|
||||
private readonly clock: PrimaryTimeoutSupervisorClock = { now: Date.now },
|
||||
) {}
|
||||
|
||||
async run(
|
||||
options: PrimaryTimeoutSupervisorOptions = {},
|
||||
): Promise<PrimaryTimeoutSupervisorSummary> {
|
||||
const pageSize = options.pageSize ?? 32;
|
||||
const maxPages = options.maxPages ?? 4;
|
||||
const nowMs = options.nowMs ?? this.clock.now();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new RangeError('nowMs must be a non-negative safe integer');
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(pageSize) ||
|
||||
pageSize < 1 ||
|
||||
pageSize > MAX_PRIMARY_TIMEOUT_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'pageSize must be between 1 and MAX_PRIMARY_TIMEOUT_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(maxPages) ||
|
||||
maxPages < 1 ||
|
||||
maxPages > MAX_PRIMARY_TIMEOUT_SUPERVISOR_PAGES
|
||||
) {
|
||||
throw new RangeError(
|
||||
'maxPages must be between 1 and MAX_PRIMARY_TIMEOUT_SUPERVISOR_PAGES',
|
||||
);
|
||||
}
|
||||
|
||||
const aggregate: PrimaryTimeoutSupervisorSummary = {
|
||||
pages: 0,
|
||||
scanned: 0,
|
||||
accepted: 0,
|
||||
alreadyRequested: 0,
|
||||
alreadyTerminal: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
let cursor = options.cursor;
|
||||
for (let pageNumber = 0; pageNumber < maxPages; pageNumber += 1) {
|
||||
const page: PrimaryTimeoutRequestSummary =
|
||||
await this.requester.requestBatch({
|
||||
nowMs,
|
||||
...(cursor === undefined ? {} : { cursor }),
|
||||
limit: pageSize,
|
||||
});
|
||||
aggregate.pages += 1;
|
||||
aggregate.scanned += page.scanned;
|
||||
aggregate.accepted += page.accepted;
|
||||
aggregate.alreadyRequested += page.alreadyRequested;
|
||||
aggregate.alreadyTerminal += page.alreadyTerminal;
|
||||
aggregate.failed += page.failed;
|
||||
|
||||
if (!page.truncated) return aggregate;
|
||||
if (!page.nextCursor || sameCursor(cursor, page.nextCursor)) {
|
||||
aggregate.stopReason = 'cursor_stalled';
|
||||
aggregate.remaining = true;
|
||||
if (page.nextCursor) aggregate.nextCursor = page.nextCursor;
|
||||
return aggregate;
|
||||
}
|
||||
cursor = page.nextCursor;
|
||||
}
|
||||
aggregate.stopReason = 'page_limit';
|
||||
aggregate.remaining = true;
|
||||
if (cursor) aggregate.nextCursor = cursor;
|
||||
return aggregate;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
import { randomBytes } from 'crypto';
|
||||
import {
|
||||
assertAuthenticatedPrincipalActive,
|
||||
normalizeAuthenticatedPrincipal,
|
||||
type AuthenticatedPrincipal,
|
||||
} from '../domain/authenticatedPrincipal';
|
||||
import { assertProjectPolicyProjectId } from '../domain/projectPolicy';
|
||||
import {
|
||||
OWNER_BOOTSTRAP_CHALLENGE_ID_BYTES,
|
||||
OWNER_BOOTSTRAP_DEFAULT_TTL_MS,
|
||||
OWNER_BOOTSTRAP_SYSTEM_SUBJECT,
|
||||
OWNER_BOOTSTRAP_TOKEN_BYTES,
|
||||
ProjectOwnerBootstrapUnauthorizedError,
|
||||
assertProjectOwnerBootstrapChallengeId,
|
||||
assertProjectOwnerBootstrapToken,
|
||||
assertProjectOwnerBootstrapTtl,
|
||||
digestProjectOwnerBootstrapToken,
|
||||
} from '../domain/projectOwnerBootstrap';
|
||||
import type { ProjectOwnerBootstrapRepository } from '../ports/projectOwnerBootstrapRepository';
|
||||
|
||||
export interface IssueProjectOwnerBootstrapRequest {
|
||||
projectId: string;
|
||||
issuer: AuthenticatedPrincipal;
|
||||
nowMs: number;
|
||||
ttlMs?: number;
|
||||
}
|
||||
|
||||
export interface IssuedProjectOwnerBootstrapChallenge {
|
||||
projectId: string;
|
||||
challengeId: string;
|
||||
token: string;
|
||||
expiresAtMs: number;
|
||||
}
|
||||
|
||||
export interface ClaimProjectOwnerBootstrapRequest {
|
||||
projectId: string;
|
||||
challengeId: string;
|
||||
token: string;
|
||||
principal: AuthenticatedPrincipal;
|
||||
nowMs: number;
|
||||
}
|
||||
|
||||
export interface ProjectOwnerBootstrapRandomSource {
|
||||
bytes(size: number): Uint8Array;
|
||||
}
|
||||
|
||||
const CRYPTO_RANDOM_SOURCE: ProjectOwnerBootstrapRandomSource = {
|
||||
bytes: randomBytes,
|
||||
};
|
||||
|
||||
function encodeRandom(
|
||||
source: ProjectOwnerBootstrapRandomSource,
|
||||
size: number,
|
||||
): string {
|
||||
const bytes = source.bytes(size);
|
||||
if (!(bytes instanceof Uint8Array) || bytes.byteLength !== size) {
|
||||
throw new TypeError('Project owner bootstrap random source is invalid');
|
||||
}
|
||||
try {
|
||||
return Buffer.from(bytes).toString('base64url');
|
||||
} finally {
|
||||
bytes.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
function assertExactRequestKeys(
|
||||
value: object,
|
||||
expected: readonly string[],
|
||||
): void {
|
||||
const keys = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
if (
|
||||
keys.length !== canonical.length ||
|
||||
keys.some((key, index) => key !== canonical[index])
|
||||
) {
|
||||
throw new TypeError('Project owner bootstrap request shape is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
export class ProjectOwnerBootstrapService {
|
||||
constructor(
|
||||
private readonly repository: ProjectOwnerBootstrapRepository,
|
||||
private readonly randomSource: ProjectOwnerBootstrapRandomSource = CRYPTO_RANDOM_SOURCE,
|
||||
) {}
|
||||
|
||||
async issue(
|
||||
request: IssueProjectOwnerBootstrapRequest,
|
||||
): Promise<Readonly<IssuedProjectOwnerBootstrapChallenge>> {
|
||||
if (!request || typeof request !== 'object' || Array.isArray(request)) {
|
||||
throw new TypeError(
|
||||
'Project owner bootstrap issue request must be an object',
|
||||
);
|
||||
}
|
||||
assertExactRequestKeys(
|
||||
request,
|
||||
request.ttlMs === undefined
|
||||
? ['projectId', 'issuer', 'nowMs']
|
||||
: ['projectId', 'issuer', 'nowMs', 'ttlMs'],
|
||||
);
|
||||
assertProjectPolicyProjectId(request.projectId);
|
||||
const issuer = normalizeAuthenticatedPrincipal(request.issuer);
|
||||
assertAuthenticatedPrincipalActive(issuer, request.nowMs);
|
||||
if (
|
||||
issuer.subject.type !== OWNER_BOOTSTRAP_SYSTEM_SUBJECT.type ||
|
||||
issuer.subject.id !== OWNER_BOOTSTRAP_SYSTEM_SUBJECT.id ||
|
||||
issuer.assurance !== 'local_console'
|
||||
) {
|
||||
throw new ProjectOwnerBootstrapUnauthorizedError();
|
||||
}
|
||||
const ttlMs = request.ttlMs ?? OWNER_BOOTSTRAP_DEFAULT_TTL_MS;
|
||||
assertProjectOwnerBootstrapTtl(ttlMs);
|
||||
const expiresAtMs = request.nowMs + ttlMs;
|
||||
if (!Number.isSafeInteger(expiresAtMs)) {
|
||||
throw new TypeError('Project owner bootstrap expiry is invalid');
|
||||
}
|
||||
const challengeId = encodeRandom(
|
||||
this.randomSource,
|
||||
OWNER_BOOTSTRAP_CHALLENGE_ID_BYTES,
|
||||
);
|
||||
const token = encodeRandom(this.randomSource, OWNER_BOOTSTRAP_TOKEN_BYTES);
|
||||
assertProjectOwnerBootstrapChallengeId(challengeId);
|
||||
assertProjectOwnerBootstrapToken(token);
|
||||
const tokenDigest = digestProjectOwnerBootstrapToken(
|
||||
request.projectId,
|
||||
challengeId,
|
||||
token,
|
||||
);
|
||||
await this.repository.issue({
|
||||
projectId: request.projectId,
|
||||
challengeId,
|
||||
tokenDigest,
|
||||
issuedAtMs: request.nowMs,
|
||||
expiresAtMs,
|
||||
});
|
||||
return Object.freeze({
|
||||
projectId: request.projectId,
|
||||
challengeId,
|
||||
token,
|
||||
expiresAtMs,
|
||||
});
|
||||
}
|
||||
|
||||
async claim(request: ClaimProjectOwnerBootstrapRequest) {
|
||||
if (!request || typeof request !== 'object' || Array.isArray(request)) {
|
||||
throw new TypeError(
|
||||
'Project owner bootstrap claim request must be an object',
|
||||
);
|
||||
}
|
||||
assertExactRequestKeys(request, [
|
||||
'projectId',
|
||||
'challengeId',
|
||||
'token',
|
||||
'principal',
|
||||
'nowMs',
|
||||
]);
|
||||
assertProjectPolicyProjectId(request.projectId);
|
||||
assertProjectOwnerBootstrapChallengeId(request.challengeId);
|
||||
assertProjectOwnerBootstrapToken(request.token);
|
||||
const principal = normalizeAuthenticatedPrincipal(request.principal);
|
||||
assertAuthenticatedPrincipalActive(principal, request.nowMs);
|
||||
if (principal.subject.type !== 'user') {
|
||||
throw new ProjectOwnerBootstrapUnauthorizedError();
|
||||
}
|
||||
return this.repository.claim({
|
||||
projectId: request.projectId,
|
||||
challengeId: request.challengeId,
|
||||
tokenDigest: digestProjectOwnerBootstrapToken(
|
||||
request.projectId,
|
||||
request.challengeId,
|
||||
request.token,
|
||||
),
|
||||
subject: principal.subject,
|
||||
claimedAtMs: request.nowMs,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
import {
|
||||
assertProjectPolicyProjectId,
|
||||
normalizePolicySubject,
|
||||
normalizeProjectPermission,
|
||||
normalizeProjectPolicySnapshot,
|
||||
ProjectPolicyUnavailableError,
|
||||
type ProjectPermission,
|
||||
type ProjectPolicyDecision,
|
||||
type ProjectPolicyDecisionWithFence,
|
||||
type ProjectPolicyRequest,
|
||||
type ProjectRole,
|
||||
type StaticProjectPermission,
|
||||
} from '../domain/projectPolicy';
|
||||
import type { ProjectPolicyRepository } from '../ports/projectPolicyRepository';
|
||||
|
||||
const READ_ONLY_PERMISSIONS = new Set<ProjectPermission>([
|
||||
'project.read',
|
||||
'task.read',
|
||||
'run.read',
|
||||
'artifact.read',
|
||||
]);
|
||||
|
||||
const OPERATOR_PERMISSIONS = new Set<ProjectPermission>([
|
||||
...READ_ONLY_PERMISSIONS,
|
||||
'task.create',
|
||||
'task.update',
|
||||
'run.start',
|
||||
'run.stop',
|
||||
'run.retry',
|
||||
'secret.use',
|
||||
]);
|
||||
|
||||
const ADMIN_EXCLUDED_PERMISSIONS = new Set<StaticProjectPermission>([
|
||||
'project.manage',
|
||||
]);
|
||||
|
||||
const AGENT_APPROVAL_PERMISSIONS = new Set<ProjectPermission>([
|
||||
'project.manage',
|
||||
'task.create',
|
||||
'task.update',
|
||||
'task.delete',
|
||||
'run.start',
|
||||
'run.stop',
|
||||
'run.retry',
|
||||
'secret.use',
|
||||
'secret.manage',
|
||||
'worker.manage',
|
||||
'policy.manage',
|
||||
'approval.decide',
|
||||
]);
|
||||
|
||||
function decision(
|
||||
effect: ProjectPolicyDecision['effect'],
|
||||
reason: string,
|
||||
): Readonly<ProjectPolicyDecision> {
|
||||
return Object.freeze({ effect, reasons: Object.freeze([reason]) });
|
||||
}
|
||||
|
||||
function roleAllows(role: ProjectRole, permission: ProjectPermission): boolean {
|
||||
if (role === 'owner') return true;
|
||||
if (role === 'admin') {
|
||||
return (
|
||||
permission.startsWith('tool.call:') ||
|
||||
!ADMIN_EXCLUDED_PERMISSIONS.has(permission as StaticProjectPermission)
|
||||
);
|
||||
}
|
||||
if (role === 'operator') {
|
||||
return (
|
||||
permission.startsWith('tool.call:') ||
|
||||
OPERATOR_PERMISSIONS.has(permission)
|
||||
);
|
||||
}
|
||||
return READ_ONLY_PERMISSIONS.has(permission);
|
||||
}
|
||||
|
||||
function agentRequiresApproval(permission: ProjectPermission): boolean {
|
||||
return (
|
||||
permission.startsWith('tool.call:') ||
|
||||
AGENT_APPROVAL_PERMISSIONS.has(permission)
|
||||
);
|
||||
}
|
||||
|
||||
export class ProjectPolicyEngine {
|
||||
constructor(private readonly repository: ProjectPolicyRepository) {}
|
||||
|
||||
async decide(
|
||||
request: ProjectPolicyRequest,
|
||||
): Promise<Readonly<ProjectPolicyDecision>> {
|
||||
return (await this.decideWithFence(request)).decision;
|
||||
}
|
||||
|
||||
async decideWithFence(
|
||||
request: ProjectPolicyRequest,
|
||||
): Promise<Readonly<ProjectPolicyDecisionWithFence>> {
|
||||
if (!request || typeof request !== 'object' || Array.isArray(request)) {
|
||||
throw new TypeError('Project policy request must be an object');
|
||||
}
|
||||
const requestKeys = Object.keys(request).sort();
|
||||
if (
|
||||
requestKeys.length !== 3 ||
|
||||
requestKeys[0] !== 'permission' ||
|
||||
requestKeys[1] !== 'projectId' ||
|
||||
requestKeys[2] !== 'subject'
|
||||
) {
|
||||
throw new TypeError('Project policy request shape is invalid');
|
||||
}
|
||||
const subject = normalizePolicySubject(request.subject);
|
||||
assertProjectPolicyProjectId(request.projectId);
|
||||
const permission = normalizeProjectPermission(request.permission);
|
||||
let resolved;
|
||||
try {
|
||||
resolved = await this.repository.resolve(request.projectId, subject);
|
||||
} catch {
|
||||
throw new ProjectPolicyUnavailableError();
|
||||
}
|
||||
if (!resolved) {
|
||||
return Object.freeze({
|
||||
decision: decision('deny', 'project_not_found'),
|
||||
fence: null,
|
||||
});
|
||||
}
|
||||
let snapshot;
|
||||
try {
|
||||
snapshot = normalizeProjectPolicySnapshot(resolved);
|
||||
} catch {
|
||||
throw new ProjectPolicyUnavailableError();
|
||||
}
|
||||
if (snapshot.project.id !== request.projectId) {
|
||||
throw new ProjectPolicyUnavailableError();
|
||||
}
|
||||
const fence = Object.freeze({
|
||||
projectVersion: snapshot.project.version,
|
||||
bindingVersion: snapshot.binding?.version ?? null,
|
||||
});
|
||||
if (!snapshot.binding || snapshot.binding.state === 'revoked') {
|
||||
return Object.freeze({
|
||||
decision: decision('deny', 'subject_unbound'),
|
||||
fence,
|
||||
});
|
||||
}
|
||||
if (
|
||||
snapshot.binding.subject.type !== subject.type ||
|
||||
snapshot.binding.subject.id !== subject.id
|
||||
) {
|
||||
throw new ProjectPolicyUnavailableError();
|
||||
}
|
||||
if (
|
||||
snapshot.project.status === 'archived' &&
|
||||
!READ_ONLY_PERMISSIONS.has(permission)
|
||||
) {
|
||||
return Object.freeze({
|
||||
decision: decision('deny', 'project_archived'),
|
||||
fence,
|
||||
});
|
||||
}
|
||||
if (!roleAllows(snapshot.binding.role!, permission)) {
|
||||
return Object.freeze({
|
||||
decision: decision('deny', 'permission_missing'),
|
||||
fence,
|
||||
});
|
||||
}
|
||||
if (subject.type === 'agent' && agentRequiresApproval(permission)) {
|
||||
return Object.freeze({
|
||||
decision: decision(
|
||||
'require_approval',
|
||||
'agent_action_requires_approval',
|
||||
),
|
||||
fence,
|
||||
});
|
||||
}
|
||||
return Object.freeze({
|
||||
decision: decision('allow', 'role_grant'),
|
||||
fence,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import type { ExecutionContext } from '../domain/execution';
|
||||
import { normalizeExecutionContext } from '../domain/executionContext';
|
||||
import { assertLocalExecutionArtifactId } from '../domain/localExecutionArtifact';
|
||||
import {
|
||||
assertLocalExecutionContextRef,
|
||||
normalizeLocalExecutionContextRecipe,
|
||||
} from '../domain/localExecutionContextRecipe';
|
||||
import { assertRunDispatchCandidate } from '../domain/runDispatchCandidate';
|
||||
import type { LocalExecutionArtifactAllocator } from '../ports/localExecutionArtifactAllocator';
|
||||
import type { LocalExecutionContextMaterializer } from '../ports/localExecutionContextMaterializer';
|
||||
import type { LocalExecutionContextRecipeSource } from '../ports/localExecutionContextRecipeSource';
|
||||
import type { LocalSecretEnvironmentProvider } from '../ports/localSecretEnvironmentProvider';
|
||||
|
||||
const VALIDATION_OUTPUT = Object.freeze({ async write() {} });
|
||||
|
||||
/** Resolves public values, ephemeral Secrets and an Attempt-scoped Artifact. */
|
||||
export class RecipeLocalExecutionContextMaterializer
|
||||
implements LocalExecutionContextMaterializer
|
||||
{
|
||||
constructor(
|
||||
private readonly recipes: LocalExecutionContextRecipeSource,
|
||||
private readonly artifacts: LocalExecutionArtifactAllocator,
|
||||
private readonly secrets?: LocalSecretEnvironmentProvider,
|
||||
) {}
|
||||
|
||||
async prepare(
|
||||
request: Parameters<LocalExecutionContextMaterializer['prepare']>[0],
|
||||
): ReturnType<LocalExecutionContextMaterializer['prepare']> {
|
||||
if (!request || typeof request !== 'object' || Array.isArray(request)) {
|
||||
throw new TypeError('Local execution context request must be an object');
|
||||
}
|
||||
assertRunDispatchCandidate(request.candidate);
|
||||
assertLocalExecutionContextRef(request.contextRef);
|
||||
const recipe = await this.recipes.resolve(request.contextRef);
|
||||
if (!recipe) return null;
|
||||
const normalized = normalizeLocalExecutionContextRecipe(recipe);
|
||||
if (normalized.contextRef !== request.contextRef) {
|
||||
throw new TypeError('Local context recipe does not match contextRef');
|
||||
}
|
||||
|
||||
const secretRefs = [
|
||||
...new Set(
|
||||
normalized.environment.flatMap((binding) =>
|
||||
binding.kind === 'secret' ? [binding.secretRef] : [],
|
||||
),
|
||||
),
|
||||
];
|
||||
let secretValues: readonly string[] = [];
|
||||
if (secretRefs.length > 0) {
|
||||
if (!this.secrets) return null;
|
||||
const resolved = await this.secrets.resolve(
|
||||
Object.freeze({
|
||||
candidate: Object.freeze({ ...request.candidate }),
|
||||
secretRefs: Object.freeze([...secretRefs]),
|
||||
}),
|
||||
);
|
||||
if (!resolved) return null;
|
||||
if (resolved.length !== secretRefs.length) {
|
||||
throw new TypeError('Local Secret provider returned an invalid result');
|
||||
}
|
||||
secretValues = resolved;
|
||||
}
|
||||
const byRef = new Map(
|
||||
secretRefs.map((secretRef, index) => [secretRef, secretValues[index]]),
|
||||
);
|
||||
const environment: Record<string, string> = Object.create(null);
|
||||
for (const binding of normalized.environment) {
|
||||
environment[binding.name] =
|
||||
binding.kind === 'public'
|
||||
? binding.value
|
||||
: (byRef.get(binding.secretRef) as string);
|
||||
}
|
||||
const validatedEnvironment = normalizeExecutionContext({
|
||||
environment,
|
||||
output: VALIDATION_OUTPUT,
|
||||
}).environment;
|
||||
|
||||
const artifact = await this.artifacts.prepare(request.candidate);
|
||||
try {
|
||||
assertLocalExecutionArtifactId(artifact.logArtifactId);
|
||||
const context: ExecutionContext = normalizeExecutionContext({
|
||||
environment: validatedEnvironment,
|
||||
output: artifact.output,
|
||||
});
|
||||
return {
|
||||
context,
|
||||
logArtifactId: artifact.logArtifactId,
|
||||
dispose: () => artifact.dispose(),
|
||||
};
|
||||
} catch (error) {
|
||||
await Promise.resolve(artifact.dispose()).catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,566 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunAttemptStatus,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
RunStatus,
|
||||
} from '../domain/run';
|
||||
import {
|
||||
assertRunDispatchId,
|
||||
assertRunDispatchLeaseToken,
|
||||
assertRunDispatchLeaseVersion,
|
||||
assertRunDispatchWorkerFence,
|
||||
type RunDispatchLeaseRecord,
|
||||
} from '../domain/runDispatchLease';
|
||||
import {
|
||||
MAX_EXECUTOR_HANDLE_LENGTH,
|
||||
MAX_LOG_ARTIFACT_ID_LENGTH,
|
||||
transitionRun,
|
||||
transitionRunAttempt,
|
||||
type RunDomainEventDraft,
|
||||
} from '../domain/runStateMachine';
|
||||
import type { RunDispatchLeaseRepository } from '../ports/runDispatchLeaseRepository';
|
||||
import type { RunRepositoryTransaction } from '../ports/runRepository';
|
||||
import type { AuthenticatedWorkerPrincipal } from './workerControlService';
|
||||
import { WorkerPrincipalMismatchError } from './workerControlService';
|
||||
|
||||
export interface RemoteRunLeaseFence {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
workerId: string;
|
||||
workerSessionId: string;
|
||||
workerGeneration: number;
|
||||
leaseGeneration: number;
|
||||
leaseToken: string;
|
||||
expectedLeaseVersion: number;
|
||||
executorType: string;
|
||||
}
|
||||
|
||||
export interface AcknowledgeRemoteRunStartingCommand
|
||||
extends RemoteRunLeaseFence {}
|
||||
|
||||
export interface AcknowledgeRemoteRunRunningCommand
|
||||
extends RemoteRunLeaseFence {
|
||||
startedAtMs: number;
|
||||
executorHandle: string;
|
||||
logArtifactId?: string;
|
||||
}
|
||||
|
||||
export interface FailRemoteRunStartCommand extends RemoteRunLeaseFence {}
|
||||
|
||||
export interface RemoteRunActivationResult {
|
||||
status:
|
||||
| 'applied'
|
||||
| 'already_starting'
|
||||
| 'already_running'
|
||||
| 'already_terminal';
|
||||
run: RunRecord;
|
||||
attempt: RunAttemptRecord;
|
||||
lease: RunDispatchLeaseRecord;
|
||||
events: readonly RunEventRecord[];
|
||||
}
|
||||
|
||||
export class RemoteRunActivationNotFoundError extends Error {
|
||||
constructor() {
|
||||
super('Remote Run activation target was not found');
|
||||
this.name = 'RemoteRunActivationNotFoundError';
|
||||
}
|
||||
}
|
||||
|
||||
export class RemoteRunActivationUnauthorizedError extends Error {
|
||||
constructor() {
|
||||
super('Remote Run activation target is not owned by this execution path');
|
||||
this.name = 'RemoteRunActivationUnauthorizedError';
|
||||
}
|
||||
}
|
||||
|
||||
export class RemoteRunActivationStateError extends Error {
|
||||
constructor(message = 'Remote Run activation state is inconsistent') {
|
||||
super(message);
|
||||
this.name = 'RemoteRunActivationStateError';
|
||||
}
|
||||
}
|
||||
|
||||
class RemoteRunActivationConcurrentWriteError extends Error {}
|
||||
|
||||
interface StartFailureMapping {
|
||||
attemptStatus: Extract<
|
||||
RunAttemptStatus,
|
||||
'failed' | 'cancelled' | 'timed_out'
|
||||
>;
|
||||
runStatus: Extract<RunStatus, 'failed' | 'cancelled' | 'timed_out'>;
|
||||
errorCode:
|
||||
| 'EXECUTOR_START_FAILED'
|
||||
| 'EXECUTION_CANCELLED'
|
||||
| 'EXECUTION_TIMED_OUT';
|
||||
errorSummary: string;
|
||||
}
|
||||
|
||||
function assertExecutorType(value: string): void {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
value.length > 64 ||
|
||||
/[\u0000-\u001f\u007f]/.test(value)
|
||||
) {
|
||||
throw new TypeError('Remote Run executorType is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function startFailureMapping(run: RunRecord): StartFailureMapping {
|
||||
if (run.cancelRequestedAtMs !== undefined) {
|
||||
if (run.cancelReason === 'timeout') {
|
||||
return {
|
||||
attemptStatus: 'timed_out',
|
||||
runStatus: 'timed_out',
|
||||
errorCode: 'EXECUTION_TIMED_OUT',
|
||||
errorSummary: 'Execution timed out before the executor started',
|
||||
};
|
||||
}
|
||||
return {
|
||||
attemptStatus: 'cancelled',
|
||||
runStatus: 'cancelled',
|
||||
errorCode: 'EXECUTION_CANCELLED',
|
||||
errorSummary: 'Execution was cancelled before the executor started',
|
||||
};
|
||||
}
|
||||
return {
|
||||
attemptStatus: 'failed',
|
||||
runStatus: 'failed',
|
||||
errorCode: 'EXECUTOR_START_FAILED',
|
||||
errorSummary: 'Executor failed before execution ownership was established',
|
||||
};
|
||||
}
|
||||
|
||||
export class RemoteRunActivationService {
|
||||
private readonly clock: { now(): number };
|
||||
private readonly createEventId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly leases: RunDispatchLeaseRepository,
|
||||
options: { clock?: { now(): number }; createEventId?: () => string } = {},
|
||||
) {
|
||||
this.clock = options.clock ?? Date;
|
||||
this.createEventId = options.createEventId ?? uuidV7;
|
||||
}
|
||||
|
||||
async acknowledgeStarting(
|
||||
principal: AuthenticatedWorkerPrincipal,
|
||||
command: AcknowledgeRemoteRunStartingCommand,
|
||||
): Promise<RemoteRunActivationResult> {
|
||||
this.assertCommand(principal, command);
|
||||
const observedAtMs = this.now();
|
||||
const eventId = this.createEventId();
|
||||
assertRunDispatchId('eventId', eventId);
|
||||
const result = await this.leases.withLease(
|
||||
this.useLeaseCommand(command, observedAtMs),
|
||||
async (transaction, lease) => {
|
||||
const { run, attempt } = await this.loadTarget(transaction, command);
|
||||
if (attempt.status === 'starting') {
|
||||
if (
|
||||
run.status !== 'dispatching' ||
|
||||
attempt.workerId !== command.workerId
|
||||
) {
|
||||
throw new RemoteRunActivationStateError();
|
||||
}
|
||||
return {
|
||||
status: 'already_starting' as const,
|
||||
run,
|
||||
attempt,
|
||||
events: [] as RunEventRecord[],
|
||||
};
|
||||
}
|
||||
if (attempt.status === 'running') {
|
||||
if (
|
||||
run.status !== 'running' ||
|
||||
attempt.workerId !== command.workerId
|
||||
) {
|
||||
throw new RemoteRunActivationStateError();
|
||||
}
|
||||
return {
|
||||
status: 'already_running' as const,
|
||||
run,
|
||||
attempt,
|
||||
events: [] as RunEventRecord[],
|
||||
};
|
||||
}
|
||||
if (attempt.status !== 'claimed' || run.status !== 'dispatching') {
|
||||
throw new RemoteRunActivationStateError(
|
||||
'Remote Run must be claimed and dispatching before start acknowledgement',
|
||||
);
|
||||
}
|
||||
const atMs = Math.max(
|
||||
observedAtMs,
|
||||
run.createdAtMs,
|
||||
attempt.createdAtMs,
|
||||
);
|
||||
const decision = transitionRunAttempt(run, attempt, {
|
||||
to: 'starting',
|
||||
expectedRunVersion: run.version,
|
||||
atMs,
|
||||
});
|
||||
const activatedAttempt: RunAttemptRecord = {
|
||||
...decision.attempt,
|
||||
workerId: command.workerId,
|
||||
};
|
||||
await this.persistAttemptTransition(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
decision.run,
|
||||
activatedAttempt,
|
||||
);
|
||||
const event = this.event(
|
||||
eventId,
|
||||
decision.run,
|
||||
decision.event,
|
||||
attempt.id,
|
||||
lease,
|
||||
command.workerId,
|
||||
'starting',
|
||||
atMs,
|
||||
);
|
||||
await transaction.appendEvent(event);
|
||||
return {
|
||||
status: 'applied' as const,
|
||||
run: decision.run,
|
||||
attempt: activatedAttempt,
|
||||
events: [event],
|
||||
};
|
||||
},
|
||||
);
|
||||
return { ...result.value, lease: result.lease };
|
||||
}
|
||||
|
||||
async acknowledgeRunning(
|
||||
principal: AuthenticatedWorkerPrincipal,
|
||||
command: AcknowledgeRemoteRunRunningCommand,
|
||||
): Promise<RemoteRunActivationResult> {
|
||||
this.assertCommand(principal, command);
|
||||
assertRunDispatchLeaseVersion('startedAtMs', command.startedAtMs);
|
||||
if (
|
||||
typeof command.executorHandle !== 'string' ||
|
||||
command.executorHandle.length < 1 ||
|
||||
command.executorHandle.length > MAX_EXECUTOR_HANDLE_LENGTH
|
||||
) {
|
||||
throw new TypeError('Remote Run executorHandle is invalid');
|
||||
}
|
||||
if (
|
||||
command.logArtifactId !== undefined &&
|
||||
(typeof command.logArtifactId !== 'string' ||
|
||||
command.logArtifactId.length < 1 ||
|
||||
command.logArtifactId.length > MAX_LOG_ARTIFACT_ID_LENGTH)
|
||||
) {
|
||||
throw new TypeError('Remote Run logArtifactId is invalid');
|
||||
}
|
||||
const observedAtMs = this.now();
|
||||
if (command.startedAtMs > observedAtMs) {
|
||||
throw new TypeError('Remote Run cannot start in the future');
|
||||
}
|
||||
const attemptEventId = this.createEventId();
|
||||
const runEventId = this.createEventId();
|
||||
assertRunDispatchId('attemptEventId', attemptEventId);
|
||||
assertRunDispatchId('runEventId', runEventId);
|
||||
const result = await this.leases.withLease(
|
||||
this.useLeaseCommand(command, observedAtMs),
|
||||
async (transaction, lease) => {
|
||||
const { run, attempt } = await this.loadTarget(transaction, command);
|
||||
if (attempt.status === 'running') {
|
||||
if (
|
||||
run.status !== 'running' ||
|
||||
attempt.workerId !== command.workerId ||
|
||||
attempt.executorHandle !== command.executorHandle ||
|
||||
attempt.logArtifactId !== command.logArtifactId
|
||||
) {
|
||||
throw new RemoteRunActivationStateError(
|
||||
'Remote Run running acknowledgement metadata does not match',
|
||||
);
|
||||
}
|
||||
return {
|
||||
status: 'already_running' as const,
|
||||
run,
|
||||
attempt,
|
||||
events: [] as RunEventRecord[],
|
||||
};
|
||||
}
|
||||
if (attempt.status !== 'starting' || run.status !== 'dispatching') {
|
||||
throw new RemoteRunActivationStateError(
|
||||
'Remote Run must be starting before running acknowledgement',
|
||||
);
|
||||
}
|
||||
const atMs = Math.max(
|
||||
command.startedAtMs,
|
||||
run.createdAtMs,
|
||||
attempt.createdAtMs,
|
||||
);
|
||||
const attemptDecision = transitionRunAttempt(run, attempt, {
|
||||
to: 'running',
|
||||
expectedRunVersion: run.version,
|
||||
atMs,
|
||||
executorHandle: command.executorHandle,
|
||||
...(command.logArtifactId === undefined
|
||||
? {}
|
||||
: { logArtifactId: command.logArtifactId }),
|
||||
});
|
||||
const runDecision = transitionRun(attemptDecision.run, {
|
||||
to: 'running',
|
||||
expectedVersion: attemptDecision.run.version,
|
||||
atMs,
|
||||
});
|
||||
await this.persistAttemptTransition(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
attemptDecision.run,
|
||||
attemptDecision.attempt,
|
||||
);
|
||||
const attemptEvent = this.event(
|
||||
attemptEventId,
|
||||
attemptDecision.run,
|
||||
attemptDecision.event,
|
||||
attempt.id,
|
||||
lease,
|
||||
command.workerId,
|
||||
'running-attempt',
|
||||
atMs,
|
||||
);
|
||||
await transaction.appendEvent(attemptEvent);
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
runDecision.run,
|
||||
attemptDecision.run.version,
|
||||
))
|
||||
) {
|
||||
throw new RemoteRunActivationConcurrentWriteError();
|
||||
}
|
||||
const runEvent = this.event(
|
||||
runEventId,
|
||||
runDecision.run,
|
||||
runDecision.event,
|
||||
attempt.id,
|
||||
lease,
|
||||
command.workerId,
|
||||
'running-run',
|
||||
atMs,
|
||||
);
|
||||
await transaction.appendEvent(runEvent);
|
||||
return {
|
||||
status: 'applied' as const,
|
||||
run: runDecision.run,
|
||||
attempt: attemptDecision.attempt,
|
||||
events: [attemptEvent, runEvent],
|
||||
};
|
||||
},
|
||||
);
|
||||
return { ...result.value, lease: result.lease };
|
||||
}
|
||||
|
||||
async failStart(
|
||||
principal: AuthenticatedWorkerPrincipal,
|
||||
command: FailRemoteRunStartCommand,
|
||||
): Promise<RemoteRunActivationResult> {
|
||||
this.assertCommand(principal, command);
|
||||
const failedAtMs = this.now();
|
||||
const attemptEventId = this.createEventId();
|
||||
const runEventId = this.createEventId();
|
||||
assertRunDispatchId('attemptEventId', attemptEventId);
|
||||
assertRunDispatchId('runEventId', runEventId);
|
||||
const result = await this.leases.completeWithLease(
|
||||
{
|
||||
...this.useLeaseCommand(command, failedAtMs),
|
||||
completedAtMs: failedAtMs,
|
||||
},
|
||||
async (transaction, lease) => {
|
||||
const { run, attempt } = await this.loadTarget(transaction, command);
|
||||
const mapping = startFailureMapping(run);
|
||||
if (
|
||||
attempt.status === mapping.attemptStatus &&
|
||||
run.status === mapping.runStatus &&
|
||||
attempt.errorCode === mapping.errorCode &&
|
||||
run.errorCode === mapping.errorCode
|
||||
) {
|
||||
return {
|
||||
status: 'already_terminal' as const,
|
||||
run,
|
||||
attempt,
|
||||
events: [] as RunEventRecord[],
|
||||
};
|
||||
}
|
||||
if (attempt.status !== 'starting' || run.status !== 'dispatching') {
|
||||
throw new RemoteRunActivationStateError(
|
||||
'Only a starting Remote Run can report executor start failure',
|
||||
);
|
||||
}
|
||||
const atMs = Math.max(failedAtMs, run.createdAtMs, attempt.createdAtMs);
|
||||
const attemptDecision = transitionRunAttempt(run, attempt, {
|
||||
to: mapping.attemptStatus,
|
||||
expectedRunVersion: run.version,
|
||||
atMs,
|
||||
callbackSequence: attempt.callbackSequence + 1,
|
||||
errorCode: mapping.errorCode,
|
||||
errorSummary: mapping.errorSummary,
|
||||
});
|
||||
const runDecision = transitionRun(attemptDecision.run, {
|
||||
to: mapping.runStatus,
|
||||
expectedVersion: attemptDecision.run.version,
|
||||
atMs,
|
||||
errorCode: mapping.errorCode,
|
||||
errorSummary: mapping.errorSummary,
|
||||
});
|
||||
await this.persistAttemptTransition(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
attemptDecision.run,
|
||||
attemptDecision.attempt,
|
||||
);
|
||||
const attemptEvent = this.event(
|
||||
attemptEventId,
|
||||
attemptDecision.run,
|
||||
attemptDecision.event,
|
||||
attempt.id,
|
||||
lease,
|
||||
command.workerId,
|
||||
'start-failed-attempt',
|
||||
atMs,
|
||||
);
|
||||
await transaction.appendEvent(attemptEvent);
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
runDecision.run,
|
||||
attemptDecision.run.version,
|
||||
))
|
||||
) {
|
||||
throw new RemoteRunActivationConcurrentWriteError();
|
||||
}
|
||||
const runEvent = this.event(
|
||||
runEventId,
|
||||
runDecision.run,
|
||||
runDecision.event,
|
||||
attempt.id,
|
||||
lease,
|
||||
command.workerId,
|
||||
'start-failed-run',
|
||||
atMs,
|
||||
);
|
||||
await transaction.appendEvent(runEvent);
|
||||
return {
|
||||
status: 'applied' as const,
|
||||
run: runDecision.run,
|
||||
attempt: attemptDecision.attempt,
|
||||
events: [attemptEvent, runEvent],
|
||||
};
|
||||
},
|
||||
);
|
||||
return { ...result.value, lease: result.lease };
|
||||
}
|
||||
|
||||
private useLeaseCommand(command: RemoteRunLeaseFence, observedAtMs: number) {
|
||||
return {
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
workerId: command.workerId,
|
||||
workerSessionId: command.workerSessionId,
|
||||
workerGeneration: command.workerGeneration,
|
||||
leaseGeneration: command.leaseGeneration,
|
||||
leaseToken: command.leaseToken,
|
||||
expectedVersion: command.expectedLeaseVersion,
|
||||
observedAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
private async loadTarget(
|
||||
transaction: RunRepositoryTransaction,
|
||||
command: RemoteRunLeaseFence,
|
||||
): Promise<{ run: RunRecord; attempt: RunAttemptRecord }> {
|
||||
const [run, attempt] = await Promise.all([
|
||||
transaction.findRunById(command.runId),
|
||||
transaction.findAttemptById(command.attemptId),
|
||||
]);
|
||||
if (!run || !attempt || attempt.runId !== run.id) {
|
||||
throw new RemoteRunActivationNotFoundError();
|
||||
}
|
||||
if (
|
||||
run.executionOwner !== 'runtime' ||
|
||||
attempt.executorType !== command.executorType
|
||||
) {
|
||||
throw new RemoteRunActivationUnauthorizedError();
|
||||
}
|
||||
return { run, attempt };
|
||||
}
|
||||
|
||||
private async persistAttemptTransition(
|
||||
transaction: RunRepositoryTransaction,
|
||||
previousRun: RunRecord,
|
||||
previousAttempt: RunAttemptRecord,
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
): Promise<void> {
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(run, previousRun.version)) ||
|
||||
!(await transaction.compareAndSetAttempt(attempt, {
|
||||
status: previousAttempt.status,
|
||||
callbackSequence: previousAttempt.callbackSequence,
|
||||
}))
|
||||
) {
|
||||
throw new RemoteRunActivationConcurrentWriteError();
|
||||
}
|
||||
}
|
||||
|
||||
private event(
|
||||
id: string,
|
||||
run: RunRecord,
|
||||
draft: RunDomainEventDraft,
|
||||
attemptId: string,
|
||||
lease: RunDispatchLeaseRecord,
|
||||
workerId: string,
|
||||
phase: string,
|
||||
createdAtMs: number,
|
||||
): RunEventRecord {
|
||||
return {
|
||||
id,
|
||||
runId: run.id,
|
||||
sequence: draft.sequence,
|
||||
type: draft.type,
|
||||
dedupeKey: `remote-activation:${attemptId}:${lease.leaseGeneration}:${phase}`,
|
||||
actorType: 'worker',
|
||||
actorId: workerId,
|
||||
attemptId,
|
||||
payload: {
|
||||
...draft.payload,
|
||||
lease_generation: lease.leaseGeneration,
|
||||
},
|
||||
createdAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
private assertCommand(
|
||||
principal: AuthenticatedWorkerPrincipal,
|
||||
command: RemoteRunLeaseFence,
|
||||
): void {
|
||||
if (principal.workerId !== command.workerId) {
|
||||
throw new WorkerPrincipalMismatchError();
|
||||
}
|
||||
assertRunDispatchId('runId', command.runId);
|
||||
assertRunDispatchId('attemptId', command.attemptId);
|
||||
assertRunDispatchWorkerFence(command);
|
||||
assertRunDispatchLeaseToken(command.leaseToken);
|
||||
assertRunDispatchLeaseVersion(
|
||||
'leaseGeneration',
|
||||
command.leaseGeneration,
|
||||
true,
|
||||
);
|
||||
assertRunDispatchLeaseVersion(
|
||||
'expectedLeaseVersion',
|
||||
command.expectedLeaseVersion,
|
||||
);
|
||||
assertExecutorType(command.executorType);
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock.now();
|
||||
assertRunDispatchLeaseVersion('observedAtMs', nowMs);
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import type { ExecutionResult } from '../domain/execution';
|
||||
import type { RunEventRecord, RunRecord } from '../domain/run';
|
||||
import {
|
||||
assertRunDispatchId,
|
||||
assertRunDispatchLeaseToken,
|
||||
assertRunDispatchLeaseVersion,
|
||||
assertRunDispatchWorkerFence,
|
||||
} from '../domain/runDispatchLease';
|
||||
import type { RunDispatchLeaseRepository } from '../ports/runDispatchLeaseRepository';
|
||||
import type {
|
||||
RunRepository,
|
||||
RunRepositoryTransaction,
|
||||
} from '../ports/runRepository';
|
||||
import {
|
||||
PrimaryRunCompletionService,
|
||||
type PrimaryRunCompletionResult,
|
||||
} from './primaryRunCompletionService';
|
||||
import type { AuthenticatedWorkerPrincipal } from './workerControlService';
|
||||
import { WorkerPrincipalMismatchError } from './workerControlService';
|
||||
|
||||
class WorkerAttributedRunTransaction implements RunRepositoryTransaction {
|
||||
constructor(
|
||||
private readonly transaction: RunRepositoryTransaction,
|
||||
private readonly workerId: string,
|
||||
) {}
|
||||
|
||||
findRunById(runId: string) {
|
||||
return this.transaction.findRunById(runId);
|
||||
}
|
||||
|
||||
findAttemptById(attemptId: string) {
|
||||
return this.transaction.findAttemptById(attemptId);
|
||||
}
|
||||
|
||||
findLatestAttemptByRunId(runId: string) {
|
||||
return this.transaction.findLatestAttemptByRunId(runId);
|
||||
}
|
||||
|
||||
findRetryPolicyByRunId(runId: string) {
|
||||
return this.transaction.findRetryPolicyByRunId(runId);
|
||||
}
|
||||
|
||||
listEvents(
|
||||
runId: string,
|
||||
options?: { afterSequence?: number; limit?: number },
|
||||
) {
|
||||
return this.transaction.listEvents(runId, options);
|
||||
}
|
||||
|
||||
listCancellationRequested(options?: { beforeMs?: number; limit?: number }) {
|
||||
return this.transaction.listCancellationRequested(options);
|
||||
}
|
||||
|
||||
insertRun(run: RunRecord) {
|
||||
return this.transaction.insertRun(run);
|
||||
}
|
||||
|
||||
insertAttempt(
|
||||
attempt: Parameters<RunRepositoryTransaction['insertAttempt']>[0],
|
||||
) {
|
||||
return this.transaction.insertAttempt(attempt);
|
||||
}
|
||||
|
||||
insertRetryPolicy(
|
||||
_policy: Parameters<RunRepositoryTransaction['insertRetryPolicy']>[0],
|
||||
): Promise<void> {
|
||||
throw new Error('Worker completion cannot create a Run retry policy');
|
||||
}
|
||||
|
||||
compareAndSetRun(run: RunRecord, expectedVersion: number): Promise<boolean> {
|
||||
return this.transaction.compareAndSetRun(run, expectedVersion);
|
||||
}
|
||||
|
||||
compareAndSetAttempt(
|
||||
attempt: Parameters<RunRepositoryTransaction['compareAndSetAttempt']>[0],
|
||||
expected: Parameters<RunRepositoryTransaction['compareAndSetAttempt']>[1],
|
||||
): Promise<boolean> {
|
||||
return this.transaction.compareAndSetAttempt(attempt, expected);
|
||||
}
|
||||
|
||||
compareAndSetRetryPolicy(
|
||||
_policy: Parameters<
|
||||
RunRepositoryTransaction['compareAndSetRetryPolicy']
|
||||
>[0],
|
||||
_expectedVersion: number,
|
||||
): Promise<boolean> {
|
||||
throw new Error('Worker completion cannot modify a Run retry policy');
|
||||
}
|
||||
|
||||
appendEvent(event: RunEventRecord): Promise<void> {
|
||||
return this.transaction.appendEvent({
|
||||
...event,
|
||||
actorType: 'worker',
|
||||
actorId: this.workerId,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export interface RemoteRunCompletionCommand {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
callbackSequence: number;
|
||||
result: ExecutionResult;
|
||||
executorType: string;
|
||||
workerId: string;
|
||||
workerSessionId: string;
|
||||
workerGeneration: number;
|
||||
leaseGeneration: number;
|
||||
leaseToken: string;
|
||||
expectedLeaseVersion: number;
|
||||
}
|
||||
|
||||
class LeaseFencedRunRepository implements RunRepository {
|
||||
constructor(
|
||||
private readonly reader: RunRepository,
|
||||
private readonly leases: RunDispatchLeaseRepository,
|
||||
private readonly command: RemoteRunCompletionCommand,
|
||||
private readonly completedAtMs: number,
|
||||
) {}
|
||||
|
||||
findRunById(runId: string) {
|
||||
return this.reader.findRunById(runId);
|
||||
}
|
||||
|
||||
findAttemptById(attemptId: string) {
|
||||
return this.reader.findAttemptById(attemptId);
|
||||
}
|
||||
|
||||
findLatestAttemptByRunId(runId: string) {
|
||||
return this.reader.findLatestAttemptByRunId(runId);
|
||||
}
|
||||
|
||||
findRetryPolicyByRunId(runId: string) {
|
||||
return this.reader.findRetryPolicyByRunId(runId);
|
||||
}
|
||||
|
||||
listEvents(
|
||||
runId: string,
|
||||
options?: { afterSequence?: number; limit?: number },
|
||||
): Promise<RunEventRecord[]> {
|
||||
return this.reader.listEvents(runId, options);
|
||||
}
|
||||
|
||||
listCancellationRequested(options?: {
|
||||
beforeMs?: number;
|
||||
limit?: number;
|
||||
}): Promise<RunRecord[]> {
|
||||
return this.reader.listCancellationRequested(options);
|
||||
}
|
||||
|
||||
async transaction<T>(
|
||||
work: (transaction: RunRepositoryTransaction) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const result = await this.leases.completeWithLease(
|
||||
{
|
||||
runId: this.command.runId,
|
||||
attemptId: this.command.attemptId,
|
||||
workerId: this.command.workerId,
|
||||
workerSessionId: this.command.workerSessionId,
|
||||
workerGeneration: this.command.workerGeneration,
|
||||
leaseGeneration: this.command.leaseGeneration,
|
||||
leaseToken: this.command.leaseToken,
|
||||
expectedVersion: this.command.expectedLeaseVersion,
|
||||
completedAtMs: this.completedAtMs,
|
||||
},
|
||||
(transaction) =>
|
||||
work(
|
||||
new WorkerAttributedRunTransaction(
|
||||
transaction,
|
||||
this.command.workerId,
|
||||
),
|
||||
),
|
||||
);
|
||||
return result.value;
|
||||
}
|
||||
}
|
||||
|
||||
export class RemoteRunCompletionService {
|
||||
private readonly clock: { now(): number };
|
||||
private readonly createEventId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly runs: RunRepository,
|
||||
private readonly leases: RunDispatchLeaseRepository,
|
||||
options: { clock?: { now(): number }; createEventId?: () => string } = {},
|
||||
) {
|
||||
this.clock = options.clock ?? Date;
|
||||
this.createEventId = options.createEventId ?? uuidV7;
|
||||
}
|
||||
|
||||
complete(
|
||||
principal: AuthenticatedWorkerPrincipal,
|
||||
command: RemoteRunCompletionCommand,
|
||||
): Promise<PrimaryRunCompletionResult> {
|
||||
this.assertCommand(principal, command);
|
||||
const completedAtMs = this.clock.now();
|
||||
assertRunDispatchLeaseVersion('completedAtMs', completedAtMs);
|
||||
if (completedAtMs < command.result.finishedAtMs) {
|
||||
throw new TypeError('Remote completion cannot be observed before finish');
|
||||
}
|
||||
const repository = new LeaseFencedRunRepository(
|
||||
this.runs,
|
||||
this.leases,
|
||||
command,
|
||||
completedAtMs,
|
||||
);
|
||||
return new PrimaryRunCompletionService(
|
||||
repository,
|
||||
this.createEventId,
|
||||
).complete({
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
callbackSequence: command.callbackSequence,
|
||||
result: command.result,
|
||||
source: {
|
||||
kind: 'executor',
|
||||
executorType: command.executorType,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private assertCommand(
|
||||
principal: AuthenticatedWorkerPrincipal,
|
||||
command: RemoteRunCompletionCommand,
|
||||
): void {
|
||||
if (principal.workerId !== command.workerId) {
|
||||
throw new WorkerPrincipalMismatchError();
|
||||
}
|
||||
assertRunDispatchId('runId', command.runId);
|
||||
assertRunDispatchId('attemptId', command.attemptId);
|
||||
assertRunDispatchWorkerFence(command);
|
||||
assertRunDispatchLeaseToken(command.leaseToken);
|
||||
assertRunDispatchLeaseVersion(
|
||||
'leaseGeneration',
|
||||
command.leaseGeneration,
|
||||
true,
|
||||
);
|
||||
assertRunDispatchLeaseVersion(
|
||||
'expectedLeaseVersion',
|
||||
command.expectedLeaseVersion,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunCancellationReason,
|
||||
RunEventActorType,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
} from '../domain/run';
|
||||
import {
|
||||
isTerminalRunAttemptStatus,
|
||||
requestRunCancellation,
|
||||
transitionRun,
|
||||
transitionRunAttempt,
|
||||
type RunAttemptTransitionCommand,
|
||||
type RunTransitionCommand,
|
||||
} from '../domain/runStateMachine';
|
||||
import { RunVersionConflictError } from '../domain/stateMachineErrors';
|
||||
import type { RunRepository } from '../ports/runRepository';
|
||||
import {
|
||||
RunAttemptConcurrentWriteError,
|
||||
RunAttemptNotFoundError,
|
||||
RunNotFoundError,
|
||||
} from './commandErrors';
|
||||
|
||||
export interface RunCommandActor {
|
||||
type: RunEventActorType;
|
||||
id?: string;
|
||||
}
|
||||
|
||||
export interface TransitionRunCommand extends RunTransitionCommand {
|
||||
runId: string;
|
||||
actor: RunCommandActor;
|
||||
dedupeKey?: string;
|
||||
}
|
||||
|
||||
export interface TransitionRunAttemptCommand
|
||||
extends RunAttemptTransitionCommand {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
actor: RunCommandActor;
|
||||
dedupeKey?: string;
|
||||
}
|
||||
|
||||
export interface RunCommandResult {
|
||||
run: RunRecord;
|
||||
event: RunEventRecord;
|
||||
}
|
||||
|
||||
export interface RunAttemptCommandResult extends RunCommandResult {
|
||||
attempt: RunAttemptRecord;
|
||||
}
|
||||
|
||||
export interface RequestRunCancellationCommand {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
atMs: number;
|
||||
reason: RunCancellationReason;
|
||||
actor: RunCommandActor;
|
||||
dedupeKey?: string;
|
||||
}
|
||||
|
||||
export type RequestRunCancellationResult =
|
||||
| {
|
||||
status: 'accepted';
|
||||
run: RunRecord;
|
||||
attempt: RunAttemptRecord;
|
||||
event: RunEventRecord;
|
||||
}
|
||||
| {
|
||||
status: 'already_requested' | 'already_terminal';
|
||||
run: RunRecord;
|
||||
attempt: RunAttemptRecord;
|
||||
};
|
||||
|
||||
export type RunEventIdFactory = () => string;
|
||||
|
||||
export class RunCommandService {
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly createEventId: RunEventIdFactory = uuidV7,
|
||||
) {}
|
||||
|
||||
async transitionRun(
|
||||
command: TransitionRunCommand,
|
||||
): Promise<RunCommandResult> {
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const current = await transaction.findRunById(command.runId);
|
||||
if (!current) throw new RunNotFoundError(command.runId);
|
||||
|
||||
const decision = transitionRun(current, command);
|
||||
const updated = await transaction.compareAndSetRun(
|
||||
decision.run,
|
||||
command.expectedVersion,
|
||||
);
|
||||
if (!updated) {
|
||||
const latest = await transaction.findRunById(command.runId);
|
||||
if (!latest) throw new RunNotFoundError(command.runId);
|
||||
throw new RunVersionConflictError(
|
||||
command.runId,
|
||||
command.expectedVersion,
|
||||
latest.version,
|
||||
);
|
||||
}
|
||||
|
||||
const event = this.createRunEvent({
|
||||
runId: command.runId,
|
||||
decision: decision.event,
|
||||
actor: command.actor,
|
||||
dedupeKey:
|
||||
command.dedupeKey ??
|
||||
`run-transition:${command.expectedVersion}:${command.to}`,
|
||||
createdAtMs: command.atMs,
|
||||
});
|
||||
await transaction.appendEvent(event);
|
||||
return { run: decision.run, event };
|
||||
});
|
||||
}
|
||||
|
||||
async transitionRunAttempt(
|
||||
command: TransitionRunAttemptCommand,
|
||||
): Promise<RunAttemptCommandResult> {
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const currentRun = await transaction.findRunById(command.runId);
|
||||
if (!currentRun) throw new RunNotFoundError(command.runId);
|
||||
|
||||
const currentAttempt = await transaction.findAttemptById(
|
||||
command.attemptId,
|
||||
);
|
||||
if (!currentAttempt) {
|
||||
throw new RunAttemptNotFoundError(command.attemptId);
|
||||
}
|
||||
|
||||
const decision = transitionRunAttempt(
|
||||
currentRun,
|
||||
currentAttempt,
|
||||
command,
|
||||
);
|
||||
const runUpdated = await transaction.compareAndSetRun(
|
||||
decision.run,
|
||||
command.expectedRunVersion,
|
||||
);
|
||||
if (!runUpdated) {
|
||||
const latest = await transaction.findRunById(command.runId);
|
||||
if (!latest) throw new RunNotFoundError(command.runId);
|
||||
throw new RunVersionConflictError(
|
||||
command.runId,
|
||||
command.expectedRunVersion,
|
||||
latest.version,
|
||||
);
|
||||
}
|
||||
|
||||
const attemptUpdated = await transaction.compareAndSetAttempt(
|
||||
decision.attempt,
|
||||
{
|
||||
status: currentAttempt.status,
|
||||
callbackSequence: currentAttempt.callbackSequence,
|
||||
},
|
||||
);
|
||||
if (!attemptUpdated) {
|
||||
throw new RunAttemptConcurrentWriteError(
|
||||
currentAttempt.id,
|
||||
currentAttempt.status,
|
||||
currentAttempt.callbackSequence,
|
||||
);
|
||||
}
|
||||
|
||||
const event = this.createRunEvent({
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
decision: decision.event,
|
||||
actor: command.actor,
|
||||
dedupeKey:
|
||||
command.dedupeKey ??
|
||||
`attempt-transition:${command.attemptId}:${command.expectedRunVersion}:${command.to}`,
|
||||
createdAtMs: command.atMs,
|
||||
});
|
||||
await transaction.appendEvent(event);
|
||||
return { run: decision.run, attempt: decision.attempt, event };
|
||||
});
|
||||
}
|
||||
|
||||
async requestCancellation(
|
||||
command: RequestRunCancellationCommand,
|
||||
): Promise<RequestRunCancellationResult> {
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const currentRun = await transaction.findRunById(command.runId);
|
||||
if (!currentRun) throw new RunNotFoundError(command.runId);
|
||||
|
||||
const currentAttempt = await transaction.findAttemptById(
|
||||
command.attemptId,
|
||||
);
|
||||
if (!currentAttempt) {
|
||||
throw new RunAttemptNotFoundError(command.attemptId);
|
||||
}
|
||||
if (currentAttempt.runId !== currentRun.id) {
|
||||
throw new RunAttemptNotFoundError(command.attemptId);
|
||||
}
|
||||
if (isTerminalRunAttemptStatus(currentAttempt.status)) {
|
||||
return {
|
||||
status: 'already_terminal',
|
||||
run: currentRun,
|
||||
attempt: currentAttempt,
|
||||
};
|
||||
}
|
||||
|
||||
const decision = requestRunCancellation(currentRun, {
|
||||
expectedVersion: currentRun.version,
|
||||
atMs: command.atMs,
|
||||
reason: command.reason,
|
||||
});
|
||||
if (decision.status !== 'accepted') {
|
||||
return {
|
||||
status: decision.status,
|
||||
run: decision.run,
|
||||
attempt: currentAttempt,
|
||||
};
|
||||
}
|
||||
|
||||
const updated = await transaction.compareAndSetRun(
|
||||
decision.run,
|
||||
currentRun.version,
|
||||
);
|
||||
if (!updated) {
|
||||
const latest = await transaction.findRunById(command.runId);
|
||||
if (!latest) throw new RunNotFoundError(command.runId);
|
||||
throw new RunVersionConflictError(
|
||||
command.runId,
|
||||
currentRun.version,
|
||||
latest.version,
|
||||
);
|
||||
}
|
||||
|
||||
const event = this.createRunEvent({
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
decision: decision.event,
|
||||
actor: command.actor,
|
||||
dedupeKey:
|
||||
command.dedupeKey ?? `run-cancel-request:${command.attemptId}`,
|
||||
createdAtMs: command.atMs,
|
||||
});
|
||||
await transaction.appendEvent(event);
|
||||
return {
|
||||
status: 'accepted',
|
||||
run: decision.run,
|
||||
attempt: currentAttempt,
|
||||
event,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
private createRunEvent(input: {
|
||||
runId: string;
|
||||
attemptId?: string;
|
||||
decision: {
|
||||
sequence: number;
|
||||
type: string;
|
||||
payload: Readonly<Record<string, unknown>>;
|
||||
};
|
||||
actor: RunCommandActor;
|
||||
dedupeKey: string;
|
||||
createdAtMs: number;
|
||||
}): RunEventRecord {
|
||||
return {
|
||||
id: this.createEventId(),
|
||||
runId: input.runId,
|
||||
sequence: input.decision.sequence,
|
||||
type: input.decision.type,
|
||||
dedupeKey: input.dedupeKey,
|
||||
actorType: input.actor.type,
|
||||
...(input.actor.id === undefined ? {} : { actorId: input.actor.id }),
|
||||
...(input.attemptId === undefined ? {} : { attemptId: input.attemptId }),
|
||||
payload: input.decision.payload,
|
||||
createdAtMs: input.createdAtMs,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
import { assertRunDispatchLeaseVersion } from '../domain/runDispatchLease';
|
||||
import {
|
||||
MAX_RUN_DISPATCH_LEASE_EXPIRY_PAGE_SIZE,
|
||||
type RunDispatchLeaseExpiryCursor,
|
||||
type RunDispatchLeaseExpirySource,
|
||||
} from '../ports/runDispatchLeaseExpirySource';
|
||||
import type {
|
||||
RunDispatchLeaseExpiryResult,
|
||||
RunDispatchLeaseExpiryService,
|
||||
RunDispatchLeaseExpiryStatus,
|
||||
} from './runDispatchLeaseExpiryService';
|
||||
|
||||
export interface RunDispatchLeaseExpiryScanSummary {
|
||||
observedAtMs: number;
|
||||
scanned: number;
|
||||
counts: Readonly<Record<RunDispatchLeaseExpiryStatus, number>>;
|
||||
failed: number;
|
||||
truncated: boolean;
|
||||
nextCursor?: RunDispatchLeaseExpiryCursor;
|
||||
}
|
||||
|
||||
const STATUSES: readonly RunDispatchLeaseExpiryStatus[] = [
|
||||
'lost',
|
||||
'cancellation_pending',
|
||||
'unstarted_released',
|
||||
'terminal_released',
|
||||
'already_expired',
|
||||
'not_due',
|
||||
'not_eligible',
|
||||
'not_found',
|
||||
];
|
||||
|
||||
/** One bounded expiry page. The caller owns cadence and cursor persistence. */
|
||||
export class RunDispatchLeaseExpiryScanner {
|
||||
private readonly clock: { now(): number };
|
||||
|
||||
constructor(
|
||||
private readonly source: RunDispatchLeaseExpirySource,
|
||||
private readonly service: Pick<RunDispatchLeaseExpiryService, 'reconcile'>,
|
||||
options: { clock?: { now(): number } } = {},
|
||||
) {
|
||||
this.clock = options.clock ?? Date;
|
||||
}
|
||||
|
||||
async scan(
|
||||
options: { after?: RunDispatchLeaseExpiryCursor; limit?: number } = {},
|
||||
): Promise<RunDispatchLeaseExpiryScanSummary> {
|
||||
const observedAtMs = this.now();
|
||||
const limit = options.limit ?? 16;
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_RUN_DISPATCH_LEASE_EXPIRY_PAGE_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
`limit must be between 1 and ${MAX_RUN_DISPATCH_LEASE_EXPIRY_PAGE_SIZE}`,
|
||||
);
|
||||
}
|
||||
const candidates = await this.source.listExpired({
|
||||
observedAtMs,
|
||||
...(options.after === undefined ? {} : { after: options.after }),
|
||||
limit,
|
||||
});
|
||||
if (candidates.length > limit) {
|
||||
throw new RangeError('Run lease expiry source exceeded page size');
|
||||
}
|
||||
const counts = Object.fromEntries(
|
||||
STATUSES.map((status) => [status, 0]),
|
||||
) as Record<RunDispatchLeaseExpiryStatus, number>;
|
||||
let failed = 0;
|
||||
let previous = options.after;
|
||||
let resumeCursor = options.after;
|
||||
for (const candidate of candidates) {
|
||||
if (
|
||||
candidate.expiresAtMs > observedAtMs ||
|
||||
(previous !== undefined &&
|
||||
(candidate.expiresAtMs < previous.expiresAtMs ||
|
||||
(candidate.expiresAtMs === previous.expiresAtMs &&
|
||||
candidate.attemptId <= previous.attemptId)))
|
||||
) {
|
||||
throw new TypeError('Run lease expiry cursor did not advance');
|
||||
}
|
||||
previous = {
|
||||
expiresAtMs: candidate.expiresAtMs,
|
||||
attemptId: candidate.attemptId,
|
||||
};
|
||||
try {
|
||||
const result: RunDispatchLeaseExpiryResult =
|
||||
await this.service.reconcile(candidate.runId, candidate.attemptId);
|
||||
counts[result.status] += 1;
|
||||
} catch {
|
||||
failed += 1;
|
||||
return {
|
||||
observedAtMs,
|
||||
scanned: candidates.length,
|
||||
counts,
|
||||
failed,
|
||||
truncated: true,
|
||||
...(resumeCursor === undefined
|
||||
? {}
|
||||
: { nextCursor: resumeCursor }),
|
||||
};
|
||||
}
|
||||
resumeCursor = previous;
|
||||
}
|
||||
const truncated = candidates.length === limit;
|
||||
return {
|
||||
observedAtMs,
|
||||
scanned: candidates.length,
|
||||
counts,
|
||||
failed,
|
||||
truncated,
|
||||
...(truncated && resumeCursor !== undefined
|
||||
? { nextCursor: resumeCursor }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock.now();
|
||||
assertRunDispatchLeaseVersion('observedAtMs', nowMs);
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
} from '../domain/run';
|
||||
import {
|
||||
assertRunDispatchId,
|
||||
assertRunDispatchLeaseVersion,
|
||||
type RunDispatchLeaseRecord,
|
||||
} from '../domain/runDispatchLease';
|
||||
import {
|
||||
isTerminalRunAttemptStatus,
|
||||
isTerminalRunStatus,
|
||||
transitionRun,
|
||||
transitionRunAttempt,
|
||||
type RunDomainEventDraft,
|
||||
} from '../domain/runStateMachine';
|
||||
import type { RunDispatchLeaseRepository } from '../ports/runDispatchLeaseRepository';
|
||||
import type { RunRepositoryTransaction } from '../ports/runRepository';
|
||||
|
||||
export type RunDispatchLeaseExpiryStatus =
|
||||
| 'lost'
|
||||
| 'cancellation_pending'
|
||||
| 'unstarted_released'
|
||||
| 'terminal_released'
|
||||
| 'already_expired'
|
||||
| 'not_due'
|
||||
| 'not_eligible'
|
||||
| 'not_found';
|
||||
|
||||
export interface RunDispatchLeaseExpiryResult {
|
||||
status: RunDispatchLeaseExpiryStatus;
|
||||
lease?: RunDispatchLeaseRecord;
|
||||
run?: RunRecord;
|
||||
attempt?: RunAttemptRecord;
|
||||
events?: readonly RunEventRecord[];
|
||||
}
|
||||
|
||||
export class RunDispatchLeaseExpiryTargetError extends Error {
|
||||
constructor(message: string) {
|
||||
super(`Run dispatch lease expiry target is inconsistent: ${message}`);
|
||||
this.name = 'RunDispatchLeaseExpiryTargetError';
|
||||
}
|
||||
}
|
||||
|
||||
interface ExpiryMutation {
|
||||
status:
|
||||
| 'lost'
|
||||
| 'cancellation_pending'
|
||||
| 'unstarted_released'
|
||||
| 'terminal_released';
|
||||
run: RunRecord;
|
||||
attempt: RunAttemptRecord;
|
||||
events: readonly RunEventRecord[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Server-owned expiry decision. No Worker principal can call this path with a
|
||||
* stale fence; the repository locks the authoritative lease and releases it
|
||||
* in the same transaction as any Attempt/Run lost transition.
|
||||
*/
|
||||
export class RunDispatchLeaseExpiryService {
|
||||
private readonly clock: { now(): number };
|
||||
private readonly createEventId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly leases: RunDispatchLeaseRepository,
|
||||
options: { clock?: { now(): number }; createEventId?: () => string } = {},
|
||||
) {
|
||||
this.clock = options.clock ?? Date;
|
||||
this.createEventId = options.createEventId ?? uuidV7;
|
||||
}
|
||||
|
||||
async reconcile(
|
||||
runId: string,
|
||||
attemptId: string,
|
||||
): Promise<RunDispatchLeaseExpiryResult> {
|
||||
assertRunDispatchId('runId', runId);
|
||||
assertRunDispatchId('attemptId', attemptId);
|
||||
const observedAtMs = this.now();
|
||||
const attemptEventId = this.eventId('attemptEventId');
|
||||
const runEventId = this.eventId('runEventId');
|
||||
const result = await this.leases.expireWithLease(
|
||||
{ runId, attemptId, observedAtMs },
|
||||
(transaction, lease) =>
|
||||
this.reconcileTarget(
|
||||
transaction,
|
||||
lease,
|
||||
observedAtMs,
|
||||
attemptEventId,
|
||||
runEventId,
|
||||
),
|
||||
);
|
||||
if (result.status === 'expired') {
|
||||
return {
|
||||
...result.value,
|
||||
lease: result.lease,
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: result.status,
|
||||
...(result.lease === undefined ? {} : { lease: result.lease }),
|
||||
};
|
||||
}
|
||||
|
||||
private async reconcileTarget(
|
||||
transaction: RunRepositoryTransaction,
|
||||
lease: RunDispatchLeaseRecord,
|
||||
observedAtMs: number,
|
||||
attemptEventId: string,
|
||||
runEventId: string,
|
||||
): Promise<ExpiryMutation> {
|
||||
const [run, attempt] = await Promise.all([
|
||||
transaction.findRunById(lease.runId),
|
||||
transaction.findAttemptById(lease.attemptId),
|
||||
]);
|
||||
if (!run || !attempt || attempt.runId !== run.id) {
|
||||
throw new RunDispatchLeaseExpiryTargetError('Run or Attempt is missing');
|
||||
}
|
||||
if (
|
||||
run.executionOwner !== 'runtime' ||
|
||||
(attempt.workerId !== undefined && attempt.workerId !== lease.workerId)
|
||||
) {
|
||||
throw new RunDispatchLeaseExpiryTargetError(
|
||||
'execution ownership does not match the expired lease',
|
||||
);
|
||||
}
|
||||
if (isTerminalRunStatus(run.status) || isTerminalRunAttemptStatus(attempt.status)) {
|
||||
return {
|
||||
status: 'terminal_released',
|
||||
run,
|
||||
attempt,
|
||||
events: [],
|
||||
};
|
||||
}
|
||||
if (run.cancelRequestedAtMs !== undefined) {
|
||||
return {
|
||||
status: 'cancellation_pending',
|
||||
run,
|
||||
attempt,
|
||||
events: [],
|
||||
};
|
||||
}
|
||||
if (attempt.status === 'claimed' && run.status === 'dispatching') {
|
||||
return {
|
||||
status: 'unstarted_released',
|
||||
run,
|
||||
attempt,
|
||||
events: [],
|
||||
};
|
||||
}
|
||||
if (
|
||||
(attempt.status !== 'starting' && attempt.status !== 'running') ||
|
||||
(run.status !== 'dispatching' && run.status !== 'running')
|
||||
) {
|
||||
throw new RunDispatchLeaseExpiryTargetError(
|
||||
'active Run and Attempt states do not match',
|
||||
);
|
||||
}
|
||||
|
||||
const atMs = Math.max(
|
||||
observedAtMs,
|
||||
run.createdAtMs,
|
||||
run.startedAtMs ?? 0,
|
||||
attempt.createdAtMs,
|
||||
attempt.startedAtMs ?? 0,
|
||||
);
|
||||
const errorCode = 'REMOTE_RUN_LEASE_EXPIRED';
|
||||
const errorSummary =
|
||||
'Remote execution authority expired before completion was observed';
|
||||
const attemptDecision = transitionRunAttempt(run, attempt, {
|
||||
to: 'lost',
|
||||
expectedRunVersion: run.version,
|
||||
atMs,
|
||||
errorCode,
|
||||
errorSummary,
|
||||
});
|
||||
const runDecision = transitionRun(attemptDecision.run, {
|
||||
to: 'lost',
|
||||
expectedVersion: attemptDecision.run.version,
|
||||
atMs,
|
||||
errorCode,
|
||||
errorSummary,
|
||||
});
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
attemptDecision.run,
|
||||
run.version,
|
||||
)) ||
|
||||
!(await transaction.compareAndSetAttempt(attemptDecision.attempt, {
|
||||
status: attempt.status,
|
||||
callbackSequence: attempt.callbackSequence,
|
||||
}))
|
||||
) {
|
||||
throw new RunDispatchLeaseExpiryTargetError(
|
||||
'Attempt lost transition lost its compare-and-set race',
|
||||
);
|
||||
}
|
||||
const attemptEvent = this.event(
|
||||
attemptEventId,
|
||||
attemptDecision.run,
|
||||
attemptDecision.event,
|
||||
attempt.id,
|
||||
lease,
|
||||
'attempt',
|
||||
atMs,
|
||||
);
|
||||
await transaction.appendEvent(attemptEvent);
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
runDecision.run,
|
||||
attemptDecision.run.version,
|
||||
))
|
||||
) {
|
||||
throw new RunDispatchLeaseExpiryTargetError(
|
||||
'Run lost transition lost its compare-and-set race',
|
||||
);
|
||||
}
|
||||
const runEvent = this.event(
|
||||
runEventId,
|
||||
runDecision.run,
|
||||
runDecision.event,
|
||||
attempt.id,
|
||||
lease,
|
||||
'run',
|
||||
atMs,
|
||||
);
|
||||
await transaction.appendEvent(runEvent);
|
||||
return {
|
||||
status: 'lost',
|
||||
run: runDecision.run,
|
||||
attempt: attemptDecision.attempt,
|
||||
events: [attemptEvent, runEvent],
|
||||
};
|
||||
}
|
||||
|
||||
private event(
|
||||
id: string,
|
||||
run: RunRecord,
|
||||
draft: RunDomainEventDraft,
|
||||
attemptId: string,
|
||||
lease: RunDispatchLeaseRecord,
|
||||
phase: 'attempt' | 'run',
|
||||
createdAtMs: number,
|
||||
): RunEventRecord {
|
||||
return {
|
||||
id,
|
||||
runId: run.id,
|
||||
sequence: draft.sequence,
|
||||
type: draft.type,
|
||||
dedupeKey: `run-lease-expiry:${attemptId}:${lease.leaseGeneration}:${phase}`,
|
||||
actorType: 'reconciler',
|
||||
attemptId,
|
||||
payload: {
|
||||
...draft.payload,
|
||||
lease_generation: lease.leaseGeneration,
|
||||
worker_id: lease.workerId,
|
||||
},
|
||||
createdAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
private eventId(name: string): string {
|
||||
const id = this.createEventId();
|
||||
assertRunDispatchId(name, id);
|
||||
return id;
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock.now();
|
||||
assertRunDispatchLeaseVersion('observedAtMs', nowMs);
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import {
|
||||
assertRunDispatchId,
|
||||
assertRunDispatchLeaseDuration,
|
||||
assertRunDispatchLeaseToken,
|
||||
assertRunDispatchLeaseVersion,
|
||||
assertRunDispatchWorkerFence,
|
||||
type RunDispatchReleaseReason,
|
||||
} from '../domain/runDispatchLease';
|
||||
import { assertWorkerId } from '../domain/worker';
|
||||
import type {
|
||||
ClaimRunDispatchLeaseResult,
|
||||
ReleaseRunDispatchLeaseResult,
|
||||
RunDispatchLeaseRepository,
|
||||
} from '../ports/runDispatchLeaseRepository';
|
||||
import type { AuthenticatedWorkerPrincipal } from './workerControlService';
|
||||
import { WorkerPrincipalMismatchError } from './workerControlService';
|
||||
|
||||
export interface RunDispatchLeaseServiceOptions {
|
||||
leaseDurationMs?: number;
|
||||
clock?: { now(): number };
|
||||
createEventId?: () => string;
|
||||
}
|
||||
|
||||
export interface ClaimRunDispatchLeaseRequest {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
workerId: string;
|
||||
workerSessionId: string;
|
||||
workerGeneration: number;
|
||||
leaseToken: string;
|
||||
}
|
||||
|
||||
export interface FencedRunDispatchLeaseRequest {
|
||||
attemptId: string;
|
||||
workerId: string;
|
||||
workerSessionId: string;
|
||||
workerGeneration: number;
|
||||
leaseGeneration: number;
|
||||
leaseToken: string;
|
||||
expectedVersion: number;
|
||||
}
|
||||
|
||||
export interface ReleaseRunDispatchLeaseRequest
|
||||
extends FencedRunDispatchLeaseRequest {
|
||||
runId: string;
|
||||
reason: Exclude<RunDispatchReleaseReason, 'lease_expired'>;
|
||||
}
|
||||
|
||||
export class RunDispatchLeaseService {
|
||||
private readonly leaseDurationMs: number;
|
||||
private readonly clock: { now(): number };
|
||||
private readonly createEventId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly repository: RunDispatchLeaseRepository,
|
||||
options: RunDispatchLeaseServiceOptions = {},
|
||||
) {
|
||||
this.leaseDurationMs = options.leaseDurationMs ?? 30_000;
|
||||
this.clock = options.clock ?? Date;
|
||||
this.createEventId = options.createEventId ?? uuidV7;
|
||||
assertRunDispatchLeaseDuration(this.leaseDurationMs);
|
||||
}
|
||||
|
||||
claim(
|
||||
principal: AuthenticatedWorkerPrincipal,
|
||||
request: ClaimRunDispatchLeaseRequest,
|
||||
): Promise<ClaimRunDispatchLeaseResult> {
|
||||
this.assertPrincipal(principal, request.workerId);
|
||||
this.assertClaimRequest(request);
|
||||
return this.repository.claim({
|
||||
...request,
|
||||
nowMs: this.now(),
|
||||
leaseDurationMs: this.leaseDurationMs,
|
||||
eventId: this.createEventId(),
|
||||
});
|
||||
}
|
||||
|
||||
renew(
|
||||
principal: AuthenticatedWorkerPrincipal,
|
||||
request: FencedRunDispatchLeaseRequest,
|
||||
) {
|
||||
this.assertPrincipal(principal, request.workerId);
|
||||
this.assertFenceRequest(request);
|
||||
return this.repository.renew({
|
||||
...request,
|
||||
nowMs: this.now(),
|
||||
leaseDurationMs: this.leaseDurationMs,
|
||||
});
|
||||
}
|
||||
|
||||
release(
|
||||
principal: AuthenticatedWorkerPrincipal,
|
||||
request: ReleaseRunDispatchLeaseRequest,
|
||||
): Promise<ReleaseRunDispatchLeaseResult> {
|
||||
this.assertPrincipal(principal, request.workerId);
|
||||
this.assertFenceRequest(request);
|
||||
return this.repository.release({
|
||||
...request,
|
||||
nowMs: this.now(),
|
||||
eventId: this.createEventId(),
|
||||
});
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock.now();
|
||||
assertRunDispatchLeaseVersion('nowMs', nowMs);
|
||||
return nowMs;
|
||||
}
|
||||
|
||||
private assertPrincipal(
|
||||
principal: AuthenticatedWorkerPrincipal,
|
||||
workerId: string,
|
||||
): void {
|
||||
assertWorkerId(principal.workerId);
|
||||
assertWorkerId(workerId);
|
||||
if (principal.workerId !== workerId) {
|
||||
throw new WorkerPrincipalMismatchError();
|
||||
}
|
||||
}
|
||||
|
||||
private assertClaimRequest(request: ClaimRunDispatchLeaseRequest): void {
|
||||
assertRunDispatchId('runId', request.runId);
|
||||
assertRunDispatchId('attemptId', request.attemptId);
|
||||
assertRunDispatchWorkerFence(request);
|
||||
assertRunDispatchLeaseToken(request.leaseToken);
|
||||
}
|
||||
|
||||
private assertFenceRequest(request: FencedRunDispatchLeaseRequest): void {
|
||||
assertRunDispatchId('attemptId', request.attemptId);
|
||||
assertRunDispatchWorkerFence(request);
|
||||
assertRunDispatchLeaseToken(request.leaseToken);
|
||||
assertRunDispatchLeaseVersion(
|
||||
'leaseGeneration',
|
||||
request.leaseGeneration,
|
||||
true,
|
||||
);
|
||||
assertRunDispatchLeaseVersion('expectedVersion', request.expectedVersion);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,523 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import {
|
||||
assertRunDispatchCandidate,
|
||||
assertRunDispatchCandidatePageSize,
|
||||
type RunDispatchCandidate,
|
||||
type RunDispatchCandidateCursor,
|
||||
} from '../domain/runDispatchCandidate';
|
||||
import {
|
||||
assertRunDispatchLeaseToken,
|
||||
assertRunDispatchLeaseVersion,
|
||||
RunDispatchLeaseFenceRejectedError,
|
||||
} from '../domain/runDispatchLease';
|
||||
import {
|
||||
createExecutionSpecDigest,
|
||||
createRunDispatchOfferId,
|
||||
type RunDispatcherIdleReason,
|
||||
type RunDispatcherResult,
|
||||
type RunDispatcherStats,
|
||||
} from '../domain/runDispatchOffer';
|
||||
import {
|
||||
assertRecoverableRunDispatch,
|
||||
assertRunDispatchRecoveryPageSize,
|
||||
type RunDispatchRecoveryCursor,
|
||||
} from '../domain/runDispatchRecovery';
|
||||
import {
|
||||
MAX_PLACEMENT_CANDIDATES,
|
||||
normalizeWorkerPlacementSpec,
|
||||
selectWorkerCandidates,
|
||||
type WorkerPlacementSpec,
|
||||
} from '../domain/workerPlacement';
|
||||
import type { RunDispatchCandidateSource } from '../ports/runDispatchCandidateSource';
|
||||
import type { ClaimRunDispatchLeaseResult } from '../ports/runDispatchLeaseRepository';
|
||||
import type { RunDispatchPlanSource } from '../ports/runDispatchPlanSource';
|
||||
import type { RunDispatchRecoverySource } from '../ports/runDispatchRecoverySource';
|
||||
import { executionSpecForRunDispatchCandidate } from '../domain/runDispatchPlan';
|
||||
import {
|
||||
MAX_AVAILABLE_WORKER_PAGE_SIZE,
|
||||
type WorkerRegistryRepository,
|
||||
} from '../ports/workerRegistryRepository';
|
||||
import type { ClaimRunDispatchLeaseRequest } from './runDispatchLeaseService';
|
||||
import type { AuthenticatedWorkerPrincipal } from './workerControlService';
|
||||
|
||||
const DEFAULT_CANDIDATE_PAGE_SIZE = 8;
|
||||
const DEFAULT_MAX_CANDIDATE_PAGES = 2;
|
||||
const DEFAULT_RECOVERY_PAGE_SIZE = 8;
|
||||
const DEFAULT_MAX_RECOVERY_PAGES = 2;
|
||||
const DEFAULT_WORKER_PAGE_SIZE = 8;
|
||||
const DEFAULT_MAX_WORKER_PAGES = 2;
|
||||
const DEFAULT_MAX_CLAIM_ATTEMPTS = 8;
|
||||
const MAX_DISPATCH_SCAN_PAGES = 16;
|
||||
|
||||
export interface RunDispatchClaimer {
|
||||
claim(
|
||||
principal: AuthenticatedWorkerPrincipal,
|
||||
request: ClaimRunDispatchLeaseRequest,
|
||||
): Promise<ClaimRunDispatchLeaseResult>;
|
||||
}
|
||||
|
||||
export interface RunDispatcherOptions {
|
||||
recoveryPageSize?: number;
|
||||
maxRecoveryPages?: number;
|
||||
candidatePageSize?: number;
|
||||
maxCandidatePages?: number;
|
||||
workerPageSize?: number;
|
||||
maxWorkerPages?: number;
|
||||
maxClaimAttempts?: number;
|
||||
clock?: { now(): number };
|
||||
createLeaseToken?: () => string;
|
||||
}
|
||||
|
||||
interface CandidatePageState {
|
||||
candidates: RunDispatchCandidate[];
|
||||
cursor?: RunDispatchCandidateCursor;
|
||||
more: boolean;
|
||||
}
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
function cursorOf(candidate: RunDispatchCandidate): RunDispatchCandidateCursor {
|
||||
return {
|
||||
priority: candidate.priority,
|
||||
queuedAtMs: candidate.queuedAtMs,
|
||||
attemptCreatedAtMs: candidate.attemptCreatedAtMs,
|
||||
attemptId: candidate.attemptId,
|
||||
};
|
||||
}
|
||||
|
||||
function cursorKey(cursor: RunDispatchCandidateCursor): string {
|
||||
return `${cursor.priority}\0${cursor.queuedAtMs}\0${cursor.attemptCreatedAtMs}\0${cursor.attemptId}`;
|
||||
}
|
||||
|
||||
function effectivePlacement(
|
||||
placementValue: unknown,
|
||||
executorType: string,
|
||||
): WorkerPlacementSpec {
|
||||
const placement = normalizeWorkerPlacementSpec(placementValue);
|
||||
const configuredExecutors = placement.required?.executors;
|
||||
if (
|
||||
configuredExecutors !== undefined &&
|
||||
!configuredExecutors.includes(executorType)
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Run dispatch plan executor placement does not match its candidate',
|
||||
);
|
||||
}
|
||||
return {
|
||||
...placement,
|
||||
required: {
|
||||
...placement.required,
|
||||
executors: configuredExecutors ?? [executorType],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function emptyStats(): RunDispatcherStats {
|
||||
return {
|
||||
recoveryPages: 0,
|
||||
recoveriesScanned: 0,
|
||||
recoveryPlansUnavailable: 0,
|
||||
candidatePages: 0,
|
||||
candidatesScanned: 0,
|
||||
workerPages: 0,
|
||||
workersScanned: 0,
|
||||
plansUnavailable: 0,
|
||||
matchingWorkers: 0,
|
||||
claimAttempts: 0,
|
||||
claimRaces: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/** One bounded dispatch cycle. The caller owns scheduling and offer delivery. */
|
||||
export class RunDispatcher {
|
||||
private readonly recoveryPageSize: number;
|
||||
private readonly maxRecoveryPages: number;
|
||||
private readonly candidatePageSize: number;
|
||||
private readonly maxCandidatePages: number;
|
||||
private readonly workerPageSize: number;
|
||||
private readonly maxWorkerPages: number;
|
||||
private readonly maxClaimAttempts: number;
|
||||
private readonly clock: { now(): number };
|
||||
private readonly createLeaseToken: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly recoveries: RunDispatchRecoverySource,
|
||||
private readonly candidates: RunDispatchCandidateSource,
|
||||
private readonly workers: WorkerRegistryRepository,
|
||||
private readonly plans: RunDispatchPlanSource,
|
||||
private readonly leases: RunDispatchClaimer,
|
||||
options: RunDispatcherOptions = {},
|
||||
) {
|
||||
this.recoveryPageSize =
|
||||
options.recoveryPageSize ?? DEFAULT_RECOVERY_PAGE_SIZE;
|
||||
this.maxRecoveryPages =
|
||||
options.maxRecoveryPages ?? DEFAULT_MAX_RECOVERY_PAGES;
|
||||
this.candidatePageSize =
|
||||
options.candidatePageSize ?? DEFAULT_CANDIDATE_PAGE_SIZE;
|
||||
this.maxCandidatePages =
|
||||
options.maxCandidatePages ?? DEFAULT_MAX_CANDIDATE_PAGES;
|
||||
this.workerPageSize = options.workerPageSize ?? DEFAULT_WORKER_PAGE_SIZE;
|
||||
this.maxWorkerPages = options.maxWorkerPages ?? DEFAULT_MAX_WORKER_PAGES;
|
||||
this.maxClaimAttempts =
|
||||
options.maxClaimAttempts ?? DEFAULT_MAX_CLAIM_ATTEMPTS;
|
||||
this.clock = options.clock ?? Date;
|
||||
this.createLeaseToken = options.createLeaseToken ?? uuidV7;
|
||||
|
||||
assertRunDispatchRecoveryPageSize(this.recoveryPageSize);
|
||||
assertIntegerBetween(
|
||||
'maxRecoveryPages',
|
||||
this.maxRecoveryPages,
|
||||
1,
|
||||
MAX_DISPATCH_SCAN_PAGES,
|
||||
);
|
||||
assertRunDispatchCandidatePageSize(this.candidatePageSize);
|
||||
assertIntegerBetween(
|
||||
'maxCandidatePages',
|
||||
this.maxCandidatePages,
|
||||
1,
|
||||
MAX_DISPATCH_SCAN_PAGES,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'workerPageSize',
|
||||
this.workerPageSize,
|
||||
1,
|
||||
MAX_AVAILABLE_WORKER_PAGE_SIZE,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'maxWorkerPages',
|
||||
this.maxWorkerPages,
|
||||
1,
|
||||
MAX_DISPATCH_SCAN_PAGES,
|
||||
);
|
||||
if (this.workerPageSize * this.maxWorkerPages > MAX_PLACEMENT_CANDIDATES) {
|
||||
throw new RangeError(
|
||||
`workerPageSize * maxWorkerPages must not exceed ${MAX_PLACEMENT_CANDIDATES}`,
|
||||
);
|
||||
}
|
||||
assertIntegerBetween(
|
||||
'maxClaimAttempts',
|
||||
this.maxClaimAttempts,
|
||||
1,
|
||||
MAX_PLACEMENT_CANDIDATES,
|
||||
);
|
||||
}
|
||||
|
||||
async dispatchOnce(): Promise<RunDispatcherResult> {
|
||||
const observedAtMs = this.now();
|
||||
const stats = emptyStats();
|
||||
const recoveryResult = await this.recoverOffer(observedAtMs, stats);
|
||||
if (recoveryResult) return recoveryResult;
|
||||
|
||||
const seenAttempts = new Set<string>();
|
||||
let page = await this.loadCandidatePage(
|
||||
observedAtMs,
|
||||
undefined,
|
||||
seenAttempts,
|
||||
stats,
|
||||
);
|
||||
if (page.candidates.length === 0) {
|
||||
return this.idle(
|
||||
stats.recoveryPlansUnavailable > 0
|
||||
? 'recovery_plans_unavailable'
|
||||
: 'no_candidates',
|
||||
stats,
|
||||
false,
|
||||
);
|
||||
}
|
||||
|
||||
const workerState = await this.loadWorkers(observedAtMs, stats);
|
||||
if (workerState.workers.length === 0) {
|
||||
return this.idle('no_workers', stats, workerState.truncated);
|
||||
}
|
||||
|
||||
let raced = false;
|
||||
while (true) {
|
||||
for (const candidate of page.candidates) {
|
||||
stats.candidatesScanned += 1;
|
||||
const plan = await this.plans.prepare({ ...candidate });
|
||||
if (!plan) {
|
||||
stats.plansUnavailable += 1;
|
||||
continue;
|
||||
}
|
||||
const executionSpec = executionSpecForRunDispatchCandidate(
|
||||
candidate,
|
||||
plan.executionSpec,
|
||||
);
|
||||
const selected = selectWorkerCandidates(
|
||||
workerState.workers,
|
||||
effectivePlacement(plan.placement, candidate.executorType),
|
||||
observedAtMs,
|
||||
MAX_PLACEMENT_CANDIDATES,
|
||||
);
|
||||
stats.matchingWorkers += selected.length;
|
||||
if (selected.length === 0) continue;
|
||||
for (const selectedWorker of selected) {
|
||||
if (stats.claimAttempts >= this.maxClaimAttempts) {
|
||||
return this.idle('claim_budget_exhausted', stats, true);
|
||||
}
|
||||
const leaseToken = this.createLeaseToken();
|
||||
assertRunDispatchLeaseToken(leaseToken);
|
||||
stats.claimAttempts += 1;
|
||||
let claim: ClaimRunDispatchLeaseResult;
|
||||
try {
|
||||
claim = await this.leases.claim(
|
||||
{ workerId: selectedWorker.worker.id },
|
||||
{
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
workerId: selectedWorker.worker.id,
|
||||
workerSessionId: selectedWorker.worker.sessionId,
|
||||
workerGeneration: selectedWorker.worker.generation,
|
||||
leaseToken,
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof RunDispatchLeaseFenceRejectedError &&
|
||||
error.reason === 'version_mismatch'
|
||||
) {
|
||||
stats.claimRaces += 1;
|
||||
raced = true;
|
||||
break;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (claim.status === 'claimed' || claim.status === 'idempotent') {
|
||||
return {
|
||||
status: 'offered',
|
||||
offer: {
|
||||
offerId: createRunDispatchOfferId(claim.lease),
|
||||
executionSpecDigest: createExecutionSpecDigest(executionSpec),
|
||||
deliveryKind: 'new_claim',
|
||||
candidate: { ...candidate },
|
||||
worker: {
|
||||
id: selectedWorker.worker.id,
|
||||
sessionId: selectedWorker.worker.sessionId,
|
||||
generation: selectedWorker.worker.generation,
|
||||
},
|
||||
lease: claim.lease,
|
||||
executionSpec,
|
||||
placementScore: selectedWorker.score,
|
||||
},
|
||||
stats,
|
||||
truncated: workerState.truncated || page.more,
|
||||
};
|
||||
}
|
||||
if (claim.status === 'leased' || claim.status === 'not_eligible') {
|
||||
stats.claimRaces += 1;
|
||||
raced = true;
|
||||
break;
|
||||
}
|
||||
stats.claimRaces += 1;
|
||||
raced = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!page.more) break;
|
||||
if (stats.candidatePages >= this.maxCandidatePages) {
|
||||
return this.idle('scan_budget_exhausted', stats, true);
|
||||
}
|
||||
page = await this.loadCandidatePage(
|
||||
observedAtMs,
|
||||
page.cursor,
|
||||
seenAttempts,
|
||||
stats,
|
||||
);
|
||||
if (page.candidates.length === 0) break;
|
||||
}
|
||||
|
||||
const truncated = workerState.truncated || page.more;
|
||||
if (stats.plansUnavailable === stats.candidatesScanned) {
|
||||
return this.idle('plans_unavailable', stats, truncated);
|
||||
}
|
||||
return this.idle(raced ? 'claim_raced' : 'no_match', stats, truncated);
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuilds one offer from durable lease authority before claiming new work.
|
||||
* A future Worker transport must deduplicate repeated delivery by offerId.
|
||||
*/
|
||||
private async recoverOffer(
|
||||
observedAtMs: number,
|
||||
stats: RunDispatcherStats,
|
||||
): Promise<RunDispatcherResult | null> {
|
||||
const seenAttempts = new Set<string>();
|
||||
let after: RunDispatchRecoveryCursor | undefined;
|
||||
for (let pageIndex = 0; pageIndex < this.maxRecoveryPages; pageIndex += 1) {
|
||||
const recoveries = await this.recoveries.listRecoverable({
|
||||
observedAtMs,
|
||||
...(after === undefined ? {} : { after }),
|
||||
limit: this.recoveryPageSize,
|
||||
});
|
||||
if (recoveries.length > this.recoveryPageSize) {
|
||||
throw new RangeError('Run dispatch recovery source exceeded page size');
|
||||
}
|
||||
stats.recoveryPages += 1;
|
||||
for (const recovery of recoveries) {
|
||||
assertRecoverableRunDispatch(recovery);
|
||||
if (recovery.lease.expiresAtMs <= observedAtMs) {
|
||||
throw new Error(
|
||||
'Run dispatch recovery source returned an expired lease',
|
||||
);
|
||||
}
|
||||
if (seenAttempts.has(recovery.candidate.attemptId)) {
|
||||
throw new Error('Run dispatch recovery source repeated an attempt');
|
||||
}
|
||||
if (
|
||||
after !== undefined &&
|
||||
(recovery.lease.expiresAtMs < after.expiresAtMs ||
|
||||
(recovery.lease.expiresAtMs === after.expiresAtMs &&
|
||||
recovery.candidate.attemptId.localeCompare(after.attemptId) <= 0))
|
||||
) {
|
||||
throw new Error('Run dispatch recovery cursor did not advance');
|
||||
}
|
||||
seenAttempts.add(recovery.candidate.attemptId);
|
||||
stats.recoveriesScanned += 1;
|
||||
const plan = await this.plans.prepare({ ...recovery.candidate });
|
||||
if (!plan) {
|
||||
stats.recoveryPlansUnavailable += 1;
|
||||
continue;
|
||||
}
|
||||
const executionSpec = executionSpecForRunDispatchCandidate(
|
||||
recovery.candidate,
|
||||
plan.executionSpec,
|
||||
);
|
||||
return {
|
||||
status: 'offered',
|
||||
offer: {
|
||||
offerId: createRunDispatchOfferId(recovery.lease),
|
||||
executionSpecDigest: createExecutionSpecDigest(executionSpec),
|
||||
deliveryKind: 'lease_recovery',
|
||||
candidate: { ...recovery.candidate },
|
||||
worker: {
|
||||
id: recovery.lease.workerId,
|
||||
sessionId: recovery.lease.workerSessionId,
|
||||
generation: recovery.lease.workerGeneration,
|
||||
},
|
||||
lease: { ...recovery.lease },
|
||||
executionSpec,
|
||||
},
|
||||
stats,
|
||||
truncated: recoveries.length === this.recoveryPageSize,
|
||||
};
|
||||
}
|
||||
if (recoveries.length < this.recoveryPageSize) return null;
|
||||
const last = recoveries[recoveries.length - 1];
|
||||
after = {
|
||||
expiresAtMs: last.lease.expiresAtMs,
|
||||
attemptId: last.candidate.attemptId,
|
||||
};
|
||||
}
|
||||
return this.idle('recovery_scan_budget_exhausted', stats, true);
|
||||
}
|
||||
|
||||
private async loadCandidatePage(
|
||||
observedAtMs: number,
|
||||
after: RunDispatchCandidateCursor | undefined,
|
||||
seenAttempts: Set<string>,
|
||||
stats: RunDispatcherStats,
|
||||
): Promise<CandidatePageState> {
|
||||
const candidates = await this.candidates.listCandidates({
|
||||
observedAtMs,
|
||||
...(after === undefined ? {} : { after }),
|
||||
limit: this.candidatePageSize,
|
||||
});
|
||||
if (candidates.length > this.candidatePageSize) {
|
||||
throw new RangeError('Run dispatch candidate source exceeded page size');
|
||||
}
|
||||
stats.candidatePages += 1;
|
||||
for (const candidate of candidates) {
|
||||
assertRunDispatchCandidate(candidate);
|
||||
if (seenAttempts.has(candidate.attemptId)) {
|
||||
throw new Error('Run dispatch candidate source repeated an attempt');
|
||||
}
|
||||
seenAttempts.add(candidate.attemptId);
|
||||
}
|
||||
const cursor =
|
||||
candidates.length === 0
|
||||
? after
|
||||
: cursorOf(candidates[candidates.length - 1]);
|
||||
if (
|
||||
after !== undefined &&
|
||||
cursor !== undefined &&
|
||||
cursorKey(after) === cursorKey(cursor)
|
||||
) {
|
||||
throw new Error('Run dispatch candidate cursor did not advance');
|
||||
}
|
||||
return {
|
||||
candidates,
|
||||
...(cursor === undefined ? {} : { cursor }),
|
||||
more: candidates.length === this.candidatePageSize,
|
||||
};
|
||||
}
|
||||
|
||||
private async loadWorkers(observedAtMs: number, stats: RunDispatcherStats) {
|
||||
const workers = [];
|
||||
const seen = new Set<string>();
|
||||
let afterWorkerId: string | undefined;
|
||||
let truncated = false;
|
||||
for (let pageIndex = 0; pageIndex < this.maxWorkerPages; pageIndex += 1) {
|
||||
const page = await this.workers.listAvailable({
|
||||
observedAtMs,
|
||||
...(afterWorkerId === undefined ? {} : { afterWorkerId }),
|
||||
limit: this.workerPageSize,
|
||||
});
|
||||
if (page.workers.length > this.workerPageSize) {
|
||||
throw new RangeError('Worker source exceeded page size');
|
||||
}
|
||||
stats.workerPages += 1;
|
||||
for (const worker of page.workers) {
|
||||
if (seen.has(worker.id)) {
|
||||
throw new Error('Worker source repeated a Worker');
|
||||
}
|
||||
if (
|
||||
afterWorkerId !== undefined &&
|
||||
worker.id.localeCompare(afterWorkerId) <= 0
|
||||
) {
|
||||
throw new Error('Worker source cursor did not advance');
|
||||
}
|
||||
seen.add(worker.id);
|
||||
workers.push(worker);
|
||||
}
|
||||
stats.workersScanned += page.workers.length;
|
||||
if (!page.truncated) {
|
||||
truncated = false;
|
||||
break;
|
||||
}
|
||||
truncated = true;
|
||||
if (
|
||||
!page.nextCursor ||
|
||||
page.nextCursor === afterWorkerId ||
|
||||
page.workers.length === 0 ||
|
||||
page.nextCursor !== page.workers[page.workers.length - 1].id
|
||||
) {
|
||||
throw new Error('Truncated Worker page has no advancing cursor');
|
||||
}
|
||||
afterWorkerId = page.nextCursor;
|
||||
}
|
||||
return { workers, truncated };
|
||||
}
|
||||
|
||||
private idle(
|
||||
reason: RunDispatcherIdleReason,
|
||||
stats: RunDispatcherStats,
|
||||
truncated: boolean,
|
||||
): RunDispatcherResult {
|
||||
return { status: 'idle', reason, stats, truncated };
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock.now();
|
||||
assertRunDispatchLeaseVersion('observedAtMs', nowMs);
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
import type {
|
||||
RunLostRetryScanSummary,
|
||||
RunLostRetryScanner,
|
||||
} from './runLostRetryScanner';
|
||||
|
||||
export const MIN_RUN_LOST_RETRY_INTERVAL_MS = 250;
|
||||
export const MAX_RUN_LOST_RETRY_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_RUN_LOST_RETRY_INITIAL_DELAY_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_RUN_LOST_RETRY_STOP_TIMEOUT_MS = 60_000;
|
||||
|
||||
interface ScheduledTimer {
|
||||
unref?: () => void;
|
||||
}
|
||||
|
||||
export interface RunLostRetryLifecycleScheduler {
|
||||
setTimeout(callback: () => void, delayMs: number): ScheduledTimer;
|
||||
clearTimeout(timer: ScheduledTimer): void;
|
||||
}
|
||||
|
||||
export interface RunLostRetryLifecycleOptions {
|
||||
intervalMs: number;
|
||||
initialDelayMs?: number;
|
||||
stopTimeoutMs?: number;
|
||||
pageSize?: number;
|
||||
scheduler?: RunLostRetryLifecycleScheduler;
|
||||
onCycle?: (summary: RunLostRetryScanSummary) => void;
|
||||
onError?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export type RunLostRetryStopResult = 'drained' | 'timed_out';
|
||||
|
||||
const defaultScheduler: RunLostRetryLifecycleScheduler = {
|
||||
setTimeout(callback, delayMs) {
|
||||
return setTimeout(callback, delayMs);
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
clearTimeout(timer as ReturnType<typeof setTimeout>);
|
||||
},
|
||||
};
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One-page, non-overlapping lost recovery cadence. It deliberately owns no
|
||||
* multi-page loop: a slow edge device pays at most one bounded scan per tick.
|
||||
*/
|
||||
export class RunLostRetryLifecycle {
|
||||
private readonly intervalMs: number;
|
||||
private readonly initialDelayMs: number;
|
||||
private readonly stopTimeoutMs: number;
|
||||
private readonly pageSize: number;
|
||||
private readonly scheduler: RunLostRetryLifecycleScheduler;
|
||||
private readonly onCycle?: (summary: RunLostRetryScanSummary) => void;
|
||||
private readonly onError?: (error: unknown) => void;
|
||||
private started = false;
|
||||
private timer?: ScheduledTimer;
|
||||
private inFlight?: Promise<void>;
|
||||
|
||||
constructor(
|
||||
private readonly scanner: Pick<RunLostRetryScanner, 'scan'>,
|
||||
options: RunLostRetryLifecycleOptions,
|
||||
) {
|
||||
this.intervalMs = options.intervalMs;
|
||||
this.initialDelayMs = options.initialDelayMs ?? 0;
|
||||
this.stopTimeoutMs = options.stopTimeoutMs ?? 5_000;
|
||||
this.pageSize = options.pageSize ?? 16;
|
||||
this.scheduler = options.scheduler ?? defaultScheduler;
|
||||
this.onCycle = options.onCycle;
|
||||
this.onError = options.onError;
|
||||
assertIntegerBetween(
|
||||
'intervalMs',
|
||||
this.intervalMs,
|
||||
MIN_RUN_LOST_RETRY_INTERVAL_MS,
|
||||
MAX_RUN_LOST_RETRY_INTERVAL_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'initialDelayMs',
|
||||
this.initialDelayMs,
|
||||
0,
|
||||
MAX_RUN_LOST_RETRY_INITIAL_DELAY_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'stopTimeoutMs',
|
||||
this.stopTimeoutMs,
|
||||
1,
|
||||
MAX_RUN_LOST_RETRY_STOP_TIMEOUT_MS,
|
||||
);
|
||||
assertIntegerBetween('pageSize', this.pageSize, 1, 64);
|
||||
}
|
||||
|
||||
start(): boolean {
|
||||
if (this.started || this.inFlight) return false;
|
||||
this.started = true;
|
||||
this.schedule(this.initialDelayMs);
|
||||
return true;
|
||||
}
|
||||
|
||||
async stop(): Promise<RunLostRetryStopResult> {
|
||||
this.started = false;
|
||||
if (this.timer) {
|
||||
this.scheduler.clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
const inFlight = this.inFlight;
|
||||
if (!inFlight) return 'drained';
|
||||
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const result = await Promise.race<RunLostRetryStopResult>([
|
||||
inFlight.then(() => 'drained' as const),
|
||||
new Promise<'timed_out'>((resolve) => {
|
||||
timeout = setTimeout(() => resolve('timed_out'), this.stopTimeoutMs);
|
||||
}),
|
||||
]);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
return result;
|
||||
}
|
||||
|
||||
private schedule(delayMs: number): void {
|
||||
if (!this.started || this.timer) return;
|
||||
const timer = this.scheduler.setTimeout(() => {
|
||||
if (this.timer === timer) this.timer = undefined;
|
||||
this.run();
|
||||
}, delayMs);
|
||||
this.timer = timer;
|
||||
timer.unref?.();
|
||||
}
|
||||
|
||||
private run(): void {
|
||||
if (!this.started || this.inFlight) return;
|
||||
const inFlight = this.scanner
|
||||
.scan({ limit: this.pageSize })
|
||||
.then((summary) => this.notifyCycle(summary))
|
||||
.catch((error) => this.notifyError(error))
|
||||
.then(() => undefined)
|
||||
.finally(() => {
|
||||
if (this.inFlight === inFlight) this.inFlight = undefined;
|
||||
if (this.started) this.schedule(this.intervalMs);
|
||||
});
|
||||
this.inFlight = inFlight;
|
||||
}
|
||||
|
||||
private notifyCycle(summary: RunLostRetryScanSummary): void {
|
||||
try {
|
||||
this.onCycle?.(summary);
|
||||
} catch (error) {
|
||||
this.notifyError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private notifyError(error: unknown): void {
|
||||
try {
|
||||
this.onError?.(error);
|
||||
} catch {
|
||||
// Diagnostics must never create another scheduler failure loop.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import type { RunLostRetryStatus } from './runLostRetryService';
|
||||
import type {
|
||||
RunLostRetrySource,
|
||||
RunLostRetryCandidate,
|
||||
} from '../ports/runLostRetrySource';
|
||||
import { MAX_RUN_LOST_RETRY_PAGE_SIZE } from '../ports/runLostRetrySource';
|
||||
|
||||
export interface RunLostRetryReconciler {
|
||||
reconcile(runId: string): Promise<{ status: RunLostRetryStatus }>;
|
||||
}
|
||||
|
||||
export interface RunLostRetryScanSummary {
|
||||
observedAtMs: number;
|
||||
scanned: number;
|
||||
failed: number;
|
||||
truncated: boolean;
|
||||
counts: Readonly<Partial<Record<RunLostRetryStatus, number>>>;
|
||||
failures: readonly { runId: string; reason: 'reconcile_failed' }[];
|
||||
}
|
||||
|
||||
export class RunLostRetryScanner {
|
||||
private readonly clock: { now(): number };
|
||||
|
||||
constructor(
|
||||
private readonly source: RunLostRetrySource,
|
||||
private readonly reconciler: RunLostRetryReconciler,
|
||||
options: { clock?: { now(): number } } = {},
|
||||
) {
|
||||
this.clock = options.clock ?? Date;
|
||||
}
|
||||
|
||||
async scan(
|
||||
options: { limit?: number } = {},
|
||||
): Promise<RunLostRetryScanSummary> {
|
||||
const observedAtMs = this.clock.now();
|
||||
const limit = options.limit ?? 16;
|
||||
if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0) {
|
||||
throw new RangeError('observedAtMs must be a non-negative safe integer');
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_RUN_LOST_RETRY_PAGE_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
`limit must be between 1 and ${MAX_RUN_LOST_RETRY_PAGE_SIZE}`,
|
||||
);
|
||||
}
|
||||
const candidates = await this.source.listCandidates({
|
||||
observedAtMs,
|
||||
limit,
|
||||
});
|
||||
this.assertPage(candidates, observedAtMs, limit);
|
||||
const counts: Partial<Record<RunLostRetryStatus, number>> = {};
|
||||
const failures: { runId: string; reason: 'reconcile_failed' }[] = [];
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
const result = await this.reconciler.reconcile(candidate.runId);
|
||||
counts[result.status] = (counts[result.status] ?? 0) + 1;
|
||||
} catch {
|
||||
failures.push({
|
||||
runId: candidate.runId,
|
||||
reason: 'reconcile_failed',
|
||||
});
|
||||
}
|
||||
}
|
||||
return {
|
||||
observedAtMs,
|
||||
scanned: candidates.length,
|
||||
failed: failures.length,
|
||||
truncated: candidates.length === limit,
|
||||
counts,
|
||||
failures,
|
||||
};
|
||||
}
|
||||
|
||||
private assertPage(
|
||||
candidates: readonly RunLostRetryCandidate[],
|
||||
observedAtMs: number,
|
||||
limit: number,
|
||||
): void {
|
||||
if (candidates.length > limit) {
|
||||
throw new TypeError('Lost retry source exceeded page size');
|
||||
}
|
||||
let previous: RunLostRetryCandidate | undefined;
|
||||
for (const candidate of candidates) {
|
||||
if (
|
||||
!candidate.runId ||
|
||||
(candidate.phase !== 'lost' && candidate.phase !== 'retry_wait') ||
|
||||
!Number.isSafeInteger(candidate.availableAtMs) ||
|
||||
candidate.availableAtMs < 0 ||
|
||||
candidate.availableAtMs > observedAtMs
|
||||
) {
|
||||
throw new TypeError('Lost retry source returned an invalid candidate');
|
||||
}
|
||||
if (
|
||||
previous &&
|
||||
(candidate.availableAtMs < previous.availableAtMs ||
|
||||
(candidate.availableAtMs === previous.availableAtMs &&
|
||||
candidate.runId <= previous.runId))
|
||||
) {
|
||||
throw new TypeError('Lost retry source page is not strictly ordered');
|
||||
}
|
||||
previous = candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,388 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
} from '../domain/run';
|
||||
import {
|
||||
assertRunRetryPolicyRecord,
|
||||
runRetryDelayMs,
|
||||
type RunRetryPolicyRecord,
|
||||
} from '../domain/runRetryPolicy';
|
||||
import {
|
||||
reserveRunEvent,
|
||||
transitionRun,
|
||||
type RunDomainEventDraft,
|
||||
} from '../domain/runStateMachine';
|
||||
import { RunVersionConflictError } from '../domain/stateMachineErrors';
|
||||
import type {
|
||||
RunRepository,
|
||||
RunRepositoryTransaction,
|
||||
} from '../ports/runRepository';
|
||||
|
||||
export type RunLostRetryStatus =
|
||||
| 'scheduled'
|
||||
| 'requeued'
|
||||
| 'cancelled'
|
||||
| 'failed_disabled'
|
||||
| 'failed_unsafe'
|
||||
| 'failed_exhausted'
|
||||
| 'not_due'
|
||||
| 'not_eligible'
|
||||
| 'not_found';
|
||||
|
||||
export interface RunLostRetryResult {
|
||||
status: RunLostRetryStatus;
|
||||
run?: RunRecord;
|
||||
attempt?: RunAttemptRecord;
|
||||
policy?: RunRetryPolicyRecord;
|
||||
events?: readonly RunEventRecord[];
|
||||
}
|
||||
|
||||
export class RunLostRetryTargetError extends Error {
|
||||
readonly code = 'RUN_LOST_RETRY_TARGET_INCONSISTENT';
|
||||
|
||||
constructor(message: string) {
|
||||
super(`Run lost retry target is inconsistent: ${message}`);
|
||||
this.name = 'RunLostRetryTargetError';
|
||||
}
|
||||
}
|
||||
|
||||
export class RunLostRetryService {
|
||||
private readonly clock: { now(): number };
|
||||
private readonly createId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
options: { clock?: { now(): number }; createId?: () => string } = {},
|
||||
) {
|
||||
this.clock = options.clock ?? Date;
|
||||
this.createId = options.createId ?? uuidV7;
|
||||
}
|
||||
|
||||
reconcile(runId: string): Promise<RunLostRetryResult> {
|
||||
if (!runId) throw new TypeError('runId is required');
|
||||
const observedAtMs = this.clock.now();
|
||||
if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0) {
|
||||
throw new RangeError('observedAtMs must be a non-negative safe integer');
|
||||
}
|
||||
return this.repository.transaction((transaction) =>
|
||||
this.reconcileInTransaction(transaction, runId, observedAtMs),
|
||||
);
|
||||
}
|
||||
|
||||
private async reconcileInTransaction(
|
||||
transaction: RunRepositoryTransaction,
|
||||
runId: string,
|
||||
observedAtMs: number,
|
||||
): Promise<RunLostRetryResult> {
|
||||
const run = await transaction.findRunById(runId);
|
||||
if (!run) return { status: 'not_found' };
|
||||
if (
|
||||
run.executionOwner !== 'runtime' ||
|
||||
(run.status !== 'lost' && run.status !== 'retry_wait')
|
||||
) {
|
||||
return { status: 'not_eligible', run };
|
||||
}
|
||||
const [attempt, policy] = await Promise.all([
|
||||
transaction.findLatestAttemptByRunId(run.id),
|
||||
transaction.findRetryPolicyByRunId(run.id),
|
||||
]);
|
||||
if (!attempt || attempt.runId !== run.id || attempt.status !== 'lost') {
|
||||
throw new RunLostRetryTargetError('latest Attempt is not lost');
|
||||
}
|
||||
if (policy) assertRunRetryPolicyRecord(policy);
|
||||
|
||||
const atMs = Math.max(
|
||||
observedAtMs,
|
||||
run.createdAtMs,
|
||||
run.startedAtMs ?? 0,
|
||||
attempt.createdAtMs,
|
||||
attempt.startedAtMs ?? 0,
|
||||
attempt.finishedAtMs ?? 0,
|
||||
policy?.updatedAtMs ?? 0,
|
||||
);
|
||||
if (run.cancelRequestedAtMs !== undefined) {
|
||||
return this.finishRun(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
policy,
|
||||
atMs,
|
||||
'cancelled',
|
||||
'cancelled',
|
||||
'RUN_CANCELLED_DURING_LOST_RECOVERY',
|
||||
'Cancellation won before a replacement Attempt was created',
|
||||
);
|
||||
}
|
||||
if (!policy || !policy.retryOnLost || policy.maxAttempts <= 1) {
|
||||
return this.finishRun(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
policy,
|
||||
atMs,
|
||||
'failed_disabled',
|
||||
'failed',
|
||||
'RUN_LOST_RETRY_DISABLED',
|
||||
'Run was lost and automatic retry was not enabled at admission',
|
||||
);
|
||||
}
|
||||
if (policy.safety === 'unknown') {
|
||||
return this.finishRun(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
policy,
|
||||
atMs,
|
||||
'failed_unsafe',
|
||||
'failed',
|
||||
'RUN_LOST_RETRY_UNSAFE',
|
||||
'Run was lost but execution safety was not declared',
|
||||
);
|
||||
}
|
||||
if (attempt.attempt >= policy.maxAttempts) {
|
||||
return this.finishRun(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
policy,
|
||||
atMs,
|
||||
'failed_exhausted',
|
||||
'failed',
|
||||
'RUN_LOST_RETRY_EXHAUSTED',
|
||||
'Run exhausted its admitted automatic retry attempts',
|
||||
);
|
||||
}
|
||||
if (run.status === 'lost') {
|
||||
return this.schedule(transaction, run, attempt, policy, atMs);
|
||||
}
|
||||
if (
|
||||
policy.nextAttemptAtMs === undefined ||
|
||||
policy.nextAttemptAtMs > observedAtMs
|
||||
) {
|
||||
return { status: 'not_due', run, attempt, policy };
|
||||
}
|
||||
return this.requeue(transaction, run, attempt, policy, atMs);
|
||||
}
|
||||
|
||||
private async schedule(
|
||||
transaction: RunRepositoryTransaction,
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
policy: RunRetryPolicyRecord,
|
||||
atMs: number,
|
||||
): Promise<RunLostRetryResult> {
|
||||
const lostAtMs = Math.max(
|
||||
run.createdAtMs,
|
||||
attempt.createdAtMs,
|
||||
attempt.finishedAtMs ?? 0,
|
||||
);
|
||||
const nextAttemptAtMs = lostAtMs + runRetryDelayMs(policy, attempt.attempt);
|
||||
if (!Number.isSafeInteger(nextAttemptAtMs)) {
|
||||
throw new RunLostRetryTargetError('next Attempt time overflowed');
|
||||
}
|
||||
const decision = transitionRun(run, {
|
||||
to: 'retry_wait',
|
||||
expectedVersion: run.version,
|
||||
atMs,
|
||||
errorCode: 'RUN_LOST_RETRY_SCHEDULED',
|
||||
errorSummary:
|
||||
'A fresh Attempt will be created after the admitted backoff',
|
||||
});
|
||||
const nextPolicy: RunRetryPolicyRecord = {
|
||||
...policy,
|
||||
nextAttemptAtMs,
|
||||
version: policy.version + 1,
|
||||
updatedAtMs: atMs,
|
||||
};
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(decision.run, run.version)) ||
|
||||
!(await transaction.compareAndSetRetryPolicy(nextPolicy, policy.version))
|
||||
) {
|
||||
throw new RunVersionConflictError(run.id, run.version, run.version);
|
||||
}
|
||||
const event = this.event(
|
||||
decision.run,
|
||||
decision.event,
|
||||
attempt.id,
|
||||
`run-lost-retry-scheduled:${attempt.id}`,
|
||||
atMs,
|
||||
{
|
||||
attempt: attempt.attempt,
|
||||
max_attempts: policy.maxAttempts,
|
||||
safety: policy.safety,
|
||||
next_attempt_at_ms: nextAttemptAtMs,
|
||||
},
|
||||
);
|
||||
await transaction.appendEvent(event);
|
||||
return {
|
||||
status: 'scheduled',
|
||||
run: decision.run,
|
||||
attempt,
|
||||
policy: nextPolicy,
|
||||
events: [event],
|
||||
};
|
||||
}
|
||||
|
||||
private async requeue(
|
||||
transaction: RunRepositoryTransaction,
|
||||
run: RunRecord,
|
||||
lostAttempt: RunAttemptRecord,
|
||||
policy: RunRetryPolicyRecord,
|
||||
atMs: number,
|
||||
): Promise<RunLostRetryResult> {
|
||||
const attempt: RunAttemptRecord = {
|
||||
id: this.createId(),
|
||||
runId: run.id,
|
||||
attempt: lostAttempt.attempt + 1,
|
||||
status: 'claimed',
|
||||
executorType: lostAttempt.executorType,
|
||||
callbackSequence: 0,
|
||||
createdAtMs: atMs,
|
||||
};
|
||||
const queued = transitionRun(run, {
|
||||
to: 'queued',
|
||||
expectedVersion: run.version,
|
||||
atMs,
|
||||
});
|
||||
const nextPolicy: RunRetryPolicyRecord = {
|
||||
...policy,
|
||||
version: policy.version + 1,
|
||||
updatedAtMs: atMs,
|
||||
};
|
||||
delete nextPolicy.nextAttemptAtMs;
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(queued.run, run.version)) ||
|
||||
!(await transaction.compareAndSetRetryPolicy(nextPolicy, policy.version))
|
||||
) {
|
||||
throw new RunVersionConflictError(run.id, run.version, run.version);
|
||||
}
|
||||
await transaction.insertAttempt(attempt);
|
||||
const queuedEvent = this.event(
|
||||
queued.run,
|
||||
queued.event,
|
||||
attempt.id,
|
||||
`run-lost-retry-queued:${attempt.id}`,
|
||||
atMs,
|
||||
{ previous_attempt_id: lostAttempt.id },
|
||||
);
|
||||
await transaction.appendEvent(queuedEvent);
|
||||
|
||||
const claimed = reserveRunEvent(queued.run, queued.run.version);
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(claimed.run, queued.run.version))
|
||||
) {
|
||||
throw new RunVersionConflictError(
|
||||
run.id,
|
||||
queued.run.version,
|
||||
queued.run.version,
|
||||
);
|
||||
}
|
||||
const claimedEvent = this.event(
|
||||
claimed.run,
|
||||
{
|
||||
sequence: claimed.sequence,
|
||||
type: 'attempt.claimed',
|
||||
payload: {
|
||||
attempt: attempt.attempt,
|
||||
executor_type: attempt.executorType,
|
||||
version: claimed.run.version,
|
||||
},
|
||||
},
|
||||
attempt.id,
|
||||
`run-lost-retry-attempt-claimed:${attempt.id}`,
|
||||
atMs,
|
||||
{ previous_attempt_id: lostAttempt.id },
|
||||
);
|
||||
await transaction.appendEvent(claimedEvent);
|
||||
return {
|
||||
status: 'requeued',
|
||||
run: claimed.run,
|
||||
attempt,
|
||||
policy: nextPolicy,
|
||||
events: [queuedEvent, claimedEvent],
|
||||
};
|
||||
}
|
||||
|
||||
private async finishRun(
|
||||
transaction: RunRepositoryTransaction,
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
policy: RunRetryPolicyRecord | null,
|
||||
atMs: number,
|
||||
status:
|
||||
| 'cancelled'
|
||||
| 'failed_disabled'
|
||||
| 'failed_unsafe'
|
||||
| 'failed_exhausted',
|
||||
to: 'cancelled' | 'failed',
|
||||
errorCode: string,
|
||||
errorSummary: string,
|
||||
): Promise<RunLostRetryResult> {
|
||||
const decision = transitionRun(run, {
|
||||
to,
|
||||
expectedVersion: run.version,
|
||||
atMs,
|
||||
errorCode,
|
||||
errorSummary,
|
||||
});
|
||||
if (!(await transaction.compareAndSetRun(decision.run, run.version))) {
|
||||
throw new RunVersionConflictError(run.id, run.version, run.version);
|
||||
}
|
||||
let nextPolicy = policy ?? undefined;
|
||||
if (policy?.nextAttemptAtMs !== undefined) {
|
||||
nextPolicy = {
|
||||
...policy,
|
||||
version: policy.version + 1,
|
||||
updatedAtMs: atMs,
|
||||
};
|
||||
delete nextPolicy.nextAttemptAtMs;
|
||||
if (
|
||||
!(await transaction.compareAndSetRetryPolicy(
|
||||
nextPolicy,
|
||||
policy.version,
|
||||
))
|
||||
) {
|
||||
throw new RunLostRetryTargetError('retry policy update lost its race');
|
||||
}
|
||||
}
|
||||
const event = this.event(
|
||||
decision.run,
|
||||
decision.event,
|
||||
attempt.id,
|
||||
`run-lost-retry-${status}:${attempt.id}`,
|
||||
atMs,
|
||||
{ attempt: attempt.attempt },
|
||||
);
|
||||
await transaction.appendEvent(event);
|
||||
return {
|
||||
status,
|
||||
run: decision.run,
|
||||
attempt,
|
||||
...(nextPolicy === undefined ? {} : { policy: nextPolicy }),
|
||||
events: [event],
|
||||
};
|
||||
}
|
||||
|
||||
private event(
|
||||
run: RunRecord,
|
||||
draft: RunDomainEventDraft,
|
||||
attemptId: string,
|
||||
dedupeKey: string,
|
||||
createdAtMs: number,
|
||||
extraPayload: Readonly<Record<string, unknown>>,
|
||||
): RunEventRecord {
|
||||
return {
|
||||
id: this.createId(),
|
||||
runId: run.id,
|
||||
sequence: draft.sequence,
|
||||
type: draft.type,
|
||||
dedupeKey,
|
||||
actorType: 'reconciler',
|
||||
attemptId,
|
||||
payload: { ...draft.payload, ...extraPayload },
|
||||
createdAtMs,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
import {
|
||||
assertWorkerConcurrency,
|
||||
assertWorkerId,
|
||||
assertWorkerSessionId,
|
||||
hashWorkerCapabilities,
|
||||
serializeWorkerCapabilities,
|
||||
type WorkerRecord,
|
||||
} from '../domain/worker';
|
||||
import type {
|
||||
HeartbeatWorkerRequest,
|
||||
RegisterWorkerRequest,
|
||||
TransitionWorkerRequest,
|
||||
WorkerControlPlaneClient,
|
||||
} from '../ports/workerControlPlaneClient';
|
||||
import type { WorkerRegistryRepository } from '../ports/workerRegistryRepository';
|
||||
|
||||
export const MIN_WORKER_LEASE_DURATION_MS = 5_000;
|
||||
export const MAX_WORKER_LEASE_DURATION_MS = 10 * 60_000;
|
||||
|
||||
export interface AuthenticatedWorkerPrincipal {
|
||||
workerId: string;
|
||||
}
|
||||
|
||||
export interface WorkerControlServiceOptions {
|
||||
leaseDurationMs?: number;
|
||||
clock?: { now(): number };
|
||||
}
|
||||
|
||||
export class WorkerPrincipalMismatchError extends Error {
|
||||
constructor() {
|
||||
super('Authenticated Worker principal does not match the requested worker');
|
||||
this.name = 'WorkerPrincipalMismatchError';
|
||||
}
|
||||
}
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
function safeExpiration(nowMs: number, durationMs: number): number {
|
||||
assertIntegerBetween('nowMs', nowMs, 0, Number.MAX_SAFE_INTEGER);
|
||||
if (nowMs > Number.MAX_SAFE_INTEGER - durationMs) {
|
||||
throw new RangeError('Worker lease expiration exceeds the safe range');
|
||||
}
|
||||
return nowMs + durationMs;
|
||||
}
|
||||
|
||||
export class WorkerControlService {
|
||||
private readonly leaseDurationMs: number;
|
||||
private readonly clock: { now(): number };
|
||||
|
||||
constructor(
|
||||
private readonly repository: WorkerRegistryRepository,
|
||||
options: WorkerControlServiceOptions = {},
|
||||
) {
|
||||
this.leaseDurationMs = options.leaseDurationMs ?? 45_000;
|
||||
this.clock = options.clock ?? Date;
|
||||
assertIntegerBetween(
|
||||
'leaseDurationMs',
|
||||
this.leaseDurationMs,
|
||||
MIN_WORKER_LEASE_DURATION_MS,
|
||||
MAX_WORKER_LEASE_DURATION_MS,
|
||||
);
|
||||
}
|
||||
|
||||
async register(
|
||||
principal: AuthenticatedWorkerPrincipal,
|
||||
request: RegisterWorkerRequest,
|
||||
): Promise<WorkerRecord> {
|
||||
this.assertPrincipal(principal, request.workerId);
|
||||
assertWorkerSessionId(request.sessionId);
|
||||
assertWorkerConcurrency(request.maxConcurrentRuns, request.availableSlots);
|
||||
const capabilitiesJson = serializeWorkerCapabilities(request.capabilities);
|
||||
const nowMs = this.clock.now();
|
||||
const result = await this.repository.register({
|
||||
workerId: request.workerId,
|
||||
sessionId: request.sessionId,
|
||||
capabilitiesJson,
|
||||
capabilitiesHash: hashWorkerCapabilities(capabilitiesJson),
|
||||
maxConcurrentRuns: request.maxConcurrentRuns,
|
||||
availableSlots: request.availableSlots,
|
||||
registeredAtMs: nowMs,
|
||||
leaseExpiresAtMs: safeExpiration(nowMs, this.leaseDurationMs),
|
||||
});
|
||||
return result.worker;
|
||||
}
|
||||
|
||||
async heartbeat(
|
||||
principal: AuthenticatedWorkerPrincipal,
|
||||
request: HeartbeatWorkerRequest,
|
||||
): Promise<WorkerRecord> {
|
||||
this.assertPrincipal(principal, request.workerId);
|
||||
const nowMs = this.clock.now();
|
||||
return this.repository.heartbeat({
|
||||
...request,
|
||||
heartbeatAtMs: nowMs,
|
||||
leaseExpiresAtMs: safeExpiration(nowMs, this.leaseDurationMs),
|
||||
});
|
||||
}
|
||||
|
||||
async drain(
|
||||
principal: AuthenticatedWorkerPrincipal,
|
||||
request: TransitionWorkerRequest,
|
||||
): Promise<WorkerRecord> {
|
||||
return this.transition(principal, request, 'draining');
|
||||
}
|
||||
|
||||
async disconnect(
|
||||
principal: AuthenticatedWorkerPrincipal,
|
||||
request: TransitionWorkerRequest,
|
||||
): Promise<WorkerRecord> {
|
||||
return this.transition(principal, request, 'offline');
|
||||
}
|
||||
|
||||
async listAvailable(
|
||||
options: {
|
||||
afterWorkerId?: string;
|
||||
limit?: number;
|
||||
} = {},
|
||||
) {
|
||||
return this.repository.listAvailable({
|
||||
observedAtMs: this.clock.now(),
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
private async transition(
|
||||
principal: AuthenticatedWorkerPrincipal,
|
||||
request: TransitionWorkerRequest,
|
||||
status: 'draining' | 'offline',
|
||||
): Promise<WorkerRecord> {
|
||||
this.assertPrincipal(principal, request.workerId);
|
||||
return this.repository.transition({
|
||||
...request,
|
||||
status,
|
||||
transitionedAtMs: this.clock.now(),
|
||||
});
|
||||
}
|
||||
|
||||
private assertPrincipal(
|
||||
principal: AuthenticatedWorkerPrincipal,
|
||||
workerId: string,
|
||||
): void {
|
||||
assertWorkerId(principal.workerId);
|
||||
assertWorkerId(workerId);
|
||||
if (principal.workerId !== workerId) {
|
||||
throw new WorkerPrincipalMismatchError();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Binds an authenticated transport principal to the application service. A
|
||||
* future HTTP/gRPC adapter must establish this principal before constructing
|
||||
* the client; request payloads never get to self-authenticate a Worker id.
|
||||
*/
|
||||
export class BoundWorkerControlPlaneClient implements WorkerControlPlaneClient {
|
||||
constructor(
|
||||
private readonly service: WorkerControlService,
|
||||
private readonly principal: AuthenticatedWorkerPrincipal,
|
||||
) {}
|
||||
|
||||
register(request: RegisterWorkerRequest): Promise<WorkerRecord> {
|
||||
return this.service.register(this.principal, request);
|
||||
}
|
||||
|
||||
heartbeat(request: HeartbeatWorkerRequest): Promise<WorkerRecord> {
|
||||
return this.service.heartbeat(this.principal, request);
|
||||
}
|
||||
|
||||
drain(request: TransitionWorkerRequest): Promise<WorkerRecord> {
|
||||
return this.service.drain(this.principal, request);
|
||||
}
|
||||
|
||||
disconnect(request: TransitionWorkerRequest): Promise<WorkerRecord> {
|
||||
return this.service.disconnect(this.principal, request);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
import type { ExecutionStopReason } from '../domain/execution';
|
||||
import {
|
||||
cloneWorkerExecutionOfferJournalRecord,
|
||||
type WorkerExecutionOfferJournalRecord,
|
||||
} from '../domain/workerExecutionOffer';
|
||||
import { assertRunDispatchLeaseRecord } from '../domain/runDispatchLease';
|
||||
import { createRunDispatchOfferId } from '../domain/runDispatchOffer';
|
||||
import type { PersistedExecutionController } from '../ports/persistedExecutionController';
|
||||
import type { WorkerExecutionOfferJournal } from '../ports/workerExecutionOfferJournal';
|
||||
import type {
|
||||
WorkerRunLeaseLoss,
|
||||
WorkerRunLeaseLossReason,
|
||||
} from './workerRunLeaseLifecycle';
|
||||
|
||||
export type WorkerExecutionLeaseLossActionStatus =
|
||||
| 'not_found'
|
||||
| 'authority_mismatch'
|
||||
| 'no_local_execution'
|
||||
| 'already_completed'
|
||||
| 'already_stopped'
|
||||
| 'already_unverified'
|
||||
| 'stop_acknowledged'
|
||||
| 'stop_unverified';
|
||||
|
||||
export interface WorkerExecutionLeaseLossActionResult {
|
||||
offerId: string;
|
||||
attemptId: string;
|
||||
reason: WorkerRunLeaseLossReason;
|
||||
status: WorkerExecutionLeaseLossActionStatus;
|
||||
stopStatus?: Awaited<
|
||||
ReturnType<PersistedExecutionController['stop']>
|
||||
>['status'];
|
||||
}
|
||||
|
||||
const LOSS_REASONS = new Set<WorkerRunLeaseLossReason>([
|
||||
'lease_expired',
|
||||
'fenced',
|
||||
'worker_session_replaced',
|
||||
'worker_unavailable',
|
||||
'invalid_renewal',
|
||||
]);
|
||||
|
||||
const EXECUTION_OWNERSHIP_STATES = new Set<
|
||||
WorkerExecutionOfferJournalRecord['state']
|
||||
>(['launching', 'started', 'running_acknowledged', 'recovery_required']);
|
||||
|
||||
/**
|
||||
* Fails closed after a Worker loses a Run lease. It may stop only the durable
|
||||
* local identity bound to the exact lost authority and never mutates the
|
||||
* control plane. Server-owned lease expiry reconciliation decides Run/Attempt
|
||||
* lost and any later retry.
|
||||
*/
|
||||
export class WorkerExecutionLeaseLossCoordinator {
|
||||
private readonly clock: { now(): number };
|
||||
private readonly inFlight = new Map<
|
||||
string,
|
||||
Promise<WorkerExecutionLeaseLossActionResult>
|
||||
>();
|
||||
|
||||
constructor(
|
||||
private readonly journal: Pick<
|
||||
WorkerExecutionOfferJournal,
|
||||
'read' | 'replace'
|
||||
>,
|
||||
private readonly controller: PersistedExecutionController,
|
||||
options: { clock?: { now(): number } } = {},
|
||||
) {
|
||||
this.clock = options.clock ?? Date;
|
||||
}
|
||||
|
||||
reconcile(
|
||||
candidate: WorkerRunLeaseLoss,
|
||||
): Promise<WorkerExecutionLeaseLossActionResult> {
|
||||
assertRunDispatchLeaseRecord(candidate.lease);
|
||||
if (!LOSS_REASONS.has(candidate.reason)) {
|
||||
throw new TypeError('Worker Run lease loss reason is invalid');
|
||||
}
|
||||
const loss: WorkerRunLeaseLoss = {
|
||||
lease: { ...candidate.lease },
|
||||
reason: candidate.reason,
|
||||
...(candidate.error === undefined ? {} : { error: candidate.error }),
|
||||
};
|
||||
const offerId = createRunDispatchOfferId(loss.lease);
|
||||
const active = this.inFlight.get(offerId);
|
||||
if (active) return active;
|
||||
const operation = this.process(offerId, loss).finally(() => {
|
||||
if (this.inFlight.get(offerId) === operation) {
|
||||
this.inFlight.delete(offerId);
|
||||
}
|
||||
});
|
||||
this.inFlight.set(offerId, operation);
|
||||
return operation;
|
||||
}
|
||||
|
||||
private async process(
|
||||
offerId: string,
|
||||
loss: WorkerRunLeaseLoss,
|
||||
): Promise<WorkerExecutionLeaseLossActionResult> {
|
||||
const base = {
|
||||
offerId,
|
||||
attemptId: loss.lease.attemptId,
|
||||
reason: loss.reason,
|
||||
} as const;
|
||||
const record = await this.journal.read(offerId);
|
||||
if (!record) return { ...base, status: 'not_found' };
|
||||
if (!this.sameAuthority(record, loss)) {
|
||||
return { ...base, status: 'authority_mismatch' };
|
||||
}
|
||||
if (record.state === 'completion_acknowledged') {
|
||||
return { ...base, status: 'already_completed' };
|
||||
}
|
||||
if (
|
||||
record.state === 'recovery_required' &&
|
||||
record.recoveryReason === 'lease_lost_local_execution_stopped'
|
||||
) {
|
||||
return { ...base, status: 'already_stopped' };
|
||||
}
|
||||
if (
|
||||
record.state === 'recovery_required' &&
|
||||
record.recoveryReason === 'lease_lost_local_execution_unverified'
|
||||
) {
|
||||
return { ...base, status: 'already_unverified' };
|
||||
}
|
||||
if (!EXECUTION_OWNERSHIP_STATES.has(record.state)) {
|
||||
return { ...base, status: 'no_local_execution' };
|
||||
}
|
||||
if (!record.executorHandle) {
|
||||
await this.mark(record, 'lease_lost_local_execution_unverified');
|
||||
return { ...base, status: 'stop_unverified' };
|
||||
}
|
||||
|
||||
const reason: ExecutionStopReason = {
|
||||
kind: 'reconcile',
|
||||
requestedAtMs: this.now(),
|
||||
};
|
||||
const stopped = await this.controller.stop({
|
||||
durableHandle: record.executorHandle,
|
||||
reason,
|
||||
});
|
||||
const acknowledged =
|
||||
stopped.status === 'termination_requested' ||
|
||||
stopped.status === 'already_exited';
|
||||
const recoveryReason = acknowledged
|
||||
? 'lease_lost_local_execution_stopped'
|
||||
: 'lease_lost_local_execution_unverified';
|
||||
await this.mark(record, recoveryReason);
|
||||
return {
|
||||
...base,
|
||||
status: acknowledged ? 'stop_acknowledged' : 'stop_unverified',
|
||||
stopStatus: stopped.status,
|
||||
};
|
||||
}
|
||||
|
||||
private sameAuthority(
|
||||
record: WorkerExecutionOfferJournalRecord,
|
||||
loss: WorkerRunLeaseLoss,
|
||||
): boolean {
|
||||
const lease = record.offer.lease;
|
||||
return (
|
||||
lease.attemptId === loss.lease.attemptId &&
|
||||
lease.runId === loss.lease.runId &&
|
||||
lease.workerId === loss.lease.workerId &&
|
||||
lease.workerSessionId === loss.lease.workerSessionId &&
|
||||
lease.workerGeneration === loss.lease.workerGeneration &&
|
||||
lease.leaseGeneration === loss.lease.leaseGeneration &&
|
||||
lease.leaseToken === loss.lease.leaseToken
|
||||
);
|
||||
}
|
||||
|
||||
private async mark(
|
||||
previous: WorkerExecutionOfferJournalRecord,
|
||||
recoveryReason:
|
||||
| 'lease_lost_local_execution_stopped'
|
||||
| 'lease_lost_local_execution_unverified',
|
||||
): Promise<void> {
|
||||
const updated = cloneWorkerExecutionOfferJournalRecord({
|
||||
...previous,
|
||||
schemaVersion: 1,
|
||||
revision: previous.revision + 1,
|
||||
state: 'recovery_required',
|
||||
recoveryReason,
|
||||
updatedAtMs: Math.max(this.now(), previous.updatedAtMs),
|
||||
});
|
||||
try {
|
||||
await this.journal.replace(updated, previous.revision);
|
||||
} catch (error) {
|
||||
const current = await this.journal.read(previous.offer.offerId);
|
||||
if (
|
||||
current?.state === 'completion_acknowledged' ||
|
||||
(current?.state === 'recovery_required' &&
|
||||
(current.recoveryReason ===
|
||||
'lease_lost_local_execution_stopped' ||
|
||||
current.recoveryReason ===
|
||||
'lease_lost_local_execution_unverified'))
|
||||
) {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock.now();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new TypeError(
|
||||
'Worker execution lease loss coordinator clock returned an invalid time',
|
||||
);
|
||||
}
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import type { WorkerRecord } from '../domain/worker';
|
||||
import type { WorkerExecutionOfferJournalOwnership } from '../ports/workerExecutionOfferJournalOwnership';
|
||||
import type {
|
||||
WorkerExecutionOfferStartupAuditResult,
|
||||
WorkerExecutionOfferStartupAuditor,
|
||||
} from './workerExecutionOfferStartupAuditor';
|
||||
import type {
|
||||
WorkerExecutionOfferStartupRecoveryResult,
|
||||
WorkerExecutionOfferStartupRecoverySupervisor,
|
||||
} from './workerExecutionOfferStartupRecoverySupervisor';
|
||||
|
||||
export type WorkerExecutionOfferInboxStartResult =
|
||||
| {
|
||||
status: 'ready' | 'reconciliation_required';
|
||||
audit: WorkerExecutionOfferStartupAuditResult;
|
||||
recovery?: WorkerExecutionOfferStartupRecoveryResult;
|
||||
}
|
||||
| {
|
||||
status: 'already_started';
|
||||
audit: WorkerExecutionOfferStartupAuditResult;
|
||||
recovery?: WorkerExecutionOfferStartupRecoveryResult;
|
||||
};
|
||||
|
||||
export type WorkerExecutionOfferInboxStopResult =
|
||||
| 'stopped'
|
||||
| 'not_started'
|
||||
| 'ownership_compromised';
|
||||
|
||||
export class WorkerExecutionOfferInboxStartupIncompleteError extends Error {
|
||||
constructor(readonly audit: WorkerExecutionOfferStartupAuditResult) {
|
||||
super('Worker execution offer inbox startup audit exhausted its budget');
|
||||
this.name = 'WorkerExecutionOfferInboxStartupIncompleteError';
|
||||
}
|
||||
}
|
||||
|
||||
export class WorkerExecutionOfferInboxRecoveryIncompleteError extends Error {
|
||||
constructor(readonly recovery: WorkerExecutionOfferStartupRecoveryResult) {
|
||||
super('Worker execution offer inbox startup recovery exhausted its budget');
|
||||
this.name = 'WorkerExecutionOfferInboxRecoveryIncompleteError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns the ordering contract: acquire the root, finish a bounded audit, then
|
||||
* keep ownership until shutdown. It does not start a delivery transport.
|
||||
*/
|
||||
export class WorkerExecutionOfferInboxLifecycle {
|
||||
private active = false;
|
||||
private lastAudit?: WorkerExecutionOfferStartupAuditResult;
|
||||
private lastRecovery?: WorkerExecutionOfferStartupRecoveryResult;
|
||||
|
||||
constructor(
|
||||
private readonly ownership: WorkerExecutionOfferJournalOwnership,
|
||||
private readonly auditor: Pick<WorkerExecutionOfferStartupAuditor, 'audit'>,
|
||||
private readonly recovery?: Pick<
|
||||
WorkerExecutionOfferStartupRecoverySupervisor,
|
||||
'recover'
|
||||
>,
|
||||
) {}
|
||||
|
||||
async start(
|
||||
currentSession: WorkerRecord,
|
||||
): Promise<WorkerExecutionOfferInboxStartResult> {
|
||||
if (this.active && this.lastAudit) {
|
||||
return {
|
||||
status: 'already_started',
|
||||
audit: this.lastAudit,
|
||||
...(this.lastRecovery === undefined
|
||||
? {}
|
||||
: { recovery: this.lastRecovery }),
|
||||
};
|
||||
}
|
||||
await this.ownership.acquireOwnership();
|
||||
try {
|
||||
let audit = await this.auditor.audit(currentSession);
|
||||
if (audit.status === 'scan_budget_exhausted') {
|
||||
throw new WorkerExecutionOfferInboxStartupIncompleteError(audit);
|
||||
}
|
||||
let recovery: WorkerExecutionOfferStartupRecoveryResult | undefined;
|
||||
if (this.recovery) {
|
||||
recovery = await this.recovery.recover(audit);
|
||||
if (recovery.status === 'action_budget_exhausted') {
|
||||
throw new WorkerExecutionOfferInboxRecoveryIncompleteError(recovery);
|
||||
}
|
||||
audit = await this.auditor.audit(currentSession);
|
||||
if (audit.status === 'scan_budget_exhausted') {
|
||||
throw new WorkerExecutionOfferInboxStartupIncompleteError(audit);
|
||||
}
|
||||
}
|
||||
this.active = true;
|
||||
this.lastAudit = audit;
|
||||
this.lastRecovery = recovery;
|
||||
return {
|
||||
status: audit.status,
|
||||
audit,
|
||||
...(recovery === undefined ? {} : { recovery }),
|
||||
};
|
||||
} catch (error) {
|
||||
await this.ownership.releaseOwnership().catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
currentAudit(): WorkerExecutionOfferStartupAuditResult | undefined {
|
||||
return this.lastAudit;
|
||||
}
|
||||
|
||||
async stop(): Promise<WorkerExecutionOfferInboxStopResult> {
|
||||
if (!this.active) return 'not_started';
|
||||
this.active = false;
|
||||
this.lastAudit = undefined;
|
||||
this.lastRecovery = undefined;
|
||||
const released = await this.ownership.releaseOwnership();
|
||||
return released === 'released' || released === 'not_owned'
|
||||
? 'stopped'
|
||||
: 'ownership_compromised';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
import type { WorkerExecutionOfferJournalState } from '../domain/workerExecutionOffer';
|
||||
import type { CompletionReceiptStore } from '../ports/completionReceiptStore';
|
||||
import type {
|
||||
WorkerExecutionOfferJournal,
|
||||
WorkerExecutionOfferJournalPage,
|
||||
} from '../ports/workerExecutionOfferJournal';
|
||||
|
||||
export const MIN_WORKER_OFFER_TERMINAL_RETENTION_MS = 60_000;
|
||||
export const MAX_WORKER_OFFER_TERMINAL_RETENTION_MS = 30 * 24 * 60 * 60_000;
|
||||
export const MAX_WORKER_OFFER_RETENTION_REMOVALS = 64;
|
||||
|
||||
export type WorkerExecutionOfferRetentionEntryOutcome =
|
||||
| 'removed'
|
||||
| 'already_absent'
|
||||
| 'receipt_cleanup_failed'
|
||||
| 'journal_remove_failed';
|
||||
|
||||
export interface WorkerExecutionOfferRetentionEntry {
|
||||
offerId: string;
|
||||
attemptId: string;
|
||||
state: Extract<
|
||||
WorkerExecutionOfferJournalState,
|
||||
'completion_acknowledged' | 'start_failure_acknowledged'
|
||||
>;
|
||||
outcome: WorkerExecutionOfferRetentionEntryOutcome;
|
||||
}
|
||||
|
||||
export interface WorkerExecutionOfferRetentionResult {
|
||||
status: 'complete' | 'page_complete' | 'removal_budget_exhausted';
|
||||
observedAtMs: number;
|
||||
recordsScanned: number;
|
||||
eligibleRecords: number;
|
||||
removalsAttempted: number;
|
||||
recordsRemoved: number;
|
||||
retainedRecords: number;
|
||||
failedRecords: number;
|
||||
entries: readonly WorkerExecutionOfferRetentionEntry[];
|
||||
nextAfterOfferId?: string;
|
||||
}
|
||||
|
||||
export interface WorkerExecutionOfferJournalRetentionOptions {
|
||||
completionRetentionMs: number;
|
||||
startFailureRetentionMs: number;
|
||||
pageSize?: number;
|
||||
maximumRemovals?: number;
|
||||
clock?: { now(): number };
|
||||
}
|
||||
|
||||
export class InvalidWorkerExecutionOfferRetentionPageError extends Error {
|
||||
constructor(message: string) {
|
||||
super(`Worker offer retention page is invalid: ${message}`);
|
||||
this.name = 'InvalidWorkerExecutionOfferRetentionPageError';
|
||||
}
|
||||
}
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** One bounded retention page; callers own cadence and cursor persistence. */
|
||||
export class WorkerExecutionOfferJournalRetentionService {
|
||||
private readonly completionRetentionMs: number;
|
||||
private readonly startFailureRetentionMs: number;
|
||||
private readonly pageSize: number;
|
||||
private readonly maximumRemovals: number;
|
||||
private readonly clock: { now(): number };
|
||||
|
||||
constructor(
|
||||
private readonly journal: Pick<
|
||||
WorkerExecutionOfferJournal,
|
||||
'list' | 'remove'
|
||||
>,
|
||||
private readonly receipts: Pick<CompletionReceiptStore, 'remove'>,
|
||||
options: WorkerExecutionOfferJournalRetentionOptions,
|
||||
) {
|
||||
this.completionRetentionMs = options.completionRetentionMs;
|
||||
this.startFailureRetentionMs = options.startFailureRetentionMs;
|
||||
this.pageSize = options.pageSize ?? 16;
|
||||
this.maximumRemovals = options.maximumRemovals ?? 8;
|
||||
this.clock = options.clock ?? Date;
|
||||
assertIntegerBetween(
|
||||
'completionRetentionMs',
|
||||
this.completionRetentionMs,
|
||||
MIN_WORKER_OFFER_TERMINAL_RETENTION_MS,
|
||||
MAX_WORKER_OFFER_TERMINAL_RETENTION_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'startFailureRetentionMs',
|
||||
this.startFailureRetentionMs,
|
||||
MIN_WORKER_OFFER_TERMINAL_RETENTION_MS,
|
||||
MAX_WORKER_OFFER_TERMINAL_RETENTION_MS,
|
||||
);
|
||||
assertIntegerBetween('pageSize', this.pageSize, 1, 64);
|
||||
assertIntegerBetween(
|
||||
'maximumRemovals',
|
||||
this.maximumRemovals,
|
||||
1,
|
||||
Math.min(this.pageSize, MAX_WORKER_OFFER_RETENTION_REMOVALS),
|
||||
);
|
||||
}
|
||||
|
||||
async sweep(
|
||||
options: {
|
||||
afterOfferId?: string;
|
||||
} = {},
|
||||
): Promise<WorkerExecutionOfferRetentionResult> {
|
||||
const observedAtMs = this.now();
|
||||
const page = await this.journal.list({
|
||||
...(options.afterOfferId === undefined
|
||||
? {}
|
||||
: { afterOfferId: options.afterOfferId }),
|
||||
limit: this.pageSize,
|
||||
});
|
||||
this.assertPage(page, options.afterOfferId);
|
||||
|
||||
const entries: WorkerExecutionOfferRetentionEntry[] = [];
|
||||
let recordsScanned = 0;
|
||||
let eligibleRecords = 0;
|
||||
let removalsAttempted = 0;
|
||||
let recordsRemoved = 0;
|
||||
let retainedRecords = 0;
|
||||
let failedRecords = 0;
|
||||
let lastProcessedOfferId = options.afterOfferId;
|
||||
|
||||
for (const record of page.records) {
|
||||
if (
|
||||
record.state !== 'completion_acknowledged' &&
|
||||
record.state !== 'start_failure_acknowledged'
|
||||
) {
|
||||
recordsScanned += 1;
|
||||
retainedRecords += 1;
|
||||
lastProcessedOfferId = record.offer.offerId;
|
||||
continue;
|
||||
}
|
||||
const settledAtMs =
|
||||
record.state === 'completion_acknowledged'
|
||||
? record.completionAcknowledgedAtMs!
|
||||
: record.updatedAtMs;
|
||||
const retentionMs =
|
||||
record.state === 'completion_acknowledged'
|
||||
? this.completionRetentionMs
|
||||
: this.startFailureRetentionMs;
|
||||
const due =
|
||||
settledAtMs <= observedAtMs &&
|
||||
observedAtMs - settledAtMs >= retentionMs;
|
||||
if (!due) {
|
||||
recordsScanned += 1;
|
||||
retainedRecords += 1;
|
||||
lastProcessedOfferId = record.offer.offerId;
|
||||
continue;
|
||||
}
|
||||
eligibleRecords += 1;
|
||||
if (removalsAttempted >= this.maximumRemovals) {
|
||||
return {
|
||||
status: 'removal_budget_exhausted',
|
||||
observedAtMs,
|
||||
recordsScanned,
|
||||
eligibleRecords,
|
||||
removalsAttempted,
|
||||
recordsRemoved,
|
||||
retainedRecords,
|
||||
failedRecords,
|
||||
entries,
|
||||
...(lastProcessedOfferId === undefined
|
||||
? {}
|
||||
: { nextAfterOfferId: lastProcessedOfferId }),
|
||||
};
|
||||
}
|
||||
recordsScanned += 1;
|
||||
removalsAttempted += 1;
|
||||
const offerId = record.offer.offerId;
|
||||
const attemptId = record.offer.candidate.attemptId;
|
||||
if (record.state === 'completion_acknowledged') {
|
||||
try {
|
||||
await this.receipts.remove(attemptId);
|
||||
} catch {
|
||||
failedRecords += 1;
|
||||
entries.push({
|
||||
offerId,
|
||||
attemptId,
|
||||
state: record.state,
|
||||
outcome: 'receipt_cleanup_failed',
|
||||
});
|
||||
lastProcessedOfferId = offerId;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
try {
|
||||
const removed = await this.journal.remove(offerId, record.revision);
|
||||
if (removed) recordsRemoved += 1;
|
||||
entries.push({
|
||||
offerId,
|
||||
attemptId,
|
||||
state: record.state,
|
||||
outcome: removed ? 'removed' : 'already_absent',
|
||||
});
|
||||
} catch {
|
||||
failedRecords += 1;
|
||||
entries.push({
|
||||
offerId,
|
||||
attemptId,
|
||||
state: record.state,
|
||||
outcome: 'journal_remove_failed',
|
||||
});
|
||||
}
|
||||
lastProcessedOfferId = offerId;
|
||||
}
|
||||
|
||||
return {
|
||||
status:
|
||||
page.nextAfterOfferId === undefined ? 'complete' : 'page_complete',
|
||||
observedAtMs,
|
||||
recordsScanned,
|
||||
eligibleRecords,
|
||||
removalsAttempted,
|
||||
recordsRemoved,
|
||||
retainedRecords,
|
||||
failedRecords,
|
||||
entries,
|
||||
...(page.nextAfterOfferId === undefined
|
||||
? {}
|
||||
: { nextAfterOfferId: page.nextAfterOfferId }),
|
||||
};
|
||||
}
|
||||
|
||||
private assertPage(
|
||||
page: WorkerExecutionOfferJournalPage,
|
||||
afterOfferId: string | undefined,
|
||||
): void {
|
||||
if (page.records.length > this.pageSize) {
|
||||
throw new InvalidWorkerExecutionOfferRetentionPageError(
|
||||
'record count exceeds pageSize',
|
||||
);
|
||||
}
|
||||
let previous = afterOfferId;
|
||||
for (const record of page.records) {
|
||||
if (previous !== undefined && record.offer.offerId <= previous) {
|
||||
throw new InvalidWorkerExecutionOfferRetentionPageError(
|
||||
'offer cursor did not advance',
|
||||
);
|
||||
}
|
||||
previous = record.offer.offerId;
|
||||
}
|
||||
if (
|
||||
page.nextAfterOfferId !== undefined &&
|
||||
(page.records.length !== this.pageSize ||
|
||||
page.records.length === 0 ||
|
||||
page.nextAfterOfferId !==
|
||||
page.records[page.records.length - 1].offer.offerId)
|
||||
) {
|
||||
throw new InvalidWorkerExecutionOfferRetentionPageError(
|
||||
'resume cursor is inconsistent',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock.now();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new TypeError(
|
||||
'Worker offer retention clock returned an invalid time',
|
||||
);
|
||||
}
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,510 @@
|
||||
import type { ExecutionHandle, ExecutionStopReason } from '../domain/execution';
|
||||
import {
|
||||
WorkerExecutionOfferConflictError,
|
||||
assertClaimedExecutionOffer,
|
||||
assertSameWorkerExecutionOffer,
|
||||
cloneClaimedExecutionOffer,
|
||||
cloneWorkerExecutionOfferJournalRecord,
|
||||
createWorkerExecutionOfferJournalRecord,
|
||||
mergeWorkerExecutionOffer,
|
||||
workerExecutionHandleMetadata,
|
||||
type WorkerExecutionOfferJournalRecord,
|
||||
} from '../domain/workerExecutionOffer';
|
||||
import {
|
||||
RunDispatchLeaseFenceRejectedError,
|
||||
type RunDispatchLeaseRecord,
|
||||
} from '../domain/runDispatchLease';
|
||||
import type { ClaimedExecutionOffer } from '../domain/runDispatchOffer';
|
||||
import type { WorkerRecord } from '../domain/worker';
|
||||
import { createWorkerExecutionCompletionReceiptAuthentication } from '../domain/workerExecutionCompletionReceiptAuthentication';
|
||||
import type { Executor } from '../ports/executor';
|
||||
import type { WorkerExecutionContextFactory } from '../ports/workerExecutionContextFactory';
|
||||
import type { WorkerExecutionOfferJournal } from '../ports/workerExecutionOfferJournal';
|
||||
import type { WorkerRemoteRunActivationClient } from '../ports/workerRemoteRunActivationClient';
|
||||
import type { WorkerRunLeaseTracker } from '../ports/workerRunLeaseTracker';
|
||||
import type {
|
||||
RemoteRunActivationResult,
|
||||
RemoteRunLeaseFence,
|
||||
} from './remoteRunActivationService';
|
||||
|
||||
export const MAX_WORKER_OFFER_ACTIVATION_RETRIES = 4;
|
||||
|
||||
export type WorkerExecutionOfferReceiveResult =
|
||||
| {
|
||||
status: 'running' | 'already_running';
|
||||
offerId: string;
|
||||
executorHandle: string;
|
||||
}
|
||||
| {
|
||||
status: 'start_failed' | 'already_failed';
|
||||
offerId: string;
|
||||
}
|
||||
| {
|
||||
status: 'already_completed';
|
||||
offerId: string;
|
||||
}
|
||||
| {
|
||||
status: 'recovery_required';
|
||||
offerId: string;
|
||||
reason:
|
||||
| 'launch_outcome_unknown'
|
||||
| 'control_plane_already_running'
|
||||
| 'control_plane_terminal'
|
||||
| 'lease_lost_local_execution_stopped'
|
||||
| 'lease_lost_local_execution_unverified';
|
||||
};
|
||||
|
||||
export interface WorkerExecutionOfferReceiverOptions {
|
||||
currentSession(): WorkerRecord | undefined;
|
||||
clock?: { now(): number };
|
||||
activationRetries?: number;
|
||||
acceptedExecutorTypes?: readonly string[];
|
||||
}
|
||||
|
||||
export class WorkerExecutionOfferTargetError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'WorkerExecutionOfferTargetError';
|
||||
}
|
||||
}
|
||||
|
||||
export class WorkerExecutionOfferExpiredError extends Error {
|
||||
constructor(readonly offerId: string) {
|
||||
super(`Worker execution offer ${offerId} has expired`);
|
||||
this.name = 'WorkerExecutionOfferExpiredError';
|
||||
}
|
||||
}
|
||||
|
||||
interface InFlightOffer {
|
||||
offer: ClaimedExecutionOffer;
|
||||
operation: Promise<WorkerExecutionOfferReceiveResult>;
|
||||
}
|
||||
|
||||
function sameLeaseAuthority(
|
||||
left: RunDispatchLeaseRecord,
|
||||
right: RunDispatchLeaseRecord,
|
||||
): boolean {
|
||||
return (
|
||||
left.attemptId === right.attemptId &&
|
||||
left.runId === right.runId &&
|
||||
left.workerId === right.workerId &&
|
||||
left.workerSessionId === right.workerSessionId &&
|
||||
left.workerGeneration === right.workerGeneration &&
|
||||
left.leaseGeneration === right.leaseGeneration &&
|
||||
left.leaseToken === right.leaseToken
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Transport-neutral Worker inbox. The transport must authenticate the control
|
||||
* plane before calling receive(); this service independently fences the target
|
||||
* Worker session and writes durable state before any process spawn.
|
||||
*/
|
||||
export class WorkerExecutionOfferReceiver {
|
||||
private readonly currentSessionProvider: WorkerExecutionOfferReceiverOptions['currentSession'];
|
||||
private readonly clock: { now(): number };
|
||||
private readonly activationRetries: number;
|
||||
private readonly acceptedExecutorTypes: ReadonlySet<string>;
|
||||
private readonly inFlight = new Map<string, InFlightOffer>();
|
||||
|
||||
constructor(
|
||||
private readonly journal: WorkerExecutionOfferJournal,
|
||||
private readonly activation: WorkerRemoteRunActivationClient,
|
||||
private readonly executor: Executor,
|
||||
private readonly leaseTracker: WorkerRunLeaseTracker,
|
||||
private readonly contexts: WorkerExecutionContextFactory,
|
||||
options: WorkerExecutionOfferReceiverOptions,
|
||||
) {
|
||||
this.currentSessionProvider = options.currentSession;
|
||||
this.clock = options.clock ?? Date;
|
||||
this.activationRetries = options.activationRetries ?? 2;
|
||||
if (
|
||||
!Number.isSafeInteger(this.activationRetries) ||
|
||||
this.activationRetries < 1 ||
|
||||
this.activationRetries > MAX_WORKER_OFFER_ACTIVATION_RETRIES
|
||||
) {
|
||||
throw new RangeError(
|
||||
`activationRetries must be between 1 and ${MAX_WORKER_OFFER_ACTIVATION_RETRIES}`,
|
||||
);
|
||||
}
|
||||
const accepted = options.acceptedExecutorTypes ?? ['remote_worker'];
|
||||
if (
|
||||
accepted.length < 1 ||
|
||||
accepted.length > 16 ||
|
||||
new Set(accepted).size !== accepted.length ||
|
||||
accepted.some(
|
||||
(value) =>
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
value.length > 64 ||
|
||||
/[\u0000-\u001f\u007f]/.test(value),
|
||||
)
|
||||
) {
|
||||
throw new TypeError('acceptedExecutorTypes is invalid');
|
||||
}
|
||||
this.acceptedExecutorTypes = new Set(accepted);
|
||||
}
|
||||
|
||||
receive(
|
||||
delivered: ClaimedExecutionOffer,
|
||||
): Promise<WorkerExecutionOfferReceiveResult> {
|
||||
assertClaimedExecutionOffer(delivered);
|
||||
const offer = cloneClaimedExecutionOffer(delivered);
|
||||
const active = this.inFlight.get(offer.offerId);
|
||||
if (active) {
|
||||
assertSameWorkerExecutionOffer(active.offer, offer);
|
||||
return active.operation;
|
||||
}
|
||||
const operation = this.process(offer).finally(() => {
|
||||
if (this.inFlight.get(offer.offerId)?.operation === operation) {
|
||||
this.inFlight.delete(offer.offerId);
|
||||
}
|
||||
});
|
||||
this.inFlight.set(offer.offerId, { offer, operation });
|
||||
return operation;
|
||||
}
|
||||
|
||||
private async process(
|
||||
delivered: ClaimedExecutionOffer,
|
||||
): Promise<WorkerExecutionOfferReceiveResult> {
|
||||
const persisted = await this.journal.read(delivered.offerId);
|
||||
if (persisted) {
|
||||
assertSameWorkerExecutionOffer(persisted.offer, delivered);
|
||||
if (persisted.state === 'completion_acknowledged') {
|
||||
return {
|
||||
status: 'already_completed',
|
||||
offerId: persisted.offer.offerId,
|
||||
};
|
||||
}
|
||||
}
|
||||
this.assertTarget(delivered);
|
||||
const initial = createWorkerExecutionOfferJournalRecord(
|
||||
delivered,
|
||||
this.now(),
|
||||
);
|
||||
await this.journal.create(initial);
|
||||
let record = await this.journal.read(delivered.offerId);
|
||||
if (!record) {
|
||||
throw new Error('Worker offer journal lost an entry after create');
|
||||
}
|
||||
assertSameWorkerExecutionOffer(record.offer, delivered);
|
||||
const merged = mergeWorkerExecutionOffer(record.offer, delivered);
|
||||
if (merged.lease.version !== record.offer.lease.version) {
|
||||
record = await this.replace(record, { offer: merged });
|
||||
}
|
||||
|
||||
if (record.state === 'completion_acknowledged') {
|
||||
return { status: 'already_completed', offerId: record.offer.offerId };
|
||||
}
|
||||
if (record.state === 'running_acknowledged') {
|
||||
return {
|
||||
status: 'already_running',
|
||||
offerId: record.offer.offerId,
|
||||
executorHandle: record.executorHandle!,
|
||||
};
|
||||
}
|
||||
if (record.state === 'start_failure_acknowledged') {
|
||||
return { status: 'already_failed', offerId: record.offer.offerId };
|
||||
}
|
||||
if (record.state === 'recovery_required') {
|
||||
return {
|
||||
status: 'recovery_required',
|
||||
offerId: record.offer.offerId,
|
||||
reason: record.recoveryReason!,
|
||||
};
|
||||
}
|
||||
if (record.state === 'launching') {
|
||||
record = await this.replace(record, {
|
||||
state: 'recovery_required',
|
||||
recoveryReason: 'launch_outcome_unknown',
|
||||
});
|
||||
return this.recoveryResult(record);
|
||||
}
|
||||
|
||||
if (record.state === 'accepted') {
|
||||
const starting = await this.activation.acknowledgeStarting(
|
||||
this.fence(record.offer),
|
||||
);
|
||||
if (starting.status === 'already_running') {
|
||||
record = await this.replace(record, {
|
||||
state: 'recovery_required',
|
||||
recoveryReason: 'control_plane_already_running',
|
||||
});
|
||||
return this.recoveryResult(record);
|
||||
}
|
||||
if (starting.status === 'already_terminal') {
|
||||
record = await this.replace(record, {
|
||||
state: 'recovery_required',
|
||||
recoveryReason: 'control_plane_terminal',
|
||||
});
|
||||
return this.recoveryResult(record);
|
||||
}
|
||||
record = await this.replace(record, {
|
||||
state: 'starting_acknowledged',
|
||||
offer: {
|
||||
...record.offer,
|
||||
lease: { ...starting.lease },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (record.state === 'start_failed') {
|
||||
return this.reportStartFailure(record, true);
|
||||
}
|
||||
|
||||
if (record.state === 'starting_acknowledged') {
|
||||
const tracked = this.trackLatest(
|
||||
record.offer.lease,
|
||||
record.offer.offerId,
|
||||
);
|
||||
if (tracked.version !== record.offer.lease.version) {
|
||||
record = await this.replace(record, {
|
||||
offer: { ...record.offer, lease: tracked },
|
||||
});
|
||||
}
|
||||
let prepared;
|
||||
let receiptAuthentication;
|
||||
try {
|
||||
prepared = await this.contexts.prepare(
|
||||
cloneClaimedExecutionOffer(record.offer),
|
||||
);
|
||||
receiptAuthentication =
|
||||
createWorkerExecutionCompletionReceiptAuthentication(
|
||||
prepared.context.completionCallback,
|
||||
);
|
||||
} catch {
|
||||
record = await this.replace(record, { state: 'start_failed' });
|
||||
return this.reportStartFailure(record, false);
|
||||
}
|
||||
record = await this.replace(record, {
|
||||
state: 'launching',
|
||||
...(receiptAuthentication === undefined
|
||||
? {}
|
||||
: {
|
||||
completionReceiptCallbackSequence:
|
||||
receiptAuthentication.callbackSequence,
|
||||
completionReceiptTokenDigest: receiptAuthentication.tokenDigest,
|
||||
}),
|
||||
});
|
||||
let handle: ExecutionHandle;
|
||||
try {
|
||||
handle = await this.executor.start(
|
||||
record.offer.executionSpec,
|
||||
prepared.context,
|
||||
);
|
||||
} catch {
|
||||
record = await this.replace(record, { state: 'start_failed' });
|
||||
return this.reportStartFailure(record, false);
|
||||
}
|
||||
if (
|
||||
handle.runId !== record.offer.candidate.runId ||
|
||||
handle.attemptId !== record.offer.candidate.attemptId ||
|
||||
handle.executorType !== this.executor.type
|
||||
) {
|
||||
await this.compensateInvalidHandle(handle);
|
||||
record = await this.replace(record, {
|
||||
state: 'recovery_required',
|
||||
recoveryReason: 'launch_outcome_unknown',
|
||||
});
|
||||
return this.recoveryResult(record);
|
||||
}
|
||||
const metadata = workerExecutionHandleMetadata(handle);
|
||||
record = await this.replace(record, {
|
||||
state: 'started',
|
||||
...metadata,
|
||||
...(prepared.logArtifactId === undefined
|
||||
? {}
|
||||
: { logArtifactId: prepared.logArtifactId }),
|
||||
});
|
||||
}
|
||||
|
||||
if (record.state !== 'started') {
|
||||
throw new WorkerExecutionOfferConflictError(record.offer.offerId);
|
||||
}
|
||||
const startedRecord = record;
|
||||
const running = await this.withCurrentLease(startedRecord, (lease) =>
|
||||
this.activation.acknowledgeRunning({
|
||||
...this.fence({ ...startedRecord.offer, lease }),
|
||||
startedAtMs: startedRecord.executorStartedAtMs!,
|
||||
executorHandle: startedRecord.executorHandle!,
|
||||
...(startedRecord.logArtifactId === undefined
|
||||
? {}
|
||||
: { logArtifactId: startedRecord.logArtifactId }),
|
||||
}),
|
||||
);
|
||||
if (running.status === 'already_terminal') {
|
||||
record = await this.replace(startedRecord, {
|
||||
state: 'recovery_required',
|
||||
recoveryReason: 'control_plane_terminal',
|
||||
});
|
||||
return this.recoveryResult(record);
|
||||
}
|
||||
record = await this.replace(startedRecord, {
|
||||
state: 'running_acknowledged',
|
||||
offer: { ...record.offer, lease: { ...running.lease } },
|
||||
});
|
||||
return {
|
||||
status:
|
||||
running.status === 'already_running' ? 'already_running' : 'running',
|
||||
offerId: record.offer.offerId,
|
||||
executorHandle: record.executorHandle!,
|
||||
};
|
||||
}
|
||||
|
||||
private async reportStartFailure(
|
||||
record: WorkerExecutionOfferJournalRecord,
|
||||
replay: boolean,
|
||||
): Promise<WorkerExecutionOfferReceiveResult> {
|
||||
const failure = await this.withCurrentLease(record, (lease) =>
|
||||
this.activation.failStart(this.fence({ ...record.offer, lease })),
|
||||
);
|
||||
if (failure.status === 'already_running') {
|
||||
const recovery = await this.replace(record, {
|
||||
state: 'recovery_required',
|
||||
recoveryReason: 'control_plane_already_running',
|
||||
});
|
||||
return this.recoveryResult(recovery);
|
||||
}
|
||||
this.leaseTracker.untrack(record.offer.candidate.attemptId);
|
||||
await this.replace(record, { state: 'start_failure_acknowledged' });
|
||||
return {
|
||||
status: replay ? 'already_failed' : 'start_failed',
|
||||
offerId: record.offer.offerId,
|
||||
};
|
||||
}
|
||||
|
||||
private async withCurrentLease(
|
||||
record: WorkerExecutionOfferJournalRecord,
|
||||
operation: (
|
||||
lease: RunDispatchLeaseRecord,
|
||||
) => Promise<RemoteRunActivationResult>,
|
||||
): Promise<RemoteRunActivationResult> {
|
||||
let lastError: unknown;
|
||||
for (let attempt = 0; attempt < this.activationRetries; attempt += 1) {
|
||||
const lease = this.trackLatest(record.offer.lease, record.offer.offerId);
|
||||
try {
|
||||
return await operation(lease);
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
if (
|
||||
!(error instanceof RunDispatchLeaseFenceRejectedError) ||
|
||||
error.reason !== 'version_mismatch'
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
private trackLatest(
|
||||
lease: RunDispatchLeaseRecord,
|
||||
offerId: string,
|
||||
): RunDispatchLeaseRecord {
|
||||
const current = this.leaseTracker
|
||||
.leases()
|
||||
.find((candidate) => candidate.attemptId === lease.attemptId);
|
||||
if (current && !sameLeaseAuthority(current, lease)) {
|
||||
throw new WorkerExecutionOfferConflictError(offerId);
|
||||
}
|
||||
const selected =
|
||||
current && current.version > lease.version ? current : lease;
|
||||
this.leaseTracker.track(selected);
|
||||
return { ...selected };
|
||||
}
|
||||
|
||||
private fence(offer: ClaimedExecutionOffer): RemoteRunLeaseFence {
|
||||
const lease = offer.lease;
|
||||
return {
|
||||
runId: offer.candidate.runId,
|
||||
attemptId: offer.candidate.attemptId,
|
||||
workerId: lease.workerId,
|
||||
workerSessionId: lease.workerSessionId,
|
||||
workerGeneration: lease.workerGeneration,
|
||||
leaseGeneration: lease.leaseGeneration,
|
||||
leaseToken: lease.leaseToken,
|
||||
expectedLeaseVersion: lease.version,
|
||||
executorType: offer.candidate.executorType,
|
||||
};
|
||||
}
|
||||
|
||||
private assertTarget(offer: ClaimedExecutionOffer): void {
|
||||
const nowMs = this.now();
|
||||
const session = this.currentSessionProvider();
|
||||
if (
|
||||
!session ||
|
||||
session.id !== offer.worker.id ||
|
||||
session.sessionId !== offer.worker.sessionId ||
|
||||
session.generation !== offer.worker.generation
|
||||
) {
|
||||
throw new WorkerExecutionOfferTargetError(
|
||||
'Execution offer does not target the current Worker session',
|
||||
);
|
||||
}
|
||||
if (
|
||||
session.status === 'offline' ||
|
||||
(session.status === 'draining' && offer.deliveryKind === 'new_claim') ||
|
||||
session.leaseExpiresAtMs <= nowMs
|
||||
) {
|
||||
throw new WorkerExecutionOfferTargetError(
|
||||
'Current Worker session cannot accept this execution offer',
|
||||
);
|
||||
}
|
||||
if (offer.lease.expiresAtMs <= nowMs) {
|
||||
throw new WorkerExecutionOfferExpiredError(offer.offerId);
|
||||
}
|
||||
if (
|
||||
!this.acceptedExecutorTypes.has(offer.candidate.executorType) ||
|
||||
!session.capabilities.executors.includes(offer.candidate.executorType)
|
||||
) {
|
||||
throw new WorkerExecutionOfferTargetError(
|
||||
'Worker does not advertise the requested execution path',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async replace(
|
||||
previous: WorkerExecutionOfferJournalRecord,
|
||||
patch: Partial<WorkerExecutionOfferJournalRecord>,
|
||||
): Promise<WorkerExecutionOfferJournalRecord> {
|
||||
const updated = cloneWorkerExecutionOfferJournalRecord({
|
||||
...previous,
|
||||
...patch,
|
||||
schemaVersion: 1,
|
||||
revision: previous.revision + 1,
|
||||
updatedAtMs: Math.max(this.now(), previous.updatedAtMs),
|
||||
});
|
||||
await this.journal.replace(updated, previous.revision);
|
||||
return updated;
|
||||
}
|
||||
|
||||
private recoveryResult(
|
||||
record: WorkerExecutionOfferJournalRecord,
|
||||
): WorkerExecutionOfferReceiveResult {
|
||||
return {
|
||||
status: 'recovery_required',
|
||||
offerId: record.offer.offerId,
|
||||
reason: record.recoveryReason!,
|
||||
};
|
||||
}
|
||||
|
||||
private async compensateInvalidHandle(
|
||||
handle: ExecutionHandle,
|
||||
): Promise<void> {
|
||||
const reason: ExecutionStopReason = {
|
||||
kind: 'reconcile',
|
||||
requestedAtMs: this.now(),
|
||||
};
|
||||
await this.executor.stop(handle, reason).catch(() => undefined);
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock.now();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new TypeError(
|
||||
'Worker offer receiver clock returned an invalid time',
|
||||
);
|
||||
}
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
import {
|
||||
cloneWorkerExecutionOfferJournalRecord,
|
||||
type WorkerExecutionOfferJournalRecord,
|
||||
} from '../domain/workerExecutionOffer';
|
||||
import type { WorkerRecord } from '../domain/worker';
|
||||
import type { CompletionReceiptStore } from '../ports/completionReceiptStore';
|
||||
import type { WorkerExecutionOfferJournal } from '../ports/workerExecutionOfferJournal';
|
||||
import type { WorkerRemoteRunActivationClient } from '../ports/workerRemoteRunActivationClient';
|
||||
import type { WorkerRemoteRunCompletionClient } from '../ports/workerRemoteRunCompletionClient';
|
||||
import type { WorkerExecutionOfferRecoveryResult } from './workerExecutionOfferRecoveryReconciler';
|
||||
import { WorkerExecutionOfferRecoveryReconciler } from './workerExecutionOfferRecoveryReconciler';
|
||||
|
||||
export type WorkerExecutionOfferRecoveryActionStatus =
|
||||
| 'not_found'
|
||||
| 'session_unavailable'
|
||||
| 'deferred'
|
||||
| 'running_acknowledged'
|
||||
| 'already_running'
|
||||
| 'control_plane_terminal'
|
||||
| 'completion_acknowledged'
|
||||
| 'already_completed';
|
||||
|
||||
export type WorkerCompletionReceiptCleanup =
|
||||
| 'removed'
|
||||
| 'already_absent'
|
||||
| 'pending';
|
||||
|
||||
export interface WorkerExecutionOfferRecoveryActionResult {
|
||||
offerId: string;
|
||||
status: WorkerExecutionOfferRecoveryActionStatus;
|
||||
evidence?: WorkerExecutionOfferRecoveryResult;
|
||||
receiptCleanup?: WorkerCompletionReceiptCleanup;
|
||||
}
|
||||
|
||||
export interface WorkerExecutionOfferRecoveryCoordinatorOptions {
|
||||
currentSession(): WorkerRecord | undefined;
|
||||
clock?: { now(): number };
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies only evidence-backed recovery mutations. Control-plane completion is
|
||||
* durable before the local journal becomes terminal; the receipt is removed
|
||||
* only after that terminal journal write succeeds.
|
||||
*/
|
||||
export class WorkerExecutionOfferRecoveryCoordinator {
|
||||
private readonly currentSessionProvider: WorkerExecutionOfferRecoveryCoordinatorOptions['currentSession'];
|
||||
private readonly clock: { now(): number };
|
||||
private readonly inFlight = new Map<
|
||||
string,
|
||||
Promise<WorkerExecutionOfferRecoveryActionResult>
|
||||
>();
|
||||
|
||||
constructor(
|
||||
private readonly journal: Pick<
|
||||
WorkerExecutionOfferJournal,
|
||||
'read' | 'replace'
|
||||
>,
|
||||
private readonly reconciler: WorkerExecutionOfferRecoveryReconciler,
|
||||
private readonly completion: WorkerRemoteRunCompletionClient,
|
||||
private readonly activation: Pick<
|
||||
WorkerRemoteRunActivationClient,
|
||||
'acknowledgeRunning'
|
||||
>,
|
||||
private readonly receipts: Pick<CompletionReceiptStore, 'remove'>,
|
||||
options: WorkerExecutionOfferRecoveryCoordinatorOptions,
|
||||
) {
|
||||
this.currentSessionProvider = options.currentSession;
|
||||
this.clock = options.clock ?? Date;
|
||||
}
|
||||
|
||||
recover(offerId: string): Promise<WorkerExecutionOfferRecoveryActionResult> {
|
||||
const active = this.inFlight.get(offerId);
|
||||
if (active) return active;
|
||||
const operation = this.process(offerId).finally(() => {
|
||||
if (this.inFlight.get(offerId) === operation) {
|
||||
this.inFlight.delete(offerId);
|
||||
}
|
||||
});
|
||||
this.inFlight.set(offerId, operation);
|
||||
return operation;
|
||||
}
|
||||
|
||||
private async process(
|
||||
offerId: string,
|
||||
): Promise<WorkerExecutionOfferRecoveryActionResult> {
|
||||
let record = await this.journal.read(offerId);
|
||||
if (!record) return { offerId, status: 'not_found' };
|
||||
if (record.state === 'completion_acknowledged') {
|
||||
return {
|
||||
offerId,
|
||||
status: 'already_completed',
|
||||
receiptCleanup: await this.cleanupReceipt(
|
||||
record.offer.candidate.attemptId,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
const currentSession = this.currentSessionProvider();
|
||||
if (!currentSession) return { offerId, status: 'session_unavailable' };
|
||||
const evidence = await this.reconciler.reconcile(record, currentSession);
|
||||
|
||||
if (
|
||||
evidence.finding === 'completion_observed' &&
|
||||
evidence.completionSubmission === 'ready' &&
|
||||
evidence.completion
|
||||
) {
|
||||
await this.completion.complete({
|
||||
runId: record.offer.candidate.runId,
|
||||
attemptId: record.offer.candidate.attemptId,
|
||||
callbackSequence: evidence.completion.callbackSequence,
|
||||
result: {
|
||||
outcome: evidence.completion.outcome,
|
||||
startedAtMs: evidence.completion.startedAtMs,
|
||||
finishedAtMs: evidence.completion.finishedAtMs,
|
||||
exitCode: evidence.completion.exitCode,
|
||||
},
|
||||
executorType: record.offer.candidate.executorType,
|
||||
workerId: record.offer.lease.workerId,
|
||||
workerSessionId: record.offer.lease.workerSessionId,
|
||||
workerGeneration: record.offer.lease.workerGeneration,
|
||||
leaseGeneration: record.offer.lease.leaseGeneration,
|
||||
leaseToken: record.offer.lease.leaseToken,
|
||||
expectedLeaseVersion: record.offer.lease.version,
|
||||
});
|
||||
record = await this.markCompletionAcknowledged(record);
|
||||
return {
|
||||
offerId,
|
||||
status: 'completion_acknowledged',
|
||||
evidence,
|
||||
receiptCleanup: await this.cleanupReceipt(
|
||||
record.offer.candidate.attemptId,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
evidence.finding === 'execution_running' &&
|
||||
evidence.authority === 'current' &&
|
||||
record.state === 'started'
|
||||
) {
|
||||
const lease = record.offer.lease;
|
||||
const activated = await this.activation.acknowledgeRunning({
|
||||
runId: record.offer.candidate.runId,
|
||||
attemptId: record.offer.candidate.attemptId,
|
||||
workerId: lease.workerId,
|
||||
workerSessionId: lease.workerSessionId,
|
||||
workerGeneration: lease.workerGeneration,
|
||||
leaseGeneration: lease.leaseGeneration,
|
||||
leaseToken: lease.leaseToken,
|
||||
expectedLeaseVersion: lease.version,
|
||||
executorType: record.offer.candidate.executorType,
|
||||
startedAtMs: record.executorStartedAtMs!,
|
||||
executorHandle: record.executorHandle!,
|
||||
...(record.logArtifactId === undefined
|
||||
? {}
|
||||
: { logArtifactId: record.logArtifactId }),
|
||||
});
|
||||
if (activated.status === 'already_terminal') {
|
||||
await this.replace(record, {
|
||||
state: 'recovery_required',
|
||||
recoveryReason: 'control_plane_terminal',
|
||||
});
|
||||
return { offerId, status: 'control_plane_terminal', evidence };
|
||||
}
|
||||
await this.replace(record, {
|
||||
state: 'running_acknowledged',
|
||||
offer: { ...record.offer, lease: { ...activated.lease } },
|
||||
});
|
||||
return {
|
||||
offerId,
|
||||
status:
|
||||
activated.status === 'already_running'
|
||||
? 'already_running'
|
||||
: 'running_acknowledged',
|
||||
evidence,
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
evidence.finding === 'execution_running' &&
|
||||
evidence.authority === 'current' &&
|
||||
record.state === 'running_acknowledged'
|
||||
) {
|
||||
return { offerId, status: 'already_running', evidence };
|
||||
}
|
||||
return { offerId, status: 'deferred', evidence };
|
||||
}
|
||||
|
||||
private async markCompletionAcknowledged(
|
||||
record: WorkerExecutionOfferJournalRecord,
|
||||
): Promise<WorkerExecutionOfferJournalRecord> {
|
||||
try {
|
||||
return await this.replace(record, {
|
||||
state: 'completion_acknowledged',
|
||||
recoveryReason: undefined,
|
||||
});
|
||||
} catch (error) {
|
||||
const current = await this.journal.read(record.offer.offerId);
|
||||
if (current?.state === 'completion_acknowledged') return current;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async replace(
|
||||
previous: WorkerExecutionOfferJournalRecord,
|
||||
patch: Partial<WorkerExecutionOfferJournalRecord>,
|
||||
): Promise<WorkerExecutionOfferJournalRecord> {
|
||||
const updatedAtMs = Math.max(this.now(), previous.updatedAtMs);
|
||||
const updated = cloneWorkerExecutionOfferJournalRecord({
|
||||
...previous,
|
||||
...patch,
|
||||
schemaVersion: 1,
|
||||
revision: previous.revision + 1,
|
||||
updatedAtMs,
|
||||
...(patch.state === 'completion_acknowledged'
|
||||
? { completionAcknowledgedAtMs: updatedAtMs }
|
||||
: {}),
|
||||
});
|
||||
await this.journal.replace(updated, previous.revision);
|
||||
return updated;
|
||||
}
|
||||
|
||||
private async cleanupReceipt(
|
||||
attemptId: string,
|
||||
): Promise<WorkerCompletionReceiptCleanup> {
|
||||
try {
|
||||
return (await this.receipts.remove(attemptId))
|
||||
? 'removed'
|
||||
: 'already_absent';
|
||||
} catch {
|
||||
return 'pending';
|
||||
}
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock.now();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new TypeError(
|
||||
'Worker offer recovery coordinator clock returned an invalid time',
|
||||
);
|
||||
}
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
import type { CompletionReceipt } from '../domain/completionReceipt';
|
||||
import type { WorkerRecord } from '../domain/worker';
|
||||
import {
|
||||
cloneWorkerExecutionOfferJournalRecord,
|
||||
type WorkerExecutionOfferJournalRecord,
|
||||
} from '../domain/workerExecutionOffer';
|
||||
import type { CompletionReceiptStore } from '../ports/completionReceiptStore';
|
||||
import type { PersistedExecutionInspector } from '../ports/persistedExecutionInspector';
|
||||
import type { WorkerExecutionCompletionReceiptAuthenticator } from '../ports/workerExecutionCompletionReceiptAuthenticator';
|
||||
|
||||
export type WorkerExecutionRecoveryAuthority =
|
||||
| 'current'
|
||||
| 'session_fenced'
|
||||
| 'worker_offline'
|
||||
| 'worker_session_expired'
|
||||
| 'run_lease_expired';
|
||||
|
||||
export type WorkerExecutionRecoveryFinding =
|
||||
| 'no_execution_expected'
|
||||
| 'completion_observed'
|
||||
| 'completion_receipt_conflict'
|
||||
| 'completion_receipt_unavailable'
|
||||
| 'execution_running'
|
||||
| 'execution_exited_without_receipt'
|
||||
| 'launch_outcome_unknown'
|
||||
| 'execution_identity_mismatch'
|
||||
| 'execution_handle_invalid'
|
||||
| 'execution_probe_unsupported'
|
||||
| 'execution_probe_unavailable';
|
||||
|
||||
export type WorkerExecutionCompletionSubmission =
|
||||
| 'ready'
|
||||
| 'blocked_session_fenced'
|
||||
| 'blocked_worker_offline'
|
||||
| 'blocked_worker_session_expired'
|
||||
| 'blocked_run_lease_expired'
|
||||
| 'blocked_control_plane_terminal';
|
||||
|
||||
export interface WorkerExecutionRecoveredCompletion {
|
||||
callbackSequence: number;
|
||||
outcome: 'succeeded' | 'failed';
|
||||
startedAtMs: number;
|
||||
finishedAtMs: number;
|
||||
exitCode: number;
|
||||
}
|
||||
|
||||
export interface WorkerExecutionOfferRecoveryResult {
|
||||
offerId: string;
|
||||
attemptId: string;
|
||||
state: WorkerExecutionOfferJournalRecord['state'];
|
||||
observedAtMs: number;
|
||||
authority: WorkerExecutionRecoveryAuthority;
|
||||
finding: WorkerExecutionRecoveryFinding;
|
||||
receiptChecks: number;
|
||||
processChecks: number;
|
||||
completionSubmission?: WorkerExecutionCompletionSubmission;
|
||||
completion?: WorkerExecutionRecoveredCompletion;
|
||||
identityPid?: number;
|
||||
}
|
||||
|
||||
export interface WorkerExecutionOfferRecoveryReconcilerOptions {
|
||||
clock?: { now(): number };
|
||||
receiptPublishGraceMs?: number;
|
||||
wait?: (delayMs: number) => Promise<void>;
|
||||
}
|
||||
|
||||
type ReceiptObservation =
|
||||
| { status: 'missing' }
|
||||
| { status: 'observed'; receipt: CompletionReceipt }
|
||||
| { status: 'conflict' }
|
||||
| { status: 'unavailable' };
|
||||
|
||||
const EXECUTION_OWNERSHIP_STATES = new Set<
|
||||
WorkerExecutionOfferJournalRecord['state']
|
||||
>(['launching', 'started', 'running_acknowledged', 'recovery_required']);
|
||||
|
||||
/**
|
||||
* Evidence-only Worker recovery pass. It reads a trusted receipt before
|
||||
* probing a durable process identity and never starts, stops, ACKs or removes
|
||||
* anything. The caller owns scheduling and all control-plane mutations.
|
||||
*/
|
||||
export class WorkerExecutionOfferRecoveryReconciler {
|
||||
private readonly clock: { now(): number };
|
||||
private readonly receiptPublishGraceMs: number;
|
||||
private readonly wait: (delayMs: number) => Promise<void>;
|
||||
|
||||
constructor(
|
||||
private readonly receipts: Pick<CompletionReceiptStore, 'read'>,
|
||||
private readonly receiptAuthenticator: WorkerExecutionCompletionReceiptAuthenticator,
|
||||
private readonly inspector: PersistedExecutionInspector,
|
||||
options: WorkerExecutionOfferRecoveryReconcilerOptions = {},
|
||||
) {
|
||||
this.clock = options.clock ?? Date;
|
||||
this.receiptPublishGraceMs = options.receiptPublishGraceMs ?? 0;
|
||||
if (
|
||||
!Number.isSafeInteger(this.receiptPublishGraceMs) ||
|
||||
this.receiptPublishGraceMs < 0 ||
|
||||
this.receiptPublishGraceMs > 5_000
|
||||
) {
|
||||
throw new RangeError('receiptPublishGraceMs must be between 0 and 5000');
|
||||
}
|
||||
this.wait =
|
||||
options.wait ??
|
||||
((delayMs) =>
|
||||
new Promise((resolve) => {
|
||||
setTimeout(resolve, delayMs);
|
||||
}));
|
||||
}
|
||||
|
||||
async reconcile(
|
||||
candidate: WorkerExecutionOfferJournalRecord,
|
||||
currentSession: WorkerRecord,
|
||||
): Promise<WorkerExecutionOfferRecoveryResult> {
|
||||
const record = cloneWorkerExecutionOfferJournalRecord(candidate);
|
||||
const observedAtMs = this.now();
|
||||
const authority = this.authority(record, currentSession, observedAtMs);
|
||||
let receiptChecks = 0;
|
||||
let processChecks = 0;
|
||||
|
||||
const observeReceipt = async (): Promise<ReceiptObservation> => {
|
||||
receiptChecks += 1;
|
||||
try {
|
||||
const receipt = await this.receipts.read(
|
||||
record.offer.candidate.attemptId,
|
||||
);
|
||||
if (!receipt) return { status: 'missing' };
|
||||
if (
|
||||
receipt.runId !== record.offer.candidate.runId ||
|
||||
receipt.attemptId !== record.offer.candidate.attemptId ||
|
||||
(record.executorStartedAtMs !== undefined &&
|
||||
receipt.startedAtMs !== record.executorStartedAtMs) ||
|
||||
!EXECUTION_OWNERSHIP_STATES.has(record.state)
|
||||
) {
|
||||
return { status: 'conflict' };
|
||||
}
|
||||
if (!(await this.receiptAuthenticator.authenticate(receipt, record))) {
|
||||
return { status: 'conflict' };
|
||||
}
|
||||
return { status: 'observed', receipt };
|
||||
} catch {
|
||||
return { status: 'unavailable' };
|
||||
}
|
||||
};
|
||||
|
||||
const result = (
|
||||
finding: WorkerExecutionRecoveryFinding,
|
||||
additions: Partial<
|
||||
Pick<
|
||||
WorkerExecutionOfferRecoveryResult,
|
||||
'completionSubmission' | 'completion' | 'identityPid'
|
||||
>
|
||||
> = {},
|
||||
): WorkerExecutionOfferRecoveryResult => ({
|
||||
offerId: record.offer.offerId,
|
||||
attemptId: record.offer.candidate.attemptId,
|
||||
state: record.state,
|
||||
observedAtMs,
|
||||
authority,
|
||||
finding,
|
||||
receiptChecks,
|
||||
processChecks,
|
||||
...additions,
|
||||
});
|
||||
|
||||
const receiptResult = (
|
||||
observation: ReceiptObservation,
|
||||
): WorkerExecutionOfferRecoveryResult | undefined => {
|
||||
if (observation.status === 'missing') return undefined;
|
||||
if (observation.status === 'conflict') {
|
||||
return result('completion_receipt_conflict');
|
||||
}
|
||||
if (observation.status === 'unavailable') {
|
||||
return result('completion_receipt_unavailable');
|
||||
}
|
||||
return result('completion_observed', {
|
||||
completionSubmission: this.completionSubmission(record, authority),
|
||||
completion: this.sanitizeCompletion(observation.receipt),
|
||||
});
|
||||
};
|
||||
|
||||
const initialReceipt = receiptResult(await observeReceipt());
|
||||
if (initialReceipt) return initialReceipt;
|
||||
|
||||
if (!EXECUTION_OWNERSHIP_STATES.has(record.state)) {
|
||||
return result('no_execution_expected');
|
||||
}
|
||||
if (!record.executorHandle) return result('launch_outcome_unknown');
|
||||
|
||||
let inspection;
|
||||
processChecks += 1;
|
||||
try {
|
||||
inspection = await this.inspector.inspect(record.executorHandle);
|
||||
} catch {
|
||||
return result('execution_probe_unavailable');
|
||||
}
|
||||
if (inspection.status === 'running') {
|
||||
return result('execution_running', {
|
||||
...(inspection.identityPid === undefined
|
||||
? {}
|
||||
: { identityPid: inspection.identityPid }),
|
||||
});
|
||||
}
|
||||
if (inspection.status === 'invalid') {
|
||||
return result('execution_handle_invalid');
|
||||
}
|
||||
if (inspection.status === 'identity_mismatch') {
|
||||
return result('execution_identity_mismatch', {
|
||||
...(inspection.identityPid === undefined
|
||||
? {}
|
||||
: { identityPid: inspection.identityPid }),
|
||||
});
|
||||
}
|
||||
if (inspection.status === 'unsupported') {
|
||||
return result('execution_probe_unsupported', {
|
||||
...(inspection.identityPid === undefined
|
||||
? {}
|
||||
: { identityPid: inspection.identityPid }),
|
||||
});
|
||||
}
|
||||
|
||||
const afterExitReceipt = receiptResult(await observeReceipt());
|
||||
if (afterExitReceipt) return afterExitReceipt;
|
||||
if (this.receiptPublishGraceMs > 0) {
|
||||
await this.wait(this.receiptPublishGraceMs);
|
||||
const afterGraceReceipt = receiptResult(await observeReceipt());
|
||||
if (afterGraceReceipt) return afterGraceReceipt;
|
||||
}
|
||||
return result('execution_exited_without_receipt', {
|
||||
...(inspection.identityPid === undefined
|
||||
? {}
|
||||
: { identityPid: inspection.identityPid }),
|
||||
});
|
||||
}
|
||||
|
||||
private authority(
|
||||
record: WorkerExecutionOfferJournalRecord,
|
||||
currentSession: WorkerRecord,
|
||||
observedAtMs: number,
|
||||
): WorkerExecutionRecoveryAuthority {
|
||||
if (
|
||||
record.offer.worker.id !== currentSession.id ||
|
||||
record.offer.worker.sessionId !== currentSession.sessionId ||
|
||||
record.offer.worker.generation !== currentSession.generation
|
||||
) {
|
||||
return 'session_fenced';
|
||||
}
|
||||
if (currentSession.status === 'offline') return 'worker_offline';
|
||||
if (currentSession.leaseExpiresAtMs <= observedAtMs) {
|
||||
return 'worker_session_expired';
|
||||
}
|
||||
if (record.offer.lease.expiresAtMs <= observedAtMs) {
|
||||
return 'run_lease_expired';
|
||||
}
|
||||
return 'current';
|
||||
}
|
||||
|
||||
private completionSubmission(
|
||||
record: WorkerExecutionOfferJournalRecord,
|
||||
authority: WorkerExecutionRecoveryAuthority,
|
||||
): WorkerExecutionCompletionSubmission {
|
||||
if (
|
||||
record.state === 'recovery_required' &&
|
||||
record.recoveryReason === 'control_plane_terminal'
|
||||
) {
|
||||
return 'blocked_control_plane_terminal';
|
||||
}
|
||||
if (authority === 'current') return 'ready';
|
||||
return `blocked_${authority}`;
|
||||
}
|
||||
|
||||
private sanitizeCompletion(
|
||||
receipt: CompletionReceipt,
|
||||
): WorkerExecutionRecoveredCompletion {
|
||||
return {
|
||||
callbackSequence: receipt.callbackSequence,
|
||||
outcome: receipt.exitCode === 0 ? 'succeeded' : 'failed',
|
||||
startedAtMs: receipt.startedAtMs,
|
||||
finishedAtMs: receipt.finishedAtMs,
|
||||
exitCode: receipt.exitCode,
|
||||
};
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const observedAtMs = this.clock.now();
|
||||
if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0) {
|
||||
throw new TypeError(
|
||||
'Worker offer recovery clock returned an invalid time',
|
||||
);
|
||||
}
|
||||
return observedAtMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
import type { WorkerExecutionOfferJournalState } from '../domain/workerExecutionOffer';
|
||||
import type { WorkerRecord } from '../domain/worker';
|
||||
import type { WorkerExecutionOfferJournal } from '../ports/workerExecutionOfferJournal';
|
||||
|
||||
export const MAX_WORKER_OFFER_STARTUP_AUDIT_PAGES = 16;
|
||||
|
||||
export type WorkerExecutionOfferStartupCategory =
|
||||
| 'settled_start_failure'
|
||||
| 'settled_completion'
|
||||
| 'redelivery_required'
|
||||
| 'fenced_without_local_execution'
|
||||
| 'expired_without_local_execution'
|
||||
| 'launch_reconciliation_required'
|
||||
| 'execution_reconciliation_required';
|
||||
|
||||
export interface WorkerExecutionOfferStartupAuditEntry {
|
||||
offerId: string;
|
||||
attemptId: string;
|
||||
state: WorkerExecutionOfferJournalState;
|
||||
category: WorkerExecutionOfferStartupCategory;
|
||||
}
|
||||
|
||||
export interface WorkerExecutionOfferStartupAuditResult {
|
||||
status: 'ready' | 'reconciliation_required' | 'scan_budget_exhausted';
|
||||
observedAtMs: number;
|
||||
pagesScanned: number;
|
||||
recordsScanned: number;
|
||||
counts: Readonly<Record<WorkerExecutionOfferStartupCategory, number>>;
|
||||
entries: readonly WorkerExecutionOfferStartupAuditEntry[];
|
||||
nextAfterOfferId?: string;
|
||||
}
|
||||
|
||||
export interface WorkerExecutionOfferStartupAuditorOptions {
|
||||
pageSize?: number;
|
||||
maxPages?: number;
|
||||
clock?: { now(): number };
|
||||
}
|
||||
|
||||
export class InvalidWorkerExecutionOfferStartupPageError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'InvalidWorkerExecutionOfferStartupPageError';
|
||||
}
|
||||
}
|
||||
|
||||
const CATEGORIES: readonly WorkerExecutionOfferStartupCategory[] = [
|
||||
'settled_start_failure',
|
||||
'settled_completion',
|
||||
'redelivery_required',
|
||||
'fenced_without_local_execution',
|
||||
'expired_without_local_execution',
|
||||
'launch_reconciliation_required',
|
||||
'execution_reconciliation_required',
|
||||
];
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read-only startup gate. It never resumes an ACK or starts/stops an Executor;
|
||||
* the future recovery coordinator must act on these low-sensitive categories.
|
||||
*/
|
||||
export class WorkerExecutionOfferStartupAuditor {
|
||||
private readonly pageSize: number;
|
||||
private readonly maxPages: number;
|
||||
private readonly clock: { now(): number };
|
||||
|
||||
constructor(
|
||||
private readonly journal: Pick<WorkerExecutionOfferJournal, 'list'>,
|
||||
options: WorkerExecutionOfferStartupAuditorOptions = {},
|
||||
) {
|
||||
this.pageSize = options.pageSize ?? 16;
|
||||
this.maxPages = options.maxPages ?? 4;
|
||||
this.clock = options.clock ?? Date;
|
||||
assertIntegerBetween('pageSize', this.pageSize, 1, 64);
|
||||
assertIntegerBetween(
|
||||
'maxPages',
|
||||
this.maxPages,
|
||||
1,
|
||||
MAX_WORKER_OFFER_STARTUP_AUDIT_PAGES,
|
||||
);
|
||||
if (this.pageSize * this.maxPages > 1024) {
|
||||
throw new RangeError(
|
||||
'Worker offer startup audit budget must not exceed 1024 records',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async audit(
|
||||
currentSession: WorkerRecord,
|
||||
): Promise<WorkerExecutionOfferStartupAuditResult> {
|
||||
const observedAtMs = this.now();
|
||||
const counts = Object.fromEntries(
|
||||
CATEGORIES.map((category) => [category, 0]),
|
||||
) as Record<WorkerExecutionOfferStartupCategory, number>;
|
||||
const entries: WorkerExecutionOfferStartupAuditEntry[] = [];
|
||||
const seen = new Set<string>();
|
||||
let afterOfferId: string | undefined;
|
||||
let pagesScanned = 0;
|
||||
|
||||
while (pagesScanned < this.maxPages) {
|
||||
const page = await this.journal.list({
|
||||
...(afterOfferId === undefined ? {} : { afterOfferId }),
|
||||
limit: this.pageSize,
|
||||
});
|
||||
pagesScanned += 1;
|
||||
if (page.records.length > this.pageSize) {
|
||||
throw new InvalidWorkerExecutionOfferStartupPageError(
|
||||
'Worker offer startup page exceeds the requested limit',
|
||||
);
|
||||
}
|
||||
let previousOfferId = afterOfferId;
|
||||
for (const record of page.records) {
|
||||
const offerId = record.offer.offerId;
|
||||
if (
|
||||
seen.has(offerId) ||
|
||||
(previousOfferId !== undefined && offerId <= previousOfferId)
|
||||
) {
|
||||
throw new InvalidWorkerExecutionOfferStartupPageError(
|
||||
'Worker offer startup cursor did not advance',
|
||||
);
|
||||
}
|
||||
seen.add(offerId);
|
||||
previousOfferId = offerId;
|
||||
const category = this.classify(record, currentSession, observedAtMs);
|
||||
counts[category] += 1;
|
||||
entries.push({
|
||||
offerId,
|
||||
attemptId: record.offer.candidate.attemptId,
|
||||
state: record.state,
|
||||
category,
|
||||
});
|
||||
}
|
||||
|
||||
if (page.nextAfterOfferId === undefined) {
|
||||
return this.result(counts, entries, observedAtMs, pagesScanned);
|
||||
}
|
||||
if (
|
||||
page.records.length !== this.pageSize ||
|
||||
page.records.length === 0 ||
|
||||
page.nextAfterOfferId !==
|
||||
page.records[page.records.length - 1].offer.offerId ||
|
||||
(afterOfferId !== undefined && page.nextAfterOfferId <= afterOfferId)
|
||||
) {
|
||||
throw new InvalidWorkerExecutionOfferStartupPageError(
|
||||
'Worker offer startup page returned an invalid resume cursor',
|
||||
);
|
||||
}
|
||||
afterOfferId = page.nextAfterOfferId;
|
||||
}
|
||||
|
||||
return {
|
||||
status: 'scan_budget_exhausted',
|
||||
observedAtMs,
|
||||
pagesScanned,
|
||||
recordsScanned: entries.length,
|
||||
counts,
|
||||
entries,
|
||||
...(afterOfferId === undefined ? {} : { nextAfterOfferId: afterOfferId }),
|
||||
};
|
||||
}
|
||||
|
||||
private classify(
|
||||
record: Awaited<
|
||||
ReturnType<WorkerExecutionOfferJournal['list']>
|
||||
>['records'][number],
|
||||
currentSession: WorkerRecord,
|
||||
observedAtMs: number,
|
||||
): WorkerExecutionOfferStartupCategory {
|
||||
if (record.state === 'start_failure_acknowledged') {
|
||||
return 'settled_start_failure';
|
||||
}
|
||||
if (record.state === 'completion_acknowledged') {
|
||||
return 'settled_completion';
|
||||
}
|
||||
if (record.state === 'launching' || record.state === 'recovery_required') {
|
||||
return 'launch_reconciliation_required';
|
||||
}
|
||||
if (record.state === 'started' || record.state === 'running_acknowledged') {
|
||||
return 'execution_reconciliation_required';
|
||||
}
|
||||
const sameSession =
|
||||
record.offer.worker.id === currentSession.id &&
|
||||
record.offer.worker.sessionId === currentSession.sessionId &&
|
||||
record.offer.worker.generation === currentSession.generation;
|
||||
if (!sameSession) return 'fenced_without_local_execution';
|
||||
if (
|
||||
record.offer.lease.expiresAtMs <= observedAtMs ||
|
||||
currentSession.leaseExpiresAtMs <= observedAtMs ||
|
||||
currentSession.status === 'offline'
|
||||
) {
|
||||
return 'expired_without_local_execution';
|
||||
}
|
||||
return 'redelivery_required';
|
||||
}
|
||||
|
||||
private result(
|
||||
counts: Record<WorkerExecutionOfferStartupCategory, number>,
|
||||
entries: WorkerExecutionOfferStartupAuditEntry[],
|
||||
observedAtMs: number,
|
||||
pagesScanned: number,
|
||||
): WorkerExecutionOfferStartupAuditResult {
|
||||
const requiresReconciliation =
|
||||
counts.launch_reconciliation_required > 0 ||
|
||||
counts.execution_reconciliation_required > 0;
|
||||
return {
|
||||
status: requiresReconciliation ? 'reconciliation_required' : 'ready',
|
||||
observedAtMs,
|
||||
pagesScanned,
|
||||
recordsScanned: entries.length,
|
||||
counts,
|
||||
entries,
|
||||
};
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const observedAtMs = this.clock.now();
|
||||
if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0) {
|
||||
throw new TypeError(
|
||||
'Worker offer startup audit clock returned an invalid time',
|
||||
);
|
||||
}
|
||||
return observedAtMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
import type {
|
||||
WorkerExecutionOfferRecoveryActionResult,
|
||||
WorkerExecutionOfferRecoveryCoordinator,
|
||||
WorkerCompletionReceiptCleanup,
|
||||
} from './workerExecutionOfferRecoveryCoordinator';
|
||||
import type {
|
||||
WorkerExecutionOfferStartupAuditResult,
|
||||
WorkerExecutionOfferStartupCategory,
|
||||
} from './workerExecutionOfferStartupAuditor';
|
||||
|
||||
export const MAX_WORKER_OFFER_STARTUP_RECOVERY_ACTIONS = 1024;
|
||||
|
||||
export interface WorkerExecutionOfferStartupRecoveryEntry {
|
||||
offerId: string;
|
||||
attemptId: string;
|
||||
category: WorkerExecutionOfferStartupCategory;
|
||||
outcome: 'applied' | 'failed';
|
||||
actionStatus?: WorkerExecutionOfferRecoveryActionResult['status'];
|
||||
receiptCleanup?: WorkerCompletionReceiptCleanup;
|
||||
}
|
||||
|
||||
export interface WorkerExecutionOfferStartupRecoveryResult {
|
||||
status: 'recovered' | 'reconciliation_required' | 'action_budget_exhausted';
|
||||
actionsPlanned: number;
|
||||
actionsAttempted: number;
|
||||
entries: readonly WorkerExecutionOfferStartupRecoveryEntry[];
|
||||
}
|
||||
|
||||
export class WorkerExecutionOfferStartupRecoveryAuditIncompleteError extends Error {
|
||||
constructor() {
|
||||
super('Worker offer startup recovery requires a complete startup audit');
|
||||
this.name = 'WorkerExecutionOfferStartupRecoveryAuditIncompleteError';
|
||||
}
|
||||
}
|
||||
|
||||
export class InvalidWorkerExecutionOfferStartupRecoveryInputError extends TypeError {
|
||||
constructor(message: string) {
|
||||
super(`Worker offer startup recovery input is invalid: ${message}`);
|
||||
this.name = 'InvalidWorkerExecutionOfferStartupRecoveryInputError';
|
||||
}
|
||||
}
|
||||
|
||||
const ACTIONABLE_CATEGORIES = new Set<WorkerExecutionOfferStartupCategory>([
|
||||
'settled_completion',
|
||||
'launch_reconciliation_required',
|
||||
'execution_reconciliation_required',
|
||||
]);
|
||||
|
||||
const RECOVERED_ACTIONS = new Set<
|
||||
WorkerExecutionOfferRecoveryActionResult['status']
|
||||
>([
|
||||
'not_found',
|
||||
'running_acknowledged',
|
||||
'already_running',
|
||||
'completion_acknowledged',
|
||||
'already_completed',
|
||||
]);
|
||||
|
||||
/** Runs one bounded, sequential recovery pass and never creates a timer. */
|
||||
export class WorkerExecutionOfferStartupRecoverySupervisor {
|
||||
private readonly maximumActions: number;
|
||||
|
||||
constructor(
|
||||
private readonly coordinator: Pick<
|
||||
WorkerExecutionOfferRecoveryCoordinator,
|
||||
'recover'
|
||||
>,
|
||||
options: { maximumActions?: number } = {},
|
||||
) {
|
||||
this.maximumActions = options.maximumActions ?? 64;
|
||||
if (
|
||||
!Number.isSafeInteger(this.maximumActions) ||
|
||||
this.maximumActions < 1 ||
|
||||
this.maximumActions > MAX_WORKER_OFFER_STARTUP_RECOVERY_ACTIONS
|
||||
) {
|
||||
throw new RangeError(
|
||||
`maximumActions must be between 1 and ${MAX_WORKER_OFFER_STARTUP_RECOVERY_ACTIONS}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async recover(
|
||||
audit: WorkerExecutionOfferStartupAuditResult,
|
||||
): Promise<WorkerExecutionOfferStartupRecoveryResult> {
|
||||
if (audit.status === 'scan_budget_exhausted') {
|
||||
throw new WorkerExecutionOfferStartupRecoveryAuditIncompleteError();
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(audit.recordsScanned) ||
|
||||
audit.recordsScanned < 0 ||
|
||||
audit.recordsScanned !== audit.entries.length
|
||||
) {
|
||||
throw new InvalidWorkerExecutionOfferStartupRecoveryInputError(
|
||||
'record count does not match entries',
|
||||
);
|
||||
}
|
||||
const seen = new Set<string>();
|
||||
for (const entry of audit.entries) {
|
||||
if (seen.has(entry.offerId)) {
|
||||
throw new InvalidWorkerExecutionOfferStartupRecoveryInputError(
|
||||
'offerId is duplicated',
|
||||
);
|
||||
}
|
||||
seen.add(entry.offerId);
|
||||
}
|
||||
const planned = audit.entries.filter((entry) =>
|
||||
ACTIONABLE_CATEGORIES.has(entry.category),
|
||||
);
|
||||
if (planned.length > this.maximumActions) {
|
||||
return {
|
||||
status: 'action_budget_exhausted',
|
||||
actionsPlanned: planned.length,
|
||||
actionsAttempted: 0,
|
||||
entries: [],
|
||||
};
|
||||
}
|
||||
|
||||
const entries: WorkerExecutionOfferStartupRecoveryEntry[] = [];
|
||||
let unresolved = false;
|
||||
for (const entry of planned) {
|
||||
try {
|
||||
const action = await this.coordinator.recover(entry.offerId);
|
||||
if (!RECOVERED_ACTIONS.has(action.status)) unresolved = true;
|
||||
entries.push({
|
||||
offerId: entry.offerId,
|
||||
attemptId: entry.attemptId,
|
||||
category: entry.category,
|
||||
outcome: 'applied',
|
||||
actionStatus: action.status,
|
||||
...(action.receiptCleanup === undefined
|
||||
? {}
|
||||
: { receiptCleanup: action.receiptCleanup }),
|
||||
});
|
||||
} catch {
|
||||
unresolved = true;
|
||||
entries.push({
|
||||
offerId: entry.offerId,
|
||||
attemptId: entry.attemptId,
|
||||
category: entry.category,
|
||||
outcome: 'failed',
|
||||
});
|
||||
}
|
||||
}
|
||||
return {
|
||||
status: unresolved ? 'reconciliation_required' : 'recovered',
|
||||
actionsPlanned: planned.length,
|
||||
actionsAttempted: entries.length,
|
||||
entries,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import {
|
||||
WorkerFenceRejectedError,
|
||||
assertWorkerConcurrency,
|
||||
assertWorkerId,
|
||||
assertWorkerSessionId,
|
||||
type WorkerCapabilities,
|
||||
type WorkerRecord,
|
||||
} from '../domain/worker';
|
||||
import type { WorkerControlPlaneClient } from '../ports/workerControlPlaneClient';
|
||||
|
||||
export const MIN_WORKER_HEARTBEAT_INTERVAL_MS = 1_000;
|
||||
export const MAX_WORKER_HEARTBEAT_INTERVAL_MS = 5 * 60_000;
|
||||
export const MAX_WORKER_HEARTBEAT_STOP_TIMEOUT_MS = 60_000;
|
||||
|
||||
interface ScheduledTimer {
|
||||
unref?: () => void;
|
||||
}
|
||||
|
||||
export interface WorkerHeartbeatScheduler {
|
||||
setTimeout(callback: () => void, delayMs: number): ScheduledTimer;
|
||||
clearTimeout(timer: ScheduledTimer): void;
|
||||
}
|
||||
|
||||
export interface WorkerHeartbeatLifecycleOptions {
|
||||
workerId: string;
|
||||
capabilities(): WorkerCapabilities | Promise<WorkerCapabilities>;
|
||||
maxConcurrentRuns: number;
|
||||
availableSlots(): number | Promise<number>;
|
||||
heartbeatIntervalMs?: number;
|
||||
stopTimeoutMs?: number;
|
||||
createSessionId?: () => string;
|
||||
scheduler?: WorkerHeartbeatScheduler;
|
||||
onSession?: (worker: WorkerRecord) => void;
|
||||
onError?: (error: unknown) => void;
|
||||
onFenced?: (error: WorkerFenceRejectedError) => void;
|
||||
}
|
||||
|
||||
export type WorkerHeartbeatStopResult =
|
||||
| 'drained'
|
||||
| 'timed_out'
|
||||
| 'disconnect_failed';
|
||||
|
||||
const defaultScheduler: WorkerHeartbeatScheduler = {
|
||||
setTimeout(callback, delayMs) {
|
||||
return setTimeout(callback, delayMs);
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
clearTimeout(timer as ReturnType<typeof setTimeout>);
|
||||
},
|
||||
};
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
export class WorkerHeartbeatLifecycle {
|
||||
private readonly workerId: string;
|
||||
private readonly maxConcurrentRuns: number;
|
||||
private readonly heartbeatIntervalMs: number;
|
||||
private readonly stopTimeoutMs: number;
|
||||
private readonly capabilitiesProvider: WorkerHeartbeatLifecycleOptions['capabilities'];
|
||||
private readonly availableSlotsProvider: WorkerHeartbeatLifecycleOptions['availableSlots'];
|
||||
private readonly createSessionId: () => string;
|
||||
private readonly scheduler: WorkerHeartbeatScheduler;
|
||||
private readonly onSession?: (worker: WorkerRecord) => void;
|
||||
private readonly onError?: (error: unknown) => void;
|
||||
private readonly onFenced?: (error: WorkerFenceRejectedError) => void;
|
||||
private started = false;
|
||||
private draining = false;
|
||||
private timer?: ScheduledTimer;
|
||||
private inFlight?: Promise<void>;
|
||||
private session?: WorkerRecord;
|
||||
|
||||
constructor(
|
||||
private readonly client: WorkerControlPlaneClient,
|
||||
options: WorkerHeartbeatLifecycleOptions,
|
||||
) {
|
||||
assertWorkerId(options.workerId);
|
||||
assertWorkerConcurrency(options.maxConcurrentRuns, 0);
|
||||
this.workerId = options.workerId;
|
||||
this.maxConcurrentRuns = options.maxConcurrentRuns;
|
||||
this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? 10_000;
|
||||
this.stopTimeoutMs = options.stopTimeoutMs ?? 5_000;
|
||||
this.capabilitiesProvider = options.capabilities;
|
||||
this.availableSlotsProvider = options.availableSlots;
|
||||
this.createSessionId = options.createSessionId ?? uuidV7;
|
||||
this.scheduler = options.scheduler ?? defaultScheduler;
|
||||
this.onSession = options.onSession;
|
||||
this.onError = options.onError;
|
||||
this.onFenced = options.onFenced;
|
||||
assertIntegerBetween(
|
||||
'heartbeatIntervalMs',
|
||||
this.heartbeatIntervalMs,
|
||||
MIN_WORKER_HEARTBEAT_INTERVAL_MS,
|
||||
MAX_WORKER_HEARTBEAT_INTERVAL_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'stopTimeoutMs',
|
||||
this.stopTimeoutMs,
|
||||
1,
|
||||
MAX_WORKER_HEARTBEAT_STOP_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
currentSession(): WorkerRecord | undefined {
|
||||
return this.session ? this.cloneRecord(this.session) : undefined;
|
||||
}
|
||||
|
||||
async start(): Promise<boolean> {
|
||||
if (this.started || this.inFlight) return false;
|
||||
this.started = true;
|
||||
this.draining = false;
|
||||
try {
|
||||
const sessionId = this.createSessionId();
|
||||
assertWorkerSessionId(sessionId);
|
||||
const availableSlots = await this.readAvailableSlots();
|
||||
const worker = await this.client.register({
|
||||
workerId: this.workerId,
|
||||
sessionId,
|
||||
capabilities: await this.capabilitiesProvider(),
|
||||
maxConcurrentRuns: this.maxConcurrentRuns,
|
||||
availableSlots,
|
||||
});
|
||||
this.acceptSession(worker, sessionId);
|
||||
this.schedule();
|
||||
return true;
|
||||
} catch (error) {
|
||||
this.started = false;
|
||||
this.notifyError(error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async drain(): Promise<WorkerRecord | undefined> {
|
||||
if (!this.started || !this.session) return undefined;
|
||||
this.draining = true;
|
||||
this.clearTimer();
|
||||
const precedingOperation = this.inFlight ?? Promise.resolve();
|
||||
const drainOperation = precedingOperation.then(async () => {
|
||||
const session = this.session;
|
||||
if (!this.started || !session) return undefined;
|
||||
try {
|
||||
const worker = await this.client.drain({
|
||||
workerId: this.workerId,
|
||||
sessionId: session.sessionId,
|
||||
generation: session.generation,
|
||||
expectedVersion: session.version,
|
||||
});
|
||||
this.acceptSession(worker, session.sessionId);
|
||||
return this.cloneRecord(worker);
|
||||
} catch (error) {
|
||||
this.handleOperationError(error);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
const trackedOperation = drainOperation
|
||||
.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
)
|
||||
.finally(() => {
|
||||
if (this.inFlight === trackedOperation) this.inFlight = undefined;
|
||||
this.schedule();
|
||||
});
|
||||
this.inFlight = trackedOperation;
|
||||
return drainOperation;
|
||||
}
|
||||
|
||||
async stop(): Promise<WorkerHeartbeatStopResult> {
|
||||
this.clearTimer();
|
||||
const deadline = Date.now() + this.stopTimeoutMs;
|
||||
if (!(await this.waitWithin(this.inFlight, deadline))) {
|
||||
this.started = false;
|
||||
this.clearTimer();
|
||||
return 'timed_out';
|
||||
}
|
||||
this.started = false;
|
||||
this.clearTimer();
|
||||
const session = this.session;
|
||||
if (!session) return 'drained';
|
||||
let disconnectFailed = false;
|
||||
const disconnected = this.client
|
||||
.disconnect({
|
||||
workerId: this.workerId,
|
||||
sessionId: session.sessionId,
|
||||
generation: session.generation,
|
||||
expectedVersion: session.version,
|
||||
})
|
||||
.then((worker) => {
|
||||
this.acceptSession(worker, session.sessionId);
|
||||
})
|
||||
.catch((error) => {
|
||||
disconnectFailed = true;
|
||||
this.handleOperationError(error);
|
||||
});
|
||||
if (!(await this.waitWithin(disconnected, deadline))) return 'timed_out';
|
||||
return disconnectFailed ? 'disconnect_failed' : 'drained';
|
||||
}
|
||||
|
||||
private schedule(): void {
|
||||
if (!this.started || this.timer || this.inFlight) return;
|
||||
const timer = this.scheduler.setTimeout(() => {
|
||||
if (this.timer === timer) this.timer = undefined;
|
||||
this.runHeartbeat();
|
||||
}, this.heartbeatIntervalMs);
|
||||
this.timer = timer;
|
||||
timer.unref?.();
|
||||
}
|
||||
|
||||
private clearTimer(): void {
|
||||
if (!this.timer) return;
|
||||
this.scheduler.clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
|
||||
private runHeartbeat(): void {
|
||||
if (!this.started || this.inFlight || !this.session) return;
|
||||
const session = this.session;
|
||||
const inFlight = this.readAvailableSlots()
|
||||
.then((availableSlots) =>
|
||||
this.client.heartbeat({
|
||||
workerId: this.workerId,
|
||||
sessionId: session.sessionId,
|
||||
generation: session.generation,
|
||||
expectedVersion: session.version,
|
||||
availableSlots: this.draining ? 0 : availableSlots,
|
||||
}),
|
||||
)
|
||||
.then((worker) => this.acceptSession(worker, session.sessionId))
|
||||
.catch((error) => this.handleOperationError(error))
|
||||
.then(() => undefined)
|
||||
.finally(() => {
|
||||
if (this.inFlight === inFlight) this.inFlight = undefined;
|
||||
this.schedule();
|
||||
});
|
||||
this.inFlight = inFlight;
|
||||
}
|
||||
|
||||
private async readAvailableSlots(): Promise<number> {
|
||||
const availableSlots = await this.availableSlotsProvider();
|
||||
assertWorkerConcurrency(this.maxConcurrentRuns, availableSlots);
|
||||
return availableSlots;
|
||||
}
|
||||
|
||||
private acceptSession(worker: WorkerRecord, sessionId: string): void {
|
||||
if (worker.id !== this.workerId || worker.sessionId !== sessionId) {
|
||||
throw new WorkerFenceRejectedError(this.workerId, 'session_mismatch');
|
||||
}
|
||||
this.session = this.cloneRecord(worker);
|
||||
try {
|
||||
this.onSession?.(this.cloneRecord(worker));
|
||||
} catch (error) {
|
||||
this.notifyError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private handleOperationError(error: unknown): void {
|
||||
if (error instanceof WorkerFenceRejectedError) {
|
||||
this.started = false;
|
||||
this.clearTimer();
|
||||
try {
|
||||
this.onFenced?.(error);
|
||||
} catch (callbackError) {
|
||||
this.notifyError(callbackError);
|
||||
}
|
||||
}
|
||||
this.notifyError(error);
|
||||
}
|
||||
|
||||
private notifyError(error: unknown): void {
|
||||
try {
|
||||
this.onError?.(error);
|
||||
} catch {
|
||||
// Diagnostics must never create a heartbeat failure loop.
|
||||
}
|
||||
}
|
||||
|
||||
private async waitWithin(
|
||||
promise: Promise<unknown> | undefined,
|
||||
deadline: number,
|
||||
): Promise<boolean> {
|
||||
if (!promise) return true;
|
||||
const remainingMs = Math.max(0, deadline - Date.now());
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const completed = await Promise.race([
|
||||
promise.then(() => true),
|
||||
new Promise<false>((resolve) => {
|
||||
timeout = setTimeout(() => resolve(false), remainingMs);
|
||||
}),
|
||||
]);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
return completed;
|
||||
}
|
||||
|
||||
private cloneRecord(worker: WorkerRecord): WorkerRecord {
|
||||
return {
|
||||
...worker,
|
||||
capabilities: {
|
||||
...worker.capabilities,
|
||||
executors: [...worker.capabilities.executors],
|
||||
runtimes: worker.capabilities.runtimes.map((runtime) => ({
|
||||
...runtime,
|
||||
})),
|
||||
labels: { ...worker.capabilities.labels },
|
||||
capacity: {
|
||||
...worker.capabilities.capacity,
|
||||
...(worker.capabilities.capacity.gpu
|
||||
? {
|
||||
gpu: worker.capabilities.capacity.gpu.map((gpu) => ({
|
||||
...gpu,
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
features: [...worker.capabilities.features],
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,477 @@
|
||||
import {
|
||||
RunDispatchLeaseFenceRejectedError,
|
||||
assertRunDispatchLeaseRecord,
|
||||
type RunDispatchLeaseRecord,
|
||||
type RunDispatchReleaseReason,
|
||||
} from '../domain/runDispatchLease';
|
||||
import type { WorkerRecord } from '../domain/worker';
|
||||
import type { WorkerRunLeaseClient } from '../ports/workerRunLeaseClient';
|
||||
|
||||
export const MIN_WORKER_RUN_LEASE_RETRY_MS = 100;
|
||||
export const MAX_WORKER_RUN_LEASE_RETRY_MS = 5_000;
|
||||
export const MAX_WORKER_RUN_LEASE_STOP_TIMEOUT_MS = 60_000;
|
||||
|
||||
interface ScheduledTimer {
|
||||
unref?: () => void;
|
||||
}
|
||||
|
||||
export interface WorkerRunLeaseScheduler {
|
||||
setTimeout(callback: () => void, delayMs: number): ScheduledTimer;
|
||||
clearTimeout(timer: ScheduledTimer): void;
|
||||
}
|
||||
|
||||
export type WorkerRunLeaseLossReason =
|
||||
| 'lease_expired'
|
||||
| 'fenced'
|
||||
| 'worker_session_replaced'
|
||||
| 'worker_unavailable'
|
||||
| 'invalid_renewal';
|
||||
|
||||
export interface WorkerRunLeaseLoss {
|
||||
lease: RunDispatchLeaseRecord;
|
||||
reason: WorkerRunLeaseLossReason;
|
||||
error?: unknown;
|
||||
}
|
||||
|
||||
export interface WorkerRunLeaseLifecycleOptions {
|
||||
currentSession(): WorkerRecord | undefined;
|
||||
clock?: { now(): number };
|
||||
scheduler?: WorkerRunLeaseScheduler;
|
||||
retryDelayMs?: number;
|
||||
stopTimeoutMs?: number;
|
||||
onRenewed?: (lease: RunDispatchLeaseRecord) => void;
|
||||
onLost?: (loss: WorkerRunLeaseLoss) => void;
|
||||
onError?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export type WorkerRunLeaseStopResult = 'stopped' | 'not_started' | 'timed_out';
|
||||
|
||||
interface TrackedLease {
|
||||
lease: RunDispatchLeaseRecord;
|
||||
renewAtMs: number;
|
||||
}
|
||||
|
||||
const defaultScheduler: WorkerRunLeaseScheduler = {
|
||||
setTimeout(callback, delayMs) {
|
||||
return setTimeout(callback, delayMs);
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
clearTimeout(timer as ReturnType<typeof setTimeout>);
|
||||
},
|
||||
};
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
function cloneLease(lease: RunDispatchLeaseRecord): RunDispatchLeaseRecord {
|
||||
return { ...lease };
|
||||
}
|
||||
|
||||
function renewalTime(lease: RunDispatchLeaseRecord): number {
|
||||
const duration = lease.expiresAtMs - lease.renewedAtMs;
|
||||
if (!Number.isSafeInteger(duration) || duration < 2) {
|
||||
throw new TypeError('Run dispatch lease renewal window is invalid');
|
||||
}
|
||||
return lease.renewedAtMs + Math.floor(duration / 2);
|
||||
}
|
||||
|
||||
export class WorkerRunLeaseLifecycle {
|
||||
private readonly currentSessionProvider: WorkerRunLeaseLifecycleOptions['currentSession'];
|
||||
private readonly clock: { now(): number };
|
||||
private readonly scheduler: WorkerRunLeaseScheduler;
|
||||
private readonly retryDelayMs: number;
|
||||
private readonly stopTimeoutMs: number;
|
||||
private readonly onRenewed?: WorkerRunLeaseLifecycleOptions['onRenewed'];
|
||||
private readonly onLost?: WorkerRunLeaseLifecycleOptions['onLost'];
|
||||
private readonly onError?: WorkerRunLeaseLifecycleOptions['onError'];
|
||||
private readonly tracked = new Map<string, TrackedLease>();
|
||||
private started = false;
|
||||
private releasing = false;
|
||||
private timer?: ScheduledTimer;
|
||||
private inFlight?: Promise<void>;
|
||||
|
||||
constructor(
|
||||
private readonly client: WorkerRunLeaseClient,
|
||||
options: WorkerRunLeaseLifecycleOptions,
|
||||
) {
|
||||
this.currentSessionProvider = options.currentSession;
|
||||
this.clock = options.clock ?? Date;
|
||||
this.scheduler = options.scheduler ?? defaultScheduler;
|
||||
this.retryDelayMs = options.retryDelayMs ?? 1_000;
|
||||
this.stopTimeoutMs = options.stopTimeoutMs ?? 5_000;
|
||||
this.onRenewed = options.onRenewed;
|
||||
this.onLost = options.onLost;
|
||||
this.onError = options.onError;
|
||||
assertIntegerBetween(
|
||||
'retryDelayMs',
|
||||
this.retryDelayMs,
|
||||
MIN_WORKER_RUN_LEASE_RETRY_MS,
|
||||
MAX_WORKER_RUN_LEASE_RETRY_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'stopTimeoutMs',
|
||||
this.stopTimeoutMs,
|
||||
1,
|
||||
MAX_WORKER_RUN_LEASE_STOP_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
start(): boolean {
|
||||
if (this.started || this.releasing) return false;
|
||||
this.started = true;
|
||||
this.schedule();
|
||||
return true;
|
||||
}
|
||||
|
||||
track(lease: RunDispatchLeaseRecord): void {
|
||||
if (this.releasing) {
|
||||
throw new Error('Run leases cannot be tracked while release is active');
|
||||
}
|
||||
assertRunDispatchLeaseRecord(lease);
|
||||
if (lease.status !== 'leased') {
|
||||
throw new TypeError('Only an active Run dispatch lease can be tracked');
|
||||
}
|
||||
const nowMs = this.now();
|
||||
if (lease.expiresAtMs <= nowMs) {
|
||||
throw new TypeError('An expired Run dispatch lease cannot be tracked');
|
||||
}
|
||||
const session = this.currentSessionProvider();
|
||||
this.assertCurrentSession(lease, session, nowMs);
|
||||
const existing = this.tracked.get(lease.attemptId);
|
||||
if (existing && !this.sameAuthority(existing.lease, lease)) {
|
||||
throw new TypeError(
|
||||
`Run dispatch lease ${lease.attemptId} cannot replace a different authority`,
|
||||
);
|
||||
}
|
||||
if (
|
||||
!existing &&
|
||||
session &&
|
||||
this.tracked.size >= session.maxConcurrentRuns
|
||||
) {
|
||||
throw new RangeError('Tracked Run leases exceed Worker concurrency');
|
||||
}
|
||||
this.tracked.set(lease.attemptId, {
|
||||
lease: cloneLease(lease),
|
||||
renewAtMs: renewalTime(lease),
|
||||
});
|
||||
this.clearTimer();
|
||||
this.schedule();
|
||||
}
|
||||
|
||||
untrack(attemptId: string): RunDispatchLeaseRecord | undefined {
|
||||
const tracked = this.tracked.get(attemptId);
|
||||
if (!tracked) return undefined;
|
||||
this.tracked.delete(attemptId);
|
||||
this.clearTimer();
|
||||
this.schedule();
|
||||
return cloneLease(tracked.lease);
|
||||
}
|
||||
|
||||
leases(): RunDispatchLeaseRecord[] {
|
||||
return [...this.tracked.values()]
|
||||
.map(({ lease }) => cloneLease(lease))
|
||||
.sort((left, right) => left.attemptId.localeCompare(right.attemptId));
|
||||
}
|
||||
|
||||
async releaseAll(
|
||||
reason: Exclude<RunDispatchReleaseReason, 'lease_expired'> = 'shutdown',
|
||||
): Promise<RunDispatchLeaseRecord[]> {
|
||||
if (this.releasing) {
|
||||
throw new Error('Run lease release is already active');
|
||||
}
|
||||
this.releasing = true;
|
||||
this.clearTimer();
|
||||
const released: RunDispatchLeaseRecord[] = [];
|
||||
try {
|
||||
await this.inFlight;
|
||||
this.clearTimer();
|
||||
for (const tracked of [...this.tracked.values()]) {
|
||||
if (this.tracked.get(tracked.lease.attemptId) !== tracked) continue;
|
||||
try {
|
||||
const result = await this.client.release({
|
||||
runId: tracked.lease.runId,
|
||||
...this.fence(tracked.lease),
|
||||
reason,
|
||||
});
|
||||
this.assertRelease(tracked.lease, result.lease, reason);
|
||||
this.tracked.delete(tracked.lease.attemptId);
|
||||
released.push(cloneLease(result.lease));
|
||||
} catch (error) {
|
||||
if (error instanceof RunDispatchLeaseFenceRejectedError) {
|
||||
this.lose(tracked, 'fenced', error);
|
||||
} else {
|
||||
this.notifyError(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
this.releasing = false;
|
||||
this.schedule();
|
||||
}
|
||||
return released;
|
||||
}
|
||||
|
||||
async stop(): Promise<WorkerRunLeaseStopResult> {
|
||||
if (!this.started) return 'not_started';
|
||||
this.started = false;
|
||||
this.clearTimer();
|
||||
const deadline = Date.now() + this.stopTimeoutMs;
|
||||
if (!(await this.waitWithin(this.inFlight, deadline))) return 'timed_out';
|
||||
return 'stopped';
|
||||
}
|
||||
|
||||
private schedule(): void {
|
||||
if (
|
||||
!this.started ||
|
||||
this.releasing ||
|
||||
this.timer ||
|
||||
this.inFlight ||
|
||||
!this.tracked.size
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const nowMs = this.now();
|
||||
const nextAtMs = Math.min(
|
||||
...[...this.tracked.values()].map((tracked) =>
|
||||
Math.min(tracked.renewAtMs, tracked.lease.expiresAtMs),
|
||||
),
|
||||
);
|
||||
const timer = this.scheduler.setTimeout(() => {
|
||||
if (this.timer === timer) this.timer = undefined;
|
||||
this.runRenewals();
|
||||
}, Math.max(0, nextAtMs - nowMs));
|
||||
this.timer = timer;
|
||||
timer.unref?.();
|
||||
}
|
||||
|
||||
private clearTimer(): void {
|
||||
if (!this.timer) return;
|
||||
this.scheduler.clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
|
||||
private runRenewals(): void {
|
||||
if (!this.started || this.releasing || this.inFlight) return;
|
||||
const operation = this.renewDue()
|
||||
.catch((error) => this.notifyError(error))
|
||||
.then(() => undefined)
|
||||
.finally(() => {
|
||||
if (this.inFlight === operation) this.inFlight = undefined;
|
||||
this.schedule();
|
||||
});
|
||||
this.inFlight = operation;
|
||||
}
|
||||
|
||||
private async renewDue(): Promise<void> {
|
||||
const due = [...this.tracked.values()]
|
||||
.filter(
|
||||
(tracked) =>
|
||||
tracked.renewAtMs <= this.now() ||
|
||||
tracked.lease.expiresAtMs <= this.now(),
|
||||
)
|
||||
.sort((left, right) =>
|
||||
left.lease.attemptId.localeCompare(right.lease.attemptId),
|
||||
);
|
||||
for (const tracked of due) await this.renewOne(tracked);
|
||||
}
|
||||
|
||||
private async renewOne(tracked: TrackedLease): Promise<void> {
|
||||
if (this.tracked.get(tracked.lease.attemptId) !== tracked) return;
|
||||
const nowMs = this.now();
|
||||
if (tracked.lease.expiresAtMs <= nowMs) {
|
||||
this.lose(tracked, 'lease_expired');
|
||||
return;
|
||||
}
|
||||
const session = this.currentSessionProvider();
|
||||
try {
|
||||
this.assertCurrentSession(tracked.lease, session, nowMs);
|
||||
} catch (error) {
|
||||
const reason =
|
||||
session && this.sameWorkerSession(tracked.lease, session)
|
||||
? 'worker_unavailable'
|
||||
: 'worker_session_replaced';
|
||||
this.lose(tracked, reason, error);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const renewed = await this.client.renew(this.fence(tracked.lease));
|
||||
this.assertRenewal(tracked.lease, renewed, this.now());
|
||||
if (this.tracked.get(tracked.lease.attemptId) !== tracked) return;
|
||||
tracked.lease = cloneLease(renewed);
|
||||
tracked.renewAtMs = renewalTime(renewed);
|
||||
this.notifyRenewed(renewed);
|
||||
} catch (error) {
|
||||
if (error instanceof RunDispatchLeaseFenceRejectedError) {
|
||||
this.lose(tracked, 'fenced', error);
|
||||
return;
|
||||
}
|
||||
this.notifyError(error);
|
||||
const retryAtMs = this.now() + this.retryDelayMs;
|
||||
if (retryAtMs >= tracked.lease.expiresAtMs) {
|
||||
tracked.renewAtMs = tracked.lease.expiresAtMs;
|
||||
} else {
|
||||
tracked.renewAtMs = retryAtMs;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private assertRenewal(
|
||||
previous: RunDispatchLeaseRecord,
|
||||
renewed: RunDispatchLeaseRecord,
|
||||
nowMs: number,
|
||||
): void {
|
||||
assertRunDispatchLeaseRecord(renewed);
|
||||
if (
|
||||
renewed.status !== 'leased' ||
|
||||
!this.sameAuthority(previous, renewed) ||
|
||||
renewed.version !== previous.version + 1 ||
|
||||
renewed.renewedAtMs < previous.renewedAtMs ||
|
||||
renewed.expiresAtMs <= nowMs
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Control plane returned an invalid Run lease renewal',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private assertRelease(
|
||||
previous: RunDispatchLeaseRecord,
|
||||
released: RunDispatchLeaseRecord,
|
||||
reason: RunDispatchReleaseReason,
|
||||
): void {
|
||||
assertRunDispatchLeaseRecord(released);
|
||||
if (
|
||||
released.status !== 'released' ||
|
||||
!this.sameAuthority(previous, released) ||
|
||||
released.version !== previous.version + 1 ||
|
||||
released.releaseReason !== reason
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Control plane returned an invalid Run lease release',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private assertCurrentSession(
|
||||
lease: RunDispatchLeaseRecord,
|
||||
session: WorkerRecord | undefined,
|
||||
nowMs: number,
|
||||
): asserts session is WorkerRecord {
|
||||
if (!session || !this.sameWorkerSession(lease, session)) {
|
||||
throw new TypeError(
|
||||
'Run lease does not belong to the current Worker session',
|
||||
);
|
||||
}
|
||||
if (
|
||||
(session.status !== 'online' && session.status !== 'draining') ||
|
||||
session.leaseExpiresAtMs <= nowMs
|
||||
) {
|
||||
throw new TypeError('Current Worker session is unavailable');
|
||||
}
|
||||
}
|
||||
|
||||
private sameWorkerSession(
|
||||
lease: RunDispatchLeaseRecord,
|
||||
session: WorkerRecord,
|
||||
): boolean {
|
||||
return (
|
||||
lease.workerId === session.id &&
|
||||
lease.workerSessionId === session.sessionId &&
|
||||
lease.workerGeneration === session.generation
|
||||
);
|
||||
}
|
||||
|
||||
private sameAuthority(
|
||||
left: RunDispatchLeaseRecord,
|
||||
right: RunDispatchLeaseRecord,
|
||||
): boolean {
|
||||
return (
|
||||
left.attemptId === right.attemptId &&
|
||||
left.runId === right.runId &&
|
||||
left.workerId === right.workerId &&
|
||||
left.workerSessionId === right.workerSessionId &&
|
||||
left.workerGeneration === right.workerGeneration &&
|
||||
left.leaseGeneration === right.leaseGeneration &&
|
||||
left.leaseToken === right.leaseToken
|
||||
);
|
||||
}
|
||||
|
||||
private fence(lease: RunDispatchLeaseRecord) {
|
||||
return {
|
||||
attemptId: lease.attemptId,
|
||||
workerId: lease.workerId,
|
||||
workerSessionId: lease.workerSessionId,
|
||||
workerGeneration: lease.workerGeneration,
|
||||
leaseGeneration: lease.leaseGeneration,
|
||||
leaseToken: lease.leaseToken,
|
||||
expectedVersion: lease.version,
|
||||
};
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock.now();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new TypeError('Worker Run lease clock returned an invalid time');
|
||||
}
|
||||
return nowMs;
|
||||
}
|
||||
|
||||
private lose(
|
||||
tracked: TrackedLease,
|
||||
reason: WorkerRunLeaseLossReason,
|
||||
error?: unknown,
|
||||
): void {
|
||||
if (this.tracked.get(tracked.lease.attemptId) !== tracked) return;
|
||||
this.tracked.delete(tracked.lease.attemptId);
|
||||
try {
|
||||
this.onLost?.({
|
||||
lease: cloneLease(tracked.lease),
|
||||
reason,
|
||||
...(error === undefined ? {} : { error }),
|
||||
});
|
||||
} catch (callbackError) {
|
||||
this.notifyError(callbackError);
|
||||
}
|
||||
}
|
||||
|
||||
private notifyRenewed(lease: RunDispatchLeaseRecord): void {
|
||||
try {
|
||||
this.onRenewed?.(cloneLease(lease));
|
||||
} catch (error) {
|
||||
this.notifyError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private notifyError(error: unknown): void {
|
||||
try {
|
||||
this.onError?.(error);
|
||||
} catch {
|
||||
// Diagnostics must not create a renewal failure loop.
|
||||
}
|
||||
}
|
||||
|
||||
private async waitWithin(
|
||||
promise: Promise<unknown> | undefined,
|
||||
deadline: number,
|
||||
): Promise<boolean> {
|
||||
if (!promise) return true;
|
||||
const remainingMs = Math.max(0, deadline - Date.now());
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const completed = await Promise.race([
|
||||
promise.then(() => true),
|
||||
new Promise<false>((resolve) => {
|
||||
timeout = setTimeout(() => resolve(false), remainingMs);
|
||||
}),
|
||||
]);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
return completed;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user