mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,606 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
RemoteWorkerCompletionFenceRejectedError,
|
||||
RemoteWorkerCompletionUnavailableError,
|
||||
} = require('@qinglong/runtime-core/remote-worker-completion');
|
||||
const {
|
||||
createStepRunRecord,
|
||||
transitionStepRunRecord,
|
||||
} = require('@qinglong/runtime-core/step-run');
|
||||
const { PostgresRemoteWorkerCompletionRepository } = require('../dist/entrypoints/runtime');
|
||||
|
||||
const SESSION_ID = '018f5c64-9b9d-7f1a-8c2d-1234567890ac';
|
||||
const LEASE_TOKEN = 'worker_generated_lease_capability_0000000000000001';
|
||||
const LEASE_DIGEST = createHash('sha256').update(LEASE_TOKEN).digest('hex');
|
||||
const LOG_ARTIFACT_ID = `wlog-${'a'.repeat(30)}`;
|
||||
const ARTIFACT_SHA256 = 'b'.repeat(64);
|
||||
const CALLBACK_DIGEST = 'c'.repeat(64);
|
||||
|
||||
function fence() {
|
||||
return {
|
||||
workerId: 'worker-1',
|
||||
workerSessionId: SESSION_ID,
|
||||
workerGeneration: 2,
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
offerId: 'offer-1',
|
||||
leaseGeneration: 3,
|
||||
leaseToken: LEASE_TOKEN,
|
||||
expectedLeaseVersion: 4,
|
||||
};
|
||||
}
|
||||
|
||||
function uploadCommand() {
|
||||
return {
|
||||
...fence(),
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
byteLength: 10,
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
|
||||
function completionCommand(overrides = {}) {
|
||||
return {
|
||||
...fence(),
|
||||
callbackSequence: 1,
|
||||
callbackTokenDigest: CALLBACK_DIGEST,
|
||||
result: {
|
||||
outcome: 'succeeded',
|
||||
startedAtMs: 100,
|
||||
finishedAtMs: 200,
|
||||
exitCode: 0,
|
||||
},
|
||||
artifact: {
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
byteLength: 10,
|
||||
sha256: ARTIFACT_SHA256,
|
||||
truncated: false,
|
||||
},
|
||||
attemptEventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a1',
|
||||
runEventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a2',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function aggregate(overrides = {}) {
|
||||
return {
|
||||
runId: 'run-1',
|
||||
projectId: 'project-1',
|
||||
runStatus: 'dispatching',
|
||||
executionOwner: 'runtime',
|
||||
cancelRequestedAtMs: null,
|
||||
cancelReason: null,
|
||||
runErrorCode: null,
|
||||
runVersion: 3,
|
||||
eventSequence: 7,
|
||||
runCreatedAtMs: 40,
|
||||
runStartedAtMs: null,
|
||||
attemptId: 'attempt-1',
|
||||
attemptRunId: 'run-1',
|
||||
attemptStatus: 'starting',
|
||||
executorType: 'remote_worker',
|
||||
attemptWorkerId: 'worker-1',
|
||||
attemptWorkerSessionId: SESSION_ID,
|
||||
attemptWorkerGeneration: 2,
|
||||
attemptLeaseGeneration: 3,
|
||||
attemptLeaseVersion: 4,
|
||||
attemptLeaseTokenDigest: LEASE_DIGEST,
|
||||
attemptOfferId: 'offer-1',
|
||||
callbackSequence: 0,
|
||||
callbackTokenDigest: null,
|
||||
logArtifactId: null,
|
||||
attemptCreatedAtMs: 50,
|
||||
startedAtMs: null,
|
||||
finishedAtMs: null,
|
||||
exitCode: null,
|
||||
attemptErrorCode: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function lease(overrides = {}) {
|
||||
return {
|
||||
attemptId: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
leaseStatus: 'leased',
|
||||
leaseVersion: 4,
|
||||
leaseGeneration: 3,
|
||||
workerId: 'worker-1',
|
||||
workerSessionId: SESSION_ID,
|
||||
workerGeneration: 2,
|
||||
leaseTokenDigest: LEASE_DIGEST,
|
||||
offerId: 'offer-1',
|
||||
leaseExpiresAtMs: 2_000,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function replayPayload(command, overrides = {}) {
|
||||
return {
|
||||
attempt_id: command.attemptId,
|
||||
lease_generation: command.leaseGeneration,
|
||||
from_status: 'running',
|
||||
to_status: 'succeeded',
|
||||
callback_sequence: command.callbackSequence,
|
||||
callback_token_digest: command.callbackTokenDigest,
|
||||
worker_started_at_ms: command.result.startedAtMs,
|
||||
worker_finished_at_ms: command.result.finishedAtMs,
|
||||
exit_code: command.result.exitCode,
|
||||
log_artifact_id: command.artifact.logArtifactId,
|
||||
artifact_byte_length: command.artifact.byteLength,
|
||||
artifact_sha256: command.artifact.sha256,
|
||||
artifact_truncated: command.artifact.truncated,
|
||||
error_code: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function workflowReadyStep() {
|
||||
return createStepRunRecord({
|
||||
id: 'workflow-step-1',
|
||||
runId: 'run-1',
|
||||
stepKey: 'collect',
|
||||
kind: 'task',
|
||||
definitionRef: 'pkg:demo:alpha',
|
||||
definitionDigest: 'a'.repeat(64),
|
||||
required: true,
|
||||
initialStatus: 'ready',
|
||||
mutationId: 'workflow-step-created',
|
||||
createdAtMs: 60,
|
||||
});
|
||||
}
|
||||
|
||||
function workflowAggregate(stepRun, overrides = {}) {
|
||||
return aggregate({
|
||||
runStatus: 'running',
|
||||
runVersion: 4,
|
||||
eventSequence: 9,
|
||||
runStartedAtMs: 50,
|
||||
attemptStepRunId: stepRun.id,
|
||||
workflowAttemptId: 'attempt-1',
|
||||
workflowStepRunId: stepRun.id,
|
||||
admittedWorkflowStepVersion: 1,
|
||||
admittedWorkflowStepDigest: workflowReadyStep().stepRunDigest,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
function fixture(options = {}) {
|
||||
const calls = [];
|
||||
const client = {
|
||||
async query(sql, params = []) {
|
||||
const normalized = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ sql: normalized, params });
|
||||
if (
|
||||
normalized === 'BEGIN' ||
|
||||
normalized.startsWith('SET LOCAL') ||
|
||||
normalized === 'COMMIT' ||
|
||||
normalized === 'ROLLBACK' ||
|
||||
normalized.startsWith('SELECT pg_advisory_xact_lock')
|
||||
)
|
||||
return { rows: [], rowCount: 0 };
|
||||
if (normalized.includes('FROM "ql3"."worker_sessions"')) {
|
||||
return {
|
||||
rows: [
|
||||
options.worker ?? {
|
||||
workerId: 'worker-1',
|
||||
sessionId: SESSION_ID,
|
||||
workerGeneration: 2,
|
||||
workerStatus: 'online',
|
||||
workerLeaseExpiresAtMs: 2_000,
|
||||
},
|
||||
],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('INNER JOIN "ql3"."run_attempts"')) {
|
||||
return { rows: [options.aggregate ?? aggregate()], rowCount: 1 };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."step_runs"')) {
|
||||
return {
|
||||
rows: options.stepRun
|
||||
? [
|
||||
{
|
||||
workflowStepVersion: options.stepRun.version,
|
||||
workflowStepDigest: options.stepRun.stepRunDigest,
|
||||
workflowStepJson: options.stepRun,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
rowCount: options.stepRun ? 1 : 0,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."run_dispatch_leases"')) {
|
||||
return { rows: [options.lease ?? lease()], rowCount: 1 };
|
||||
}
|
||||
if (normalized.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: options.nowMs ?? 1_000 }], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
normalized.startsWith('SELECT payload') &&
|
||||
normalized.includes('FROM "ql3"."run_events"')
|
||||
) {
|
||||
return {
|
||||
rows: options.replayPayload
|
||||
? [{ payload: options.replayPayload }]
|
||||
: [],
|
||||
rowCount: options.replayPayload ? 1 : 0,
|
||||
};
|
||||
}
|
||||
if (
|
||||
normalized.startsWith('UPDATE "ql3"."run_dispatch_leases"') ||
|
||||
normalized.startsWith('UPDATE "ql3"."run_attempts"') ||
|
||||
normalized.startsWith('UPDATE "ql3"."runs"') ||
|
||||
normalized.startsWith('UPDATE "ql3"."step_runs"')
|
||||
)
|
||||
return { rows: [], rowCount: options.updateRowCount ?? 1 };
|
||||
if (
|
||||
normalized.startsWith('INSERT INTO "ql3"."run_events"') ||
|
||||
normalized.startsWith('INSERT INTO "ql3"."step_run_mutations"')
|
||||
) {
|
||||
if (options.failEvent) throw new Error('injected event failure');
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`Unexpected SQL: ${normalized}`);
|
||||
},
|
||||
release() {
|
||||
calls.push({ sql: 'RELEASE', params: [] });
|
||||
},
|
||||
};
|
||||
return {
|
||||
repository: new PostgresRemoteWorkerCompletionRepository({
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
}),
|
||||
calls,
|
||||
};
|
||||
}
|
||||
|
||||
test('authorizes Artifact upload only after the shared Attempt lock and DB clock', async () => {
|
||||
const { repository, calls } = fixture();
|
||||
await repository.authorizeArtifactUpload(uploadCommand());
|
||||
const index = (needle) => calls.findIndex(({ sql }) => sql.includes(needle));
|
||||
assert.ok(index('pg_advisory_xact_lock') < index('worker_sessions'));
|
||||
assert.ok(
|
||||
index('worker_sessions') < index('INNER JOIN "ql3"."run_attempts"'),
|
||||
);
|
||||
assert.ok(
|
||||
index('INNER JOIN "ql3"."run_attempts"') < index('run_dispatch_leases'),
|
||||
);
|
||||
assert.ok(index('run_dispatch_leases') < index('statement_timestamp()'));
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql === 'COMMIT'),
|
||||
true,
|
||||
);
|
||||
assert.equal(JSON.stringify(calls).includes(LEASE_TOKEN), false);
|
||||
assert.equal(JSON.stringify(calls).includes(LEASE_DIGEST), false);
|
||||
});
|
||||
|
||||
test('completes directly from the durable starting crash window in one transaction', async () => {
|
||||
const { repository, calls } = fixture();
|
||||
assert.deepEqual(await repository.complete(completionCommand()), {
|
||||
status: 'applied',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
callbackSequence: 1,
|
||||
});
|
||||
const attemptUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."run_attempts"'),
|
||||
);
|
||||
assert.equal(attemptUpdate.params[9], 'succeeded');
|
||||
assert.equal(attemptUpdate.params[10], 5);
|
||||
assert.equal(attemptUpdate.params[11], 1);
|
||||
assert.equal(attemptUpdate.params[12], CALLBACK_DIGEST);
|
||||
assert.equal(attemptUpdate.params[13], LOG_ARTIFACT_ID);
|
||||
assert.equal(attemptUpdate.params[14], 100);
|
||||
assert.equal(attemptUpdate.params[15], 1_000);
|
||||
assert.equal(attemptUpdate.params[16], 0);
|
||||
assert.equal(attemptUpdate.params[19], 'starting');
|
||||
assert.equal(
|
||||
calls.filter(({ sql }) => sql.startsWith('INSERT INTO "ql3"."run_events"'))
|
||||
.length,
|
||||
2,
|
||||
);
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql === 'COMMIT'),
|
||||
true,
|
||||
);
|
||||
assert.equal(JSON.stringify(calls).includes(LEASE_TOKEN), false);
|
||||
assert.equal(JSON.stringify(calls).includes(ARTIFACT_SHA256), true);
|
||||
});
|
||||
|
||||
test('completes a Workflow Task from the starting crash window without terminalizing its parent Run', async () => {
|
||||
const stepRun = workflowReadyStep();
|
||||
const { repository, calls } = fixture({
|
||||
stepRun,
|
||||
aggregate: workflowAggregate(stepRun),
|
||||
});
|
||||
assert.equal(
|
||||
(await repository.complete(completionCommand())).status,
|
||||
'applied',
|
||||
);
|
||||
|
||||
const runUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"'),
|
||||
);
|
||||
assert.match(runUpdate.sql, /SET version = \$2, event_sequence = \$3/);
|
||||
assert.equal(runUpdate.sql.includes('SET status ='), false);
|
||||
assert.deepEqual(runUpdate.params, ['run-1', 7, 12, 4]);
|
||||
|
||||
const stepUpdates = calls.filter(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."step_runs"'),
|
||||
);
|
||||
assert.equal(stepUpdates.length, 2);
|
||||
assert.equal(stepUpdates[0].params[0], 'running');
|
||||
assert.equal(stepUpdates[0].params[2], 1);
|
||||
assert.equal(stepUpdates[1].params[0], 'succeeded');
|
||||
assert.equal(stepUpdates[1].params[2], 1);
|
||||
assert.equal(
|
||||
calls.filter(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."step_run_mutations"'),
|
||||
).length,
|
||||
2,
|
||||
);
|
||||
|
||||
const attemptEvent = calls.find(
|
||||
({ sql, params }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"') &&
|
||||
params[3] === 'workflow.task_attempt.succeeded',
|
||||
);
|
||||
assert.equal(attemptEvent.params[2], 11);
|
||||
assert.equal(attemptEvent.params[7], stepRun.id);
|
||||
assert.match(attemptEvent.params[8], /"execution_scope":"workflow_task"/);
|
||||
assert.match(attemptEvent.params[8], /"step_run_id":"workflow-step-1"/);
|
||||
});
|
||||
|
||||
test('makes parent Workflow cancellation win over a successful stopped Worker completion', async () => {
|
||||
const ready = workflowReadyStep();
|
||||
const running = transitionStepRunRecord(ready, {
|
||||
expectedVersion: ready.version,
|
||||
expectedDigest: ready.stepRunDigest,
|
||||
mutationId: 'workflow-step-running-before-stop',
|
||||
to: 'running',
|
||||
atMs: 100,
|
||||
});
|
||||
const { repository, calls } = fixture({
|
||||
stepRun: running,
|
||||
aggregate: workflowAggregate(running, {
|
||||
attemptStatus: 'running',
|
||||
callbackSequence: 1,
|
||||
callbackTokenDigest: CALLBACK_DIGEST,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
cancelRequestedAtMs: 150,
|
||||
cancelReason: 'user',
|
||||
startedAtMs: 100,
|
||||
}),
|
||||
});
|
||||
assert.equal(
|
||||
(await repository.complete(completionCommand())).status,
|
||||
'applied',
|
||||
);
|
||||
|
||||
const attemptUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."run_attempts"'),
|
||||
);
|
||||
assert.equal(attemptUpdate.params[9], 'cancelled');
|
||||
assert.equal(attemptUpdate.params[17], 'EXECUTION_CANCELLED');
|
||||
const runUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"'),
|
||||
);
|
||||
assert.match(runUpdate.sql, /SET version = \$2, event_sequence = \$3/);
|
||||
assert.equal(runUpdate.sql.includes('SET status ='), false);
|
||||
const stepUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."step_runs"'),
|
||||
);
|
||||
assert.equal(stepUpdate.params[0], 'cancelled');
|
||||
const attemptEvent = calls.find(
|
||||
({ sql, params }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"') &&
|
||||
params[3] === 'workflow.task_attempt.cancelled',
|
||||
);
|
||||
assert.equal(attemptEvent.params[7], running.id);
|
||||
assert.match(attemptEvent.params[8], /"error_code":"EXECUTION_CANCELLED"/);
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql === 'COMMIT'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('makes timeout intent win over a successful Worker exit', async () => {
|
||||
const running = aggregate({
|
||||
runStatus: 'running',
|
||||
attemptStatus: 'running',
|
||||
cancelRequestedAtMs: 900,
|
||||
cancelReason: 'timeout',
|
||||
callbackSequence: 1,
|
||||
callbackTokenDigest: CALLBACK_DIGEST,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
runStartedAtMs: 100,
|
||||
startedAtMs: 100,
|
||||
});
|
||||
const { repository, calls } = fixture({ aggregate: running });
|
||||
await repository.complete(completionCommand());
|
||||
const attemptUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."run_attempts"'),
|
||||
);
|
||||
assert.equal(attemptUpdate.params[9], 'timed_out');
|
||||
assert.equal(attemptUpdate.params[17], 'EXECUTION_TIMED_OUT');
|
||||
const runUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"'),
|
||||
);
|
||||
assert.equal(runUpdate.params[1], 'timed_out');
|
||||
});
|
||||
|
||||
test('makes a Workflow Task deadline win without timing out its parent Run', async () => {
|
||||
const ready = workflowReadyStep();
|
||||
const running = transitionStepRunRecord(ready, {
|
||||
expectedVersion: ready.version,
|
||||
expectedDigest: ready.stepRunDigest,
|
||||
mutationId: 'workflow-step-running',
|
||||
to: 'running',
|
||||
atMs: 100,
|
||||
});
|
||||
const { repository, calls } = fixture({
|
||||
stepRun: running,
|
||||
aggregate: workflowAggregate(running, {
|
||||
attemptStatus: 'running',
|
||||
callbackSequence: 1,
|
||||
callbackTokenDigest: CALLBACK_DIGEST,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
deadlineAtMs: 950,
|
||||
startedAtMs: 100,
|
||||
}),
|
||||
});
|
||||
await repository.complete(completionCommand());
|
||||
|
||||
const attemptUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."run_attempts"'),
|
||||
);
|
||||
assert.equal(attemptUpdate.params[9], 'timed_out');
|
||||
assert.equal(attemptUpdate.params[17], 'EXECUTION_TIMED_OUT');
|
||||
const stepUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."step_runs"'),
|
||||
);
|
||||
assert.equal(stepUpdate.params[0], 'timed_out');
|
||||
assert.equal(stepUpdate.params[8], 'execution_timed_out');
|
||||
const runUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"'),
|
||||
);
|
||||
assert.equal(runUpdate.sql.includes('SET status ='), false);
|
||||
});
|
||||
|
||||
test('replays a completed Workflow Task from immutable Attempt state after its parent Run is cancelled', async () => {
|
||||
const command = completionCommand();
|
||||
const ready = workflowReadyStep();
|
||||
const running = transitionStepRunRecord(ready, {
|
||||
expectedVersion: ready.version,
|
||||
expectedDigest: ready.stepRunDigest,
|
||||
mutationId: 'workflow-step-running',
|
||||
to: 'running',
|
||||
atMs: 100,
|
||||
});
|
||||
const succeeded = transitionStepRunRecord(running, {
|
||||
expectedVersion: running.version,
|
||||
expectedDigest: running.stepRunDigest,
|
||||
mutationId: command.runEventId,
|
||||
to: 'succeeded',
|
||||
atMs: 1_000,
|
||||
});
|
||||
const aggregateRow = workflowAggregate(succeeded, {
|
||||
runStatus: 'cancelled',
|
||||
cancelRequestedAtMs: 1_200,
|
||||
cancelReason: 'user',
|
||||
runErrorCode: 'EXECUTION_CANCELLED',
|
||||
attemptStatus: 'succeeded',
|
||||
attemptLeaseVersion: 5,
|
||||
callbackSequence: 1,
|
||||
callbackTokenDigest: CALLBACK_DIGEST,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
startedAtMs: 100,
|
||||
finishedAtMs: 1_000,
|
||||
exitCode: 0,
|
||||
});
|
||||
const exact = fixture({
|
||||
aggregate: aggregateRow,
|
||||
stepRun: succeeded,
|
||||
lease: lease({ leaseStatus: 'completed', leaseVersion: 5 }),
|
||||
replayPayload: replayPayload(command, {
|
||||
execution_scope: 'workflow_task',
|
||||
step_run_id: succeeded.id,
|
||||
}),
|
||||
});
|
||||
assert.equal(
|
||||
(await exact.repository.complete(command)).status,
|
||||
'already_completed',
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts only an event-authenticated exact completed replay', async () => {
|
||||
const command = completionCommand();
|
||||
const terminal = aggregate({
|
||||
runStatus: 'succeeded',
|
||||
attemptStatus: 'succeeded',
|
||||
attemptLeaseVersion: 5,
|
||||
callbackSequence: 1,
|
||||
callbackTokenDigest: CALLBACK_DIGEST,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
runStartedAtMs: 100,
|
||||
startedAtMs: 100,
|
||||
finishedAtMs: 1_000,
|
||||
exitCode: 0,
|
||||
});
|
||||
const exact = fixture({
|
||||
aggregate: terminal,
|
||||
lease: lease({ leaseStatus: 'completed', leaseVersion: 5 }),
|
||||
replayPayload: replayPayload(command),
|
||||
});
|
||||
assert.equal(
|
||||
(await exact.repository.complete(command)).status,
|
||||
'already_completed',
|
||||
);
|
||||
|
||||
const drifted = fixture({
|
||||
aggregate: terminal,
|
||||
lease: lease({ leaseStatus: 'completed', leaseVersion: 5 }),
|
||||
replayPayload: replayPayload(command, { artifact_sha256: 'd'.repeat(64) }),
|
||||
});
|
||||
await assert.rejects(
|
||||
drifted.repository.complete(command),
|
||||
(error) =>
|
||||
error instanceof RemoteWorkerCompletionFenceRejectedError &&
|
||||
error.reason === 'replay_mismatch',
|
||||
);
|
||||
|
||||
const invalidOrigin = fixture({
|
||||
aggregate: terminal,
|
||||
lease: lease({ leaseStatus: 'completed', leaseVersion: 5 }),
|
||||
replayPayload: replayPayload(command, { from_status: 'queued' }),
|
||||
});
|
||||
await assert.rejects(
|
||||
invalidOrigin.repository.complete(command),
|
||||
(error) =>
|
||||
error instanceof RemoteWorkerCompletionFenceRejectedError &&
|
||||
error.reason === 'replay_mismatch',
|
||||
);
|
||||
});
|
||||
|
||||
test('fences future Worker timestamps and rolls storage failures back', async () => {
|
||||
const future = fixture();
|
||||
await assert.rejects(
|
||||
future.repository.complete(
|
||||
completionCommand({
|
||||
result: {
|
||||
outcome: 'succeeded',
|
||||
startedAtMs: 100,
|
||||
finishedAtMs: 302_000,
|
||||
exitCode: 0,
|
||||
},
|
||||
}),
|
||||
),
|
||||
(error) =>
|
||||
error instanceof RemoteWorkerCompletionFenceRejectedError &&
|
||||
error.reason === 'state_mismatch',
|
||||
);
|
||||
|
||||
const failed = fixture({ failEvent: true });
|
||||
await assert.rejects(
|
||||
failed.repository.complete(completionCommand()),
|
||||
(error) =>
|
||||
error instanceof RemoteWorkerCompletionUnavailableError &&
|
||||
error.cause?.message === 'injected event failure',
|
||||
);
|
||||
assert.equal(
|
||||
failed.calls.some(({ sql }) => sql === 'ROLLBACK'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
failed.calls.some(({ sql }) => sql === 'COMMIT'),
|
||||
false,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user