mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
417 lines
13 KiB
JavaScript
417 lines
13 KiB
JavaScript
require('ts-node/register/transpile-only');
|
|
|
|
const assert = require('node:assert/strict');
|
|
const { test } = require('node:test');
|
|
const {
|
|
Sha256WorkerExecutionCompletionReceiptAuthenticator,
|
|
} = require('../../back/runtime/adapters/crypto/sha256WorkerExecutionCompletionReceiptAuthenticator');
|
|
const {
|
|
WorkerExecutionOfferRecoveryCoordinator,
|
|
} = require('../../back/runtime/application/workerExecutionOfferRecoveryCoordinator');
|
|
const {
|
|
WorkerExecutionOfferRecoveryReconciler,
|
|
} = require('../../back/runtime/application/workerExecutionOfferRecoveryReconciler');
|
|
const {
|
|
cloneWorkerExecutionOfferJournalRecord,
|
|
createWorkerExecutionOfferJournalRecord,
|
|
} = require('../../back/runtime/domain/workerExecutionOffer');
|
|
const {
|
|
createWorkerExecutionCompletionReceiptAuthentication,
|
|
} = require('../../back/runtime/domain/workerExecutionCompletionReceiptAuthentication');
|
|
const {
|
|
createExecutionSpecDigest,
|
|
createRunDispatchOfferId,
|
|
} = require('../../back/runtime/domain/runDispatchOffer');
|
|
|
|
const START = 1_760_800_000_000;
|
|
const RUN_ID = '019f8200-0000-7000-8000-000000000001';
|
|
const ATTEMPT_ID = '019f8200-0000-7000-8000-000000000002';
|
|
const SESSION_A = '019f8200-0000-7000-8000-000000000003';
|
|
const SESSION_B = '019f8200-0000-7000-8000-000000000004';
|
|
const RECEIPT_TOKEN = 'receipt_capability_abcdefghijklmnopqrstuvwxyz012345';
|
|
|
|
function offer(overrides = {}) {
|
|
const lease = {
|
|
attemptId: ATTEMPT_ID,
|
|
runId: RUN_ID,
|
|
status: 'leased',
|
|
version: 2,
|
|
leaseGeneration: 1,
|
|
workerId: 'worker-edge',
|
|
workerSessionId: SESSION_A,
|
|
workerGeneration: 1,
|
|
leaseToken: 'lease_capability_abcdefghijklmnopqrstuvwxyz0123456',
|
|
acquiredAtMs: START,
|
|
renewedAtMs: START,
|
|
expiresAtMs: START + 60_000,
|
|
updatedAtMs: START,
|
|
...overrides.lease,
|
|
};
|
|
const executionSpec = {
|
|
runId: lease.runId,
|
|
attemptId: lease.attemptId,
|
|
projectId: 'default',
|
|
taskId: 'task-1',
|
|
taskRevision: 'revision-1',
|
|
command: { kind: 'argv', file: '/bin/true', args: [] },
|
|
environmentPolicy: 'isolated',
|
|
terminationGraceMs: 1_000,
|
|
};
|
|
return {
|
|
offerId: createRunDispatchOfferId(lease),
|
|
executionSpecDigest: createExecutionSpecDigest(executionSpec),
|
|
deliveryKind: 'lease_recovery',
|
|
candidate: {
|
|
runId: lease.runId,
|
|
attemptId: lease.attemptId,
|
|
projectId: 'default',
|
|
taskId: 'task-1',
|
|
taskRevision: 'revision-1',
|
|
executorType: 'remote_worker',
|
|
priority: 0,
|
|
queuedAtMs: START,
|
|
attemptCreatedAtMs: START,
|
|
},
|
|
worker: {
|
|
id: lease.workerId,
|
|
sessionId: lease.workerSessionId,
|
|
generation: lease.workerGeneration,
|
|
},
|
|
lease,
|
|
executionSpec,
|
|
};
|
|
}
|
|
|
|
function record(state = 'started') {
|
|
const initial = createWorkerExecutionOfferJournalRecord(offer(), START);
|
|
const authentication = createWorkerExecutionCompletionReceiptAuthentication({
|
|
token: RECEIPT_TOKEN,
|
|
callbackSequence: 1,
|
|
});
|
|
return cloneWorkerExecutionOfferJournalRecord({
|
|
...initial,
|
|
revision: 1,
|
|
state,
|
|
updatedAtMs: START + 10,
|
|
executorHandle: 'ql3lp1.durable-handle',
|
|
executorStartedAtMs: START + 5,
|
|
completionReceiptCallbackSequence: authentication.callbackSequence,
|
|
completionReceiptTokenDigest: authentication.tokenDigest,
|
|
...(state === 'recovery_required'
|
|
? { recoveryReason: 'launch_outcome_unknown' }
|
|
: {}),
|
|
});
|
|
}
|
|
|
|
function session(overrides = {}) {
|
|
return {
|
|
id: 'worker-edge',
|
|
sessionId: SESSION_A,
|
|
generation: 1,
|
|
status: 'online',
|
|
version: 0,
|
|
capabilities: {
|
|
architecture: 'arm64',
|
|
operatingSystem: 'linux',
|
|
executors: ['remote_worker'],
|
|
runtimes: [{ name: 'node', version: '24.14.0' }],
|
|
labels: {},
|
|
capacity: { memoryBytes: 256 * 1024 * 1024 },
|
|
features: ['direct_file_log'],
|
|
},
|
|
capabilitiesHash: 'a'.repeat(64),
|
|
maxConcurrentRuns: 2,
|
|
availableSlots: 1,
|
|
registeredAtMs: START,
|
|
lastHeartbeatAtMs: START,
|
|
leaseExpiresAtMs: START + 120_000,
|
|
updatedAtMs: START,
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
function receipt(overrides = {}) {
|
|
return {
|
|
schemaVersion: 1,
|
|
runId: RUN_ID,
|
|
attemptId: ATTEMPT_ID,
|
|
callbackSequence: 1,
|
|
token: RECEIPT_TOKEN,
|
|
startedAtMs: START + 5,
|
|
finishedAtMs: START + 50,
|
|
exitCode: 0,
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
class MemoryJournal {
|
|
constructor(value) {
|
|
this.value = cloneWorkerExecutionOfferJournalRecord(value);
|
|
this.replaceCalls = 0;
|
|
this.failReplace = 0;
|
|
}
|
|
|
|
async read(offerId) {
|
|
if (this.value?.offer.offerId !== offerId) return undefined;
|
|
return cloneWorkerExecutionOfferJournalRecord(this.value);
|
|
}
|
|
|
|
async replace(value, expectedRevision) {
|
|
this.replaceCalls += 1;
|
|
if (this.failReplace > 0) {
|
|
this.failReplace -= 1;
|
|
throw new Error('simulated journal write failure');
|
|
}
|
|
assert.equal(this.value.revision, expectedRevision);
|
|
assert.equal(value.revision, expectedRevision + 1);
|
|
this.value = cloneWorkerExecutionOfferJournalRecord(value);
|
|
}
|
|
}
|
|
|
|
function fixture(options = {}) {
|
|
const journal = options.journal || new MemoryJournal(record());
|
|
const calls = [];
|
|
let receiptValue = options.receiptValue;
|
|
if (!Object.hasOwn(options, 'receiptValue')) receiptValue = receipt();
|
|
let cleanupFailure = options.cleanupFailure;
|
|
const receipts = {
|
|
async read(attemptId) {
|
|
calls.push(['read_receipt', attemptId]);
|
|
if (receiptValue instanceof Error) throw receiptValue;
|
|
if (options.readBarrier) await options.readBarrier;
|
|
return receiptValue;
|
|
},
|
|
async remove(attemptId) {
|
|
calls.push(['remove_receipt', attemptId]);
|
|
if (cleanupFailure) {
|
|
cleanupFailure = false;
|
|
throw new Error('simulated receipt cleanup failure');
|
|
}
|
|
const removed = receiptValue !== undefined;
|
|
receiptValue = undefined;
|
|
return removed;
|
|
},
|
|
};
|
|
const reconciler = new WorkerExecutionOfferRecoveryReconciler(
|
|
receipts,
|
|
new Sha256WorkerExecutionCompletionReceiptAuthenticator(),
|
|
{
|
|
executorType: 'local_process',
|
|
async inspect(handle) {
|
|
calls.push(['inspect', handle]);
|
|
return options.inspection || { status: 'running', identityPid: 42 };
|
|
},
|
|
},
|
|
{ clock: { now: () => START + 55 } },
|
|
);
|
|
const completion = {
|
|
async complete(command) {
|
|
calls.push(['complete', command]);
|
|
if (options.completionError) throw options.completionError;
|
|
return { status: options.completionStatus || 'applied' };
|
|
},
|
|
};
|
|
const activation = {
|
|
async acknowledgeRunning(command) {
|
|
calls.push(['ack_running', command]);
|
|
if (options.activationError) throw options.activationError;
|
|
return {
|
|
status: options.activationStatus || 'applied',
|
|
lease: { ...journal.value.offer.lease, version: 3 },
|
|
};
|
|
},
|
|
};
|
|
const coordinator = new WorkerExecutionOfferRecoveryCoordinator(
|
|
journal,
|
|
reconciler,
|
|
completion,
|
|
activation,
|
|
receipts,
|
|
{
|
|
currentSession: () =>
|
|
Object.hasOwn(options, 'currentSession')
|
|
? options.currentSession
|
|
: session(),
|
|
clock: { now: () => START + 60 },
|
|
},
|
|
);
|
|
return { calls, coordinator, journal };
|
|
}
|
|
|
|
test('submits a trusted completion before terminal journal and receipt cleanup', async () => {
|
|
const context = fixture();
|
|
const result = await context.coordinator.recover(
|
|
context.journal.value.offer.offerId,
|
|
);
|
|
assert.equal(result.status, 'completion_acknowledged');
|
|
assert.equal(result.receiptCleanup, 'removed');
|
|
assert.equal(context.journal.value.state, 'completion_acknowledged');
|
|
assert.equal(context.journal.value.completionAcknowledgedAtMs, START + 60);
|
|
assert.deepEqual(
|
|
context.calls.map(([name]) => name),
|
|
['read_receipt', 'complete', 'remove_receipt'],
|
|
);
|
|
const command = context.calls.find(([name]) => name === 'complete')[1];
|
|
assert.deepEqual(command.result, {
|
|
outcome: 'succeeded',
|
|
startedAtMs: START + 5,
|
|
finishedAtMs: START + 50,
|
|
exitCode: 0,
|
|
});
|
|
assert.equal(command.expectedLeaseVersion, 2);
|
|
assert.doesNotMatch(
|
|
JSON.stringify(result),
|
|
/receipt_capability|lease_capability/,
|
|
);
|
|
});
|
|
|
|
test('keeps the receipt when terminal journal persistence fails and converges on replay', async () => {
|
|
const journal = new MemoryJournal(record());
|
|
journal.failReplace = 1;
|
|
const first = fixture({ journal });
|
|
await assert.rejects(
|
|
first.coordinator.recover(journal.value.offer.offerId),
|
|
/simulated journal write failure/,
|
|
);
|
|
assert.equal(journal.value.state, 'started');
|
|
assert.equal(
|
|
first.calls.filter(([name]) => name === 'remove_receipt').length,
|
|
0,
|
|
);
|
|
|
|
const replay = fixture({
|
|
journal,
|
|
completionStatus: 'already_terminal',
|
|
});
|
|
const result = await replay.coordinator.recover(journal.value.offer.offerId);
|
|
assert.equal(result.status, 'completion_acknowledged');
|
|
assert.equal(journal.value.state, 'completion_acknowledged');
|
|
assert.equal(result.receiptCleanup, 'removed');
|
|
});
|
|
|
|
test('keeps a terminal journal when receipt cleanup fails and retries cleanup only', async () => {
|
|
const context = fixture({ cleanupFailure: true });
|
|
const offerId = context.journal.value.offer.offerId;
|
|
const first = await context.coordinator.recover(offerId);
|
|
assert.equal(first.status, 'completion_acknowledged');
|
|
assert.equal(first.receiptCleanup, 'pending');
|
|
assert.equal(context.journal.value.state, 'completion_acknowledged');
|
|
|
|
const second = await context.coordinator.recover(offerId);
|
|
assert.equal(second.status, 'already_completed');
|
|
assert.equal(second.receiptCleanup, 'removed');
|
|
assert.equal(context.calls.filter(([name]) => name === 'complete').length, 1);
|
|
});
|
|
|
|
test('fenced or untrusted evidence cannot trigger a control-plane mutation', async () => {
|
|
const cases = [
|
|
fixture({
|
|
currentSession: session({ sessionId: SESSION_B, generation: 2 }),
|
|
}),
|
|
fixture({ receiptValue: receipt({ token: 'x'.repeat(43) }) }),
|
|
fixture({ receiptValue: new Error('EIO') }),
|
|
];
|
|
for (const context of cases) {
|
|
const result = await context.coordinator.recover(
|
|
context.journal.value.offer.offerId,
|
|
);
|
|
assert.equal(result.status, 'deferred');
|
|
assert.equal(
|
|
context.calls.some(([name]) =>
|
|
['complete', 'ack_running', 'remove_receipt'].includes(name),
|
|
),
|
|
false,
|
|
);
|
|
assert.equal(context.journal.value.state, 'started');
|
|
}
|
|
});
|
|
|
|
test('preserves journal and receipt when the control plane fences completion', async () => {
|
|
const context = fixture({
|
|
completionError: new Error('simulated control-plane fence'),
|
|
});
|
|
await assert.rejects(
|
|
context.coordinator.recover(context.journal.value.offer.offerId),
|
|
/simulated control-plane fence/,
|
|
);
|
|
assert.equal(context.journal.value.state, 'started');
|
|
assert.equal(context.journal.replaceCalls, 0);
|
|
assert.equal(
|
|
context.calls.filter(([name]) => name === 'remove_receipt').length,
|
|
0,
|
|
);
|
|
});
|
|
|
|
test('recovers a running ACK only from a current started execution', async () => {
|
|
const context = fixture({ receiptValue: undefined });
|
|
const result = await context.coordinator.recover(
|
|
context.journal.value.offer.offerId,
|
|
);
|
|
assert.equal(result.status, 'running_acknowledged');
|
|
assert.equal(context.journal.value.state, 'running_acknowledged');
|
|
assert.equal(context.journal.value.offer.lease.version, 3);
|
|
assert.deepEqual(
|
|
context.calls.map(([name]) => name),
|
|
['read_receipt', 'inspect', 'ack_running'],
|
|
);
|
|
});
|
|
|
|
test('does not treat a fenced running record as recovered ownership', async () => {
|
|
const context = fixture({
|
|
journal: new MemoryJournal(record('running_acknowledged')),
|
|
receiptValue: undefined,
|
|
currentSession: session({ sessionId: SESSION_B, generation: 2 }),
|
|
});
|
|
const result = await context.coordinator.recover(
|
|
context.journal.value.offer.offerId,
|
|
);
|
|
assert.equal(result.status, 'deferred');
|
|
assert.equal(result.evidence.authority, 'session_fenced');
|
|
assert.equal(
|
|
context.calls.some(([name]) => name === 'ack_running'),
|
|
false,
|
|
);
|
|
});
|
|
|
|
test('records a terminal control-plane response without inventing completion', async () => {
|
|
const context = fixture({
|
|
receiptValue: undefined,
|
|
activationStatus: 'already_terminal',
|
|
});
|
|
const result = await context.coordinator.recover(
|
|
context.journal.value.offer.offerId,
|
|
);
|
|
assert.equal(result.status, 'control_plane_terminal');
|
|
assert.equal(context.journal.value.state, 'recovery_required');
|
|
assert.equal(context.journal.value.recoveryReason, 'control_plane_terminal');
|
|
assert.equal(
|
|
context.calls.some(([name]) => name === 'remove_receipt'),
|
|
false,
|
|
);
|
|
});
|
|
|
|
test('coalesces concurrent recovery and does nothing without a current session', async () => {
|
|
let releaseRead;
|
|
const readBarrier = new Promise((resolve) => {
|
|
releaseRead = resolve;
|
|
});
|
|
const concurrent = fixture({ readBarrier });
|
|
const offerId = concurrent.journal.value.offer.offerId;
|
|
const first = concurrent.coordinator.recover(offerId);
|
|
const second = concurrent.coordinator.recover(offerId);
|
|
assert.equal(first, second);
|
|
releaseRead();
|
|
await Promise.all([first, second]);
|
|
assert.equal(
|
|
concurrent.calls.filter(([name]) => name === 'complete').length,
|
|
1,
|
|
);
|
|
|
|
const unavailable = fixture({ currentSession: undefined });
|
|
const result = await unavailable.coordinator.recover(
|
|
unavailable.journal.value.offer.offerId,
|
|
);
|
|
assert.equal(result.status, 'session_unavailable');
|
|
assert.deepEqual(unavailable.calls, []);
|
|
});
|