mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 01:32:44 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,416 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
Sha256WorkerExecutionCompletionReceiptAuthenticator,
|
||||
} = require('../../back/runtime/adapters/crypto/sha256WorkerExecutionCompletionReceiptAuthenticator');
|
||||
const {
|
||||
WorkerExecutionOfferRecoveryCoordinator,
|
||||
} = require('../../back/runtime/application/workerExecutionOfferRecoveryCoordinator');
|
||||
const {
|
||||
WorkerExecutionOfferRecoveryReconciler,
|
||||
} = require('../../back/runtime/application/workerExecutionOfferRecoveryReconciler');
|
||||
const {
|
||||
cloneWorkerExecutionOfferJournalRecord,
|
||||
createWorkerExecutionOfferJournalRecord,
|
||||
} = require('../../back/runtime/domain/workerExecutionOffer');
|
||||
const {
|
||||
createWorkerExecutionCompletionReceiptAuthentication,
|
||||
} = require('../../back/runtime/domain/workerExecutionCompletionReceiptAuthentication');
|
||||
const {
|
||||
createExecutionSpecDigest,
|
||||
createRunDispatchOfferId,
|
||||
} = require('../../back/runtime/domain/runDispatchOffer');
|
||||
|
||||
const START = 1_760_800_000_000;
|
||||
const RUN_ID = '019f8200-0000-7000-8000-000000000001';
|
||||
const ATTEMPT_ID = '019f8200-0000-7000-8000-000000000002';
|
||||
const SESSION_A = '019f8200-0000-7000-8000-000000000003';
|
||||
const SESSION_B = '019f8200-0000-7000-8000-000000000004';
|
||||
const RECEIPT_TOKEN = 'receipt_capability_abcdefghijklmnopqrstuvwxyz012345';
|
||||
|
||||
function offer(overrides = {}) {
|
||||
const lease = {
|
||||
attemptId: ATTEMPT_ID,
|
||||
runId: RUN_ID,
|
||||
status: 'leased',
|
||||
version: 2,
|
||||
leaseGeneration: 1,
|
||||
workerId: 'worker-edge',
|
||||
workerSessionId: SESSION_A,
|
||||
workerGeneration: 1,
|
||||
leaseToken: 'lease_capability_abcdefghijklmnopqrstuvwxyz0123456',
|
||||
acquiredAtMs: START,
|
||||
renewedAtMs: START,
|
||||
expiresAtMs: START + 60_000,
|
||||
updatedAtMs: START,
|
||||
...overrides.lease,
|
||||
};
|
||||
const executionSpec = {
|
||||
runId: lease.runId,
|
||||
attemptId: lease.attemptId,
|
||||
projectId: 'default',
|
||||
taskId: 'task-1',
|
||||
taskRevision: 'revision-1',
|
||||
command: { kind: 'argv', file: '/bin/true', args: [] },
|
||||
environmentPolicy: 'isolated',
|
||||
terminationGraceMs: 1_000,
|
||||
};
|
||||
return {
|
||||
offerId: createRunDispatchOfferId(lease),
|
||||
executionSpecDigest: createExecutionSpecDigest(executionSpec),
|
||||
deliveryKind: 'lease_recovery',
|
||||
candidate: {
|
||||
runId: lease.runId,
|
||||
attemptId: lease.attemptId,
|
||||
projectId: 'default',
|
||||
taskId: 'task-1',
|
||||
taskRevision: 'revision-1',
|
||||
executorType: 'remote_worker',
|
||||
priority: 0,
|
||||
queuedAtMs: START,
|
||||
attemptCreatedAtMs: START,
|
||||
},
|
||||
worker: {
|
||||
id: lease.workerId,
|
||||
sessionId: lease.workerSessionId,
|
||||
generation: lease.workerGeneration,
|
||||
},
|
||||
lease,
|
||||
executionSpec,
|
||||
};
|
||||
}
|
||||
|
||||
function record(state = 'started') {
|
||||
const initial = createWorkerExecutionOfferJournalRecord(offer(), START);
|
||||
const authentication = createWorkerExecutionCompletionReceiptAuthentication({
|
||||
token: RECEIPT_TOKEN,
|
||||
callbackSequence: 1,
|
||||
});
|
||||
return cloneWorkerExecutionOfferJournalRecord({
|
||||
...initial,
|
||||
revision: 1,
|
||||
state,
|
||||
updatedAtMs: START + 10,
|
||||
executorHandle: 'ql3lp1.durable-handle',
|
||||
executorStartedAtMs: START + 5,
|
||||
completionReceiptCallbackSequence: authentication.callbackSequence,
|
||||
completionReceiptTokenDigest: authentication.tokenDigest,
|
||||
...(state === 'recovery_required'
|
||||
? { recoveryReason: 'launch_outcome_unknown' }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
function session(overrides = {}) {
|
||||
return {
|
||||
id: 'worker-edge',
|
||||
sessionId: SESSION_A,
|
||||
generation: 1,
|
||||
status: 'online',
|
||||
version: 0,
|
||||
capabilities: {
|
||||
architecture: 'arm64',
|
||||
operatingSystem: 'linux',
|
||||
executors: ['remote_worker'],
|
||||
runtimes: [{ name: 'node', version: '24.14.0' }],
|
||||
labels: {},
|
||||
capacity: { memoryBytes: 256 * 1024 * 1024 },
|
||||
features: ['direct_file_log'],
|
||||
},
|
||||
capabilitiesHash: 'a'.repeat(64),
|
||||
maxConcurrentRuns: 2,
|
||||
availableSlots: 1,
|
||||
registeredAtMs: START,
|
||||
lastHeartbeatAtMs: START,
|
||||
leaseExpiresAtMs: START + 120_000,
|
||||
updatedAtMs: START,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function receipt(overrides = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
runId: RUN_ID,
|
||||
attemptId: ATTEMPT_ID,
|
||||
callbackSequence: 1,
|
||||
token: RECEIPT_TOKEN,
|
||||
startedAtMs: START + 5,
|
||||
finishedAtMs: START + 50,
|
||||
exitCode: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
class MemoryJournal {
|
||||
constructor(value) {
|
||||
this.value = cloneWorkerExecutionOfferJournalRecord(value);
|
||||
this.replaceCalls = 0;
|
||||
this.failReplace = 0;
|
||||
}
|
||||
|
||||
async read(offerId) {
|
||||
if (this.value?.offer.offerId !== offerId) return undefined;
|
||||
return cloneWorkerExecutionOfferJournalRecord(this.value);
|
||||
}
|
||||
|
||||
async replace(value, expectedRevision) {
|
||||
this.replaceCalls += 1;
|
||||
if (this.failReplace > 0) {
|
||||
this.failReplace -= 1;
|
||||
throw new Error('simulated journal write failure');
|
||||
}
|
||||
assert.equal(this.value.revision, expectedRevision);
|
||||
assert.equal(value.revision, expectedRevision + 1);
|
||||
this.value = cloneWorkerExecutionOfferJournalRecord(value);
|
||||
}
|
||||
}
|
||||
|
||||
function fixture(options = {}) {
|
||||
const journal = options.journal || new MemoryJournal(record());
|
||||
const calls = [];
|
||||
let receiptValue = options.receiptValue;
|
||||
if (!Object.hasOwn(options, 'receiptValue')) receiptValue = receipt();
|
||||
let cleanupFailure = options.cleanupFailure;
|
||||
const receipts = {
|
||||
async read(attemptId) {
|
||||
calls.push(['read_receipt', attemptId]);
|
||||
if (receiptValue instanceof Error) throw receiptValue;
|
||||
if (options.readBarrier) await options.readBarrier;
|
||||
return receiptValue;
|
||||
},
|
||||
async remove(attemptId) {
|
||||
calls.push(['remove_receipt', attemptId]);
|
||||
if (cleanupFailure) {
|
||||
cleanupFailure = false;
|
||||
throw new Error('simulated receipt cleanup failure');
|
||||
}
|
||||
const removed = receiptValue !== undefined;
|
||||
receiptValue = undefined;
|
||||
return removed;
|
||||
},
|
||||
};
|
||||
const reconciler = new WorkerExecutionOfferRecoveryReconciler(
|
||||
receipts,
|
||||
new Sha256WorkerExecutionCompletionReceiptAuthenticator(),
|
||||
{
|
||||
executorType: 'local_process',
|
||||
async inspect(handle) {
|
||||
calls.push(['inspect', handle]);
|
||||
return options.inspection || { status: 'running', identityPid: 42 };
|
||||
},
|
||||
},
|
||||
{ clock: { now: () => START + 55 } },
|
||||
);
|
||||
const completion = {
|
||||
async complete(command) {
|
||||
calls.push(['complete', command]);
|
||||
if (options.completionError) throw options.completionError;
|
||||
return { status: options.completionStatus || 'applied' };
|
||||
},
|
||||
};
|
||||
const activation = {
|
||||
async acknowledgeRunning(command) {
|
||||
calls.push(['ack_running', command]);
|
||||
if (options.activationError) throw options.activationError;
|
||||
return {
|
||||
status: options.activationStatus || 'applied',
|
||||
lease: { ...journal.value.offer.lease, version: 3 },
|
||||
};
|
||||
},
|
||||
};
|
||||
const coordinator = new WorkerExecutionOfferRecoveryCoordinator(
|
||||
journal,
|
||||
reconciler,
|
||||
completion,
|
||||
activation,
|
||||
receipts,
|
||||
{
|
||||
currentSession: () =>
|
||||
Object.hasOwn(options, 'currentSession')
|
||||
? options.currentSession
|
||||
: session(),
|
||||
clock: { now: () => START + 60 },
|
||||
},
|
||||
);
|
||||
return { calls, coordinator, journal };
|
||||
}
|
||||
|
||||
test('submits a trusted completion before terminal journal and receipt cleanup', async () => {
|
||||
const context = fixture();
|
||||
const result = await context.coordinator.recover(
|
||||
context.journal.value.offer.offerId,
|
||||
);
|
||||
assert.equal(result.status, 'completion_acknowledged');
|
||||
assert.equal(result.receiptCleanup, 'removed');
|
||||
assert.equal(context.journal.value.state, 'completion_acknowledged');
|
||||
assert.equal(context.journal.value.completionAcknowledgedAtMs, START + 60);
|
||||
assert.deepEqual(
|
||||
context.calls.map(([name]) => name),
|
||||
['read_receipt', 'complete', 'remove_receipt'],
|
||||
);
|
||||
const command = context.calls.find(([name]) => name === 'complete')[1];
|
||||
assert.deepEqual(command.result, {
|
||||
outcome: 'succeeded',
|
||||
startedAtMs: START + 5,
|
||||
finishedAtMs: START + 50,
|
||||
exitCode: 0,
|
||||
});
|
||||
assert.equal(command.expectedLeaseVersion, 2);
|
||||
assert.doesNotMatch(
|
||||
JSON.stringify(result),
|
||||
/receipt_capability|lease_capability/,
|
||||
);
|
||||
});
|
||||
|
||||
test('keeps the receipt when terminal journal persistence fails and converges on replay', async () => {
|
||||
const journal = new MemoryJournal(record());
|
||||
journal.failReplace = 1;
|
||||
const first = fixture({ journal });
|
||||
await assert.rejects(
|
||||
first.coordinator.recover(journal.value.offer.offerId),
|
||||
/simulated journal write failure/,
|
||||
);
|
||||
assert.equal(journal.value.state, 'started');
|
||||
assert.equal(
|
||||
first.calls.filter(([name]) => name === 'remove_receipt').length,
|
||||
0,
|
||||
);
|
||||
|
||||
const replay = fixture({
|
||||
journal,
|
||||
completionStatus: 'already_terminal',
|
||||
});
|
||||
const result = await replay.coordinator.recover(journal.value.offer.offerId);
|
||||
assert.equal(result.status, 'completion_acknowledged');
|
||||
assert.equal(journal.value.state, 'completion_acknowledged');
|
||||
assert.equal(result.receiptCleanup, 'removed');
|
||||
});
|
||||
|
||||
test('keeps a terminal journal when receipt cleanup fails and retries cleanup only', async () => {
|
||||
const context = fixture({ cleanupFailure: true });
|
||||
const offerId = context.journal.value.offer.offerId;
|
||||
const first = await context.coordinator.recover(offerId);
|
||||
assert.equal(first.status, 'completion_acknowledged');
|
||||
assert.equal(first.receiptCleanup, 'pending');
|
||||
assert.equal(context.journal.value.state, 'completion_acknowledged');
|
||||
|
||||
const second = await context.coordinator.recover(offerId);
|
||||
assert.equal(second.status, 'already_completed');
|
||||
assert.equal(second.receiptCleanup, 'removed');
|
||||
assert.equal(context.calls.filter(([name]) => name === 'complete').length, 1);
|
||||
});
|
||||
|
||||
test('fenced or untrusted evidence cannot trigger a control-plane mutation', async () => {
|
||||
const cases = [
|
||||
fixture({
|
||||
currentSession: session({ sessionId: SESSION_B, generation: 2 }),
|
||||
}),
|
||||
fixture({ receiptValue: receipt({ token: 'x'.repeat(43) }) }),
|
||||
fixture({ receiptValue: new Error('EIO') }),
|
||||
];
|
||||
for (const context of cases) {
|
||||
const result = await context.coordinator.recover(
|
||||
context.journal.value.offer.offerId,
|
||||
);
|
||||
assert.equal(result.status, 'deferred');
|
||||
assert.equal(
|
||||
context.calls.some(([name]) =>
|
||||
['complete', 'ack_running', 'remove_receipt'].includes(name),
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.equal(context.journal.value.state, 'started');
|
||||
}
|
||||
});
|
||||
|
||||
test('preserves journal and receipt when the control plane fences completion', async () => {
|
||||
const context = fixture({
|
||||
completionError: new Error('simulated control-plane fence'),
|
||||
});
|
||||
await assert.rejects(
|
||||
context.coordinator.recover(context.journal.value.offer.offerId),
|
||||
/simulated control-plane fence/,
|
||||
);
|
||||
assert.equal(context.journal.value.state, 'started');
|
||||
assert.equal(context.journal.replaceCalls, 0);
|
||||
assert.equal(
|
||||
context.calls.filter(([name]) => name === 'remove_receipt').length,
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
test('recovers a running ACK only from a current started execution', async () => {
|
||||
const context = fixture({ receiptValue: undefined });
|
||||
const result = await context.coordinator.recover(
|
||||
context.journal.value.offer.offerId,
|
||||
);
|
||||
assert.equal(result.status, 'running_acknowledged');
|
||||
assert.equal(context.journal.value.state, 'running_acknowledged');
|
||||
assert.equal(context.journal.value.offer.lease.version, 3);
|
||||
assert.deepEqual(
|
||||
context.calls.map(([name]) => name),
|
||||
['read_receipt', 'inspect', 'ack_running'],
|
||||
);
|
||||
});
|
||||
|
||||
test('does not treat a fenced running record as recovered ownership', async () => {
|
||||
const context = fixture({
|
||||
journal: new MemoryJournal(record('running_acknowledged')),
|
||||
receiptValue: undefined,
|
||||
currentSession: session({ sessionId: SESSION_B, generation: 2 }),
|
||||
});
|
||||
const result = await context.coordinator.recover(
|
||||
context.journal.value.offer.offerId,
|
||||
);
|
||||
assert.equal(result.status, 'deferred');
|
||||
assert.equal(result.evidence.authority, 'session_fenced');
|
||||
assert.equal(
|
||||
context.calls.some(([name]) => name === 'ack_running'),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('records a terminal control-plane response without inventing completion', async () => {
|
||||
const context = fixture({
|
||||
receiptValue: undefined,
|
||||
activationStatus: 'already_terminal',
|
||||
});
|
||||
const result = await context.coordinator.recover(
|
||||
context.journal.value.offer.offerId,
|
||||
);
|
||||
assert.equal(result.status, 'control_plane_terminal');
|
||||
assert.equal(context.journal.value.state, 'recovery_required');
|
||||
assert.equal(context.journal.value.recoveryReason, 'control_plane_terminal');
|
||||
assert.equal(
|
||||
context.calls.some(([name]) => name === 'remove_receipt'),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('coalesces concurrent recovery and does nothing without a current session', async () => {
|
||||
let releaseRead;
|
||||
const readBarrier = new Promise((resolve) => {
|
||||
releaseRead = resolve;
|
||||
});
|
||||
const concurrent = fixture({ readBarrier });
|
||||
const offerId = concurrent.journal.value.offer.offerId;
|
||||
const first = concurrent.coordinator.recover(offerId);
|
||||
const second = concurrent.coordinator.recover(offerId);
|
||||
assert.equal(first, second);
|
||||
releaseRead();
|
||||
await Promise.all([first, second]);
|
||||
assert.equal(
|
||||
concurrent.calls.filter(([name]) => name === 'complete').length,
|
||||
1,
|
||||
);
|
||||
|
||||
const unavailable = fixture({ currentSession: undefined });
|
||||
const result = await unavailable.coordinator.recover(
|
||||
unavailable.journal.value.offer.offerId,
|
||||
);
|
||||
assert.equal(result.status, 'session_unavailable');
|
||||
assert.deepEqual(unavailable.calls, []);
|
||||
});
|
||||
Reference in New Issue
Block a user