mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,437 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
migrateLocalSqlitePath,
|
||||
openLocalSqliteRuntimeDatabase,
|
||||
} = require('@qinglong/local-sqlite');
|
||||
const {
|
||||
LocalCompletionReceiptProcessor,
|
||||
LocalExecutionControlCoordinator,
|
||||
LocalExecutionControlLifecycle,
|
||||
LocalExecutionControlScanner,
|
||||
} = require('../dist/control');
|
||||
|
||||
const IDS = Object.freeze({
|
||||
completionRun: '019f7130-0000-7000-8000-000000000001',
|
||||
completionAttempt: '019f7130-0000-7000-8000-000000000002',
|
||||
deadlineRun: '019f7130-0000-7000-8000-000000000003',
|
||||
deadlineAttempt: '019f7130-0000-7000-8000-000000000004',
|
||||
cancelRun: '019f7130-0000-7000-8000-000000000005',
|
||||
cancelAttempt: '019f7130-0000-7000-8000-000000000006',
|
||||
shutdownRun: '019f7130-0000-7000-8000-000000000007',
|
||||
shutdownAttempt: '019f7130-0000-7000-8000-000000000008',
|
||||
});
|
||||
const TOKEN = 'A'.repeat(32);
|
||||
|
||||
function eventIds() {
|
||||
let value = 0;
|
||||
return () => `control-event-${++value}`;
|
||||
}
|
||||
|
||||
function receiptStore() {
|
||||
const receipts = new Map();
|
||||
return {
|
||||
receipts,
|
||||
removed: [],
|
||||
async publish(receipt) {
|
||||
receipts.set(receipt.attemptId, receipt);
|
||||
},
|
||||
async read(attemptId) {
|
||||
return receipts.get(attemptId);
|
||||
},
|
||||
async remove(attemptId) {
|
||||
this.removed.push(attemptId);
|
||||
return receipts.delete(attemptId);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function fixture(t) {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-control-'));
|
||||
const databasePath = path.join(directory, 'qinglong3.sqlite');
|
||||
await migrateLocalSqlitePath({ databasePath, profile: 'edge' });
|
||||
const runtime = await openLocalSqliteRuntimeDatabase({
|
||||
databasePath,
|
||||
profile: 'edge',
|
||||
});
|
||||
t.after(async () => {
|
||||
await runtime.close();
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
return runtime;
|
||||
}
|
||||
|
||||
async function insertActive(runtime, options) {
|
||||
await runtime.runRepository.transaction(async (transaction) => {
|
||||
await transaction.insertRun({
|
||||
id: options.runId,
|
||||
projectId: 'default',
|
||||
taskId: 'task-1',
|
||||
taskRevision: 'revision-1',
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
status: options.runStatus || 'running',
|
||||
version: 0,
|
||||
eventSequence: 0,
|
||||
priority: 0,
|
||||
createdAtMs: 1,
|
||||
startedAtMs: 2,
|
||||
...(options.cancelRequestedAtMs === undefined
|
||||
? {}
|
||||
: {
|
||||
cancelRequestedAtMs: options.cancelRequestedAtMs,
|
||||
cancelReason: options.cancelReason,
|
||||
}),
|
||||
});
|
||||
await transaction.insertAttempt({
|
||||
id: options.attemptId,
|
||||
runId: options.runId,
|
||||
attempt: 1,
|
||||
status: options.attemptStatus || 'running',
|
||||
executorType: 'local_process',
|
||||
executorHandle: `handle:${options.attemptId}`,
|
||||
pid: 1234,
|
||||
callbackTokenHash: createHash('sha256').update(TOKEN).digest('hex'),
|
||||
callbackSequence: options.callbackSequence || 0,
|
||||
createdAtMs: 1,
|
||||
startedAtMs: 2,
|
||||
...(options.deadlineAtMs === undefined
|
||||
? {}
|
||||
: { deadlineAtMs: options.deadlineAtMs }),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test('authenticates one receipt and commits Attempt then Run terminal facts', async (t) => {
|
||||
const runtime = await fixture(t);
|
||||
const store = receiptStore();
|
||||
await insertActive(runtime, {
|
||||
runId: IDS.completionRun,
|
||||
attemptId: IDS.completionAttempt,
|
||||
});
|
||||
store.receipts.set(IDS.completionAttempt, {
|
||||
schemaVersion: 1,
|
||||
runId: IDS.completionRun,
|
||||
attemptId: IDS.completionAttempt,
|
||||
callbackSequence: 1,
|
||||
token: TOKEN,
|
||||
startedAtMs: 2,
|
||||
finishedAtMs: 10,
|
||||
exitCode: 0,
|
||||
});
|
||||
const processor = new LocalCompletionReceiptProcessor(
|
||||
runtime.runRepository,
|
||||
store,
|
||||
{ clock: { now: () => 10 }, createEventId: eventIds() },
|
||||
);
|
||||
assert.equal(await processor.process(IDS.completionAttempt), 'completed');
|
||||
assert.equal(
|
||||
(await runtime.runRepository.findRunById(IDS.completionRun)).status,
|
||||
'succeeded',
|
||||
);
|
||||
assert.equal(
|
||||
(await runtime.runRepository.findAttemptById(IDS.completionAttempt)).status,
|
||||
'succeeded',
|
||||
);
|
||||
assert.deepEqual(
|
||||
(await runtime.runRepository.listEvents(IDS.completionRun)).map(
|
||||
(event) => event.type,
|
||||
),
|
||||
['attempt.succeeded', 'run.succeeded'],
|
||||
);
|
||||
assert.deepEqual(store.removed, [IDS.completionAttempt]);
|
||||
});
|
||||
|
||||
test('discovers due deadlines and cancellation intents through stable SQLite pages', async (t) => {
|
||||
const runtime = await fixture(t);
|
||||
await insertActive(runtime, {
|
||||
runId: IDS.deadlineRun,
|
||||
attemptId: IDS.deadlineAttempt,
|
||||
deadlineAtMs: 50,
|
||||
});
|
||||
await insertActive(runtime, {
|
||||
runId: IDS.cancelRun,
|
||||
attemptId: IDS.cancelAttempt,
|
||||
cancelRequestedAtMs: 40,
|
||||
cancelReason: 'user',
|
||||
deadlineAtMs: 30,
|
||||
});
|
||||
const first =
|
||||
await runtime.executionControl.listLocalExecutionControlCandidates({
|
||||
observedAtMs: 100,
|
||||
limit: 1,
|
||||
});
|
||||
assert.equal(first.truncated, true);
|
||||
assert.deepEqual(first.candidates[0], {
|
||||
kind: 'cancellation',
|
||||
runId: IDS.cancelRun,
|
||||
attemptId: IDS.cancelAttempt,
|
||||
dueAtMs: 40,
|
||||
cancelReason: 'user',
|
||||
});
|
||||
const second =
|
||||
await runtime.executionControl.listLocalExecutionControlCandidates({
|
||||
observedAtMs: 100,
|
||||
limit: 1,
|
||||
after: first.nextCursor,
|
||||
});
|
||||
assert.equal(second.truncated, false);
|
||||
assert.deepEqual(second.candidates[0], {
|
||||
kind: 'deadline',
|
||||
runId: IDS.deadlineRun,
|
||||
attemptId: IDS.deadlineAttempt,
|
||||
dueAtMs: 50,
|
||||
});
|
||||
});
|
||||
|
||||
test('turns deadline and user cancellation into exact terminal aggregates', async (t) => {
|
||||
const runtime = await fixture(t);
|
||||
await insertActive(runtime, {
|
||||
runId: IDS.deadlineRun,
|
||||
attemptId: IDS.deadlineAttempt,
|
||||
deadlineAtMs: 50,
|
||||
});
|
||||
await insertActive(runtime, {
|
||||
runId: IDS.cancelRun,
|
||||
attemptId: IDS.cancelAttempt,
|
||||
cancelRequestedAtMs: 40,
|
||||
cancelReason: 'user',
|
||||
});
|
||||
const stopped = [];
|
||||
const store = receiptStore();
|
||||
const completion = new LocalCompletionReceiptProcessor(
|
||||
runtime.runRepository,
|
||||
store,
|
||||
{ clock: { now: () => 100 }, createEventId: eventIds() },
|
||||
);
|
||||
const coordinator = new LocalExecutionControlCoordinator(
|
||||
runtime.runRepository,
|
||||
completion,
|
||||
{
|
||||
async stop(handle) {
|
||||
stopped.push(handle);
|
||||
return { status: 'stopped', signal: 'SIGTERM' };
|
||||
},
|
||||
},
|
||||
{ clock: { now: () => 100 }, createEventId: eventIds() },
|
||||
);
|
||||
const scanner = new LocalExecutionControlScanner(
|
||||
runtime.executionControl,
|
||||
coordinator,
|
||||
{ now: () => 100 },
|
||||
);
|
||||
const summary = await scanner.scan({ limit: 8 });
|
||||
assert.equal(summary.scanned, 2);
|
||||
assert.equal(summary.terminal, 2);
|
||||
assert.equal(
|
||||
(await runtime.runRepository.findRunById(IDS.deadlineRun)).status,
|
||||
'timed_out',
|
||||
);
|
||||
assert.equal(
|
||||
(await runtime.runRepository.findRunById(IDS.cancelRun)).status,
|
||||
'cancelled',
|
||||
);
|
||||
assert.equal(stopped.length, 2);
|
||||
});
|
||||
|
||||
test('routes a Workflow Task deadline to Step scope without cancelling its parent', async () => {
|
||||
const run = {
|
||||
id: 'workflow-run',
|
||||
projectId: 'default',
|
||||
taskId: 'workflow',
|
||||
taskRevision: 'revision-1',
|
||||
triggerType: 'system',
|
||||
executionOrigin: 'system',
|
||||
executionOwner: 'runtime',
|
||||
status: 'running',
|
||||
version: 7,
|
||||
eventSequence: 7,
|
||||
priority: 0,
|
||||
createdAtMs: 1,
|
||||
startedAtMs: 2,
|
||||
};
|
||||
const attempt = {
|
||||
id: 'workflow-attempt',
|
||||
runId: run.id,
|
||||
stepRunId: 'workflow-step',
|
||||
attempt: 1,
|
||||
status: 'running',
|
||||
executorType: 'local_process',
|
||||
executorHandle: 'workflow-handle',
|
||||
pid: 321,
|
||||
callbackTokenHash: 'a'.repeat(64),
|
||||
callbackSequence: 0,
|
||||
deadlineAtMs: 50,
|
||||
createdAtMs: 1,
|
||||
startedAtMs: 2,
|
||||
};
|
||||
let latestReads = 0;
|
||||
const repository = {
|
||||
transaction(work) {
|
||||
return work({
|
||||
findRunById: async (runId) => (runId === run.id ? run : null),
|
||||
findAttemptById: async (attemptId) =>
|
||||
attemptId === attempt.id ? attempt : null,
|
||||
findLatestAttemptByRunId: async () => {
|
||||
latestReads += 1;
|
||||
return attempt;
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
const workflowCalls = [];
|
||||
const coordinator = new LocalExecutionControlCoordinator(
|
||||
repository,
|
||||
{ process: async () => 'missing' },
|
||||
{ stop: async () => ({ status: 'stopped', signal: 'SIGTERM' }) },
|
||||
{
|
||||
clock: { now: () => 100 },
|
||||
createEventId: eventIds(),
|
||||
workflowTasks: {
|
||||
async requestTimeout(command) {
|
||||
workflowCalls.push(['timeout', command]);
|
||||
return 'requested';
|
||||
},
|
||||
async recordControlTerminal(command) {
|
||||
workflowCalls.push(['terminal', command]);
|
||||
return 'terminal';
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
await coordinator.process({
|
||||
kind: 'deadline',
|
||||
runId: run.id,
|
||||
attemptId: attempt.id,
|
||||
dueAtMs: 50,
|
||||
}),
|
||||
'terminal',
|
||||
);
|
||||
assert.equal(latestReads, 0);
|
||||
assert.equal(workflowCalls[0][0], 'timeout');
|
||||
assert.equal(workflowCalls[1][0], 'terminal');
|
||||
assert.equal(workflowCalls[1][1].terminalStatus, 'timed_out');
|
||||
assert.equal(run.cancelRequestedAtMs, undefined);
|
||||
});
|
||||
|
||||
test('shutdown drain requests shutdown cancellation before stopping work', async (t) => {
|
||||
const runtime = await fixture(t);
|
||||
await insertActive(runtime, {
|
||||
runId: IDS.shutdownRun,
|
||||
attemptId: IDS.shutdownAttempt,
|
||||
});
|
||||
const completion = new LocalCompletionReceiptProcessor(
|
||||
runtime.runRepository,
|
||||
receiptStore(),
|
||||
{ clock: { now: () => 100 }, createEventId: eventIds() },
|
||||
);
|
||||
const coordinator = new LocalExecutionControlCoordinator(
|
||||
runtime.runRepository,
|
||||
completion,
|
||||
{ stop: async () => ({ status: 'already_exited' }) },
|
||||
{ clock: { now: () => 100 }, createEventId: eventIds() },
|
||||
);
|
||||
const scanner = new LocalExecutionControlScanner(
|
||||
runtime.executionControl,
|
||||
coordinator,
|
||||
{ now: () => 100 },
|
||||
);
|
||||
const summary = await scanner.drain({ limit: 4, maxPages: 2 });
|
||||
assert.deepEqual(summary, {
|
||||
scanned: 1,
|
||||
terminal: 1,
|
||||
remaining: 0,
|
||||
failed: 0,
|
||||
truncated: false,
|
||||
});
|
||||
const run = await runtime.runRepository.findRunById(IDS.shutdownRun);
|
||||
assert.equal(run.status, 'cancelled');
|
||||
assert.equal(run.cancelReason, 'shutdown');
|
||||
assert.deepEqual(
|
||||
(await runtime.runRepository.listEvents(IDS.shutdownRun)).map(
|
||||
(event) => event.type,
|
||||
),
|
||||
['run.cancel_requested', 'attempt.cancelled', 'run.cancelled'],
|
||||
);
|
||||
});
|
||||
|
||||
test('coalesces completion notifications and owns one idempotent shutdown drain', async () => {
|
||||
const completions = [];
|
||||
let scans = 0;
|
||||
let drains = 0;
|
||||
let cleanups = 0;
|
||||
const lifecycle = new LocalExecutionControlLifecycle(
|
||||
{
|
||||
async process(attemptId) {
|
||||
completions.push(attemptId);
|
||||
return 'missing';
|
||||
},
|
||||
},
|
||||
{
|
||||
async scan() {
|
||||
scans += 1;
|
||||
return {
|
||||
scanned: 0,
|
||||
terminal: 0,
|
||||
cancelRequested: 0,
|
||||
stale: 0,
|
||||
remaining: 0,
|
||||
failed: 0,
|
||||
truncated: false,
|
||||
};
|
||||
},
|
||||
async drain() {
|
||||
drains += 1;
|
||||
return {
|
||||
scanned: 0,
|
||||
terminal: 0,
|
||||
remaining: 0,
|
||||
failed: 0,
|
||||
truncated: false,
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
async scan() {
|
||||
cleanups += 1;
|
||||
return {
|
||||
scanned: 0,
|
||||
removed: 0,
|
||||
expiredMissing: 0,
|
||||
purgedQuarantines: 0,
|
||||
remaining: 0,
|
||||
failed: 0,
|
||||
truncated: false,
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
intervalMs: 60_000,
|
||||
pageSize: 4,
|
||||
cleanupIntervalMs: 60_000,
|
||||
cleanupPageSize: 4,
|
||||
stopTimeoutMs: 1_000,
|
||||
maxDrainPages: 1,
|
||||
clock: { now: () => 100 },
|
||||
},
|
||||
);
|
||||
assert.equal(lifecycle.notifyCompletion(IDS.completionAttempt), true);
|
||||
assert.equal(lifecycle.notifyCompletion(IDS.completionAttempt), true);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.deepEqual(completions, [IDS.completionAttempt]);
|
||||
assert.equal(scans, 1);
|
||||
assert.equal(cleanups, 1);
|
||||
const first = lifecycle.stopAndDrain();
|
||||
const second = lifecycle.stopAndDrain();
|
||||
assert.equal(first, second);
|
||||
assert.equal((await first).status, 'stopped');
|
||||
assert.equal(drains, 1);
|
||||
assert.equal(cleanups, 2);
|
||||
});
|
||||
@@ -0,0 +1,232 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
createLocalExecutionContextRecipe,
|
||||
createLocalTaskExecutionRevision,
|
||||
} = require('@qinglong/runtime-core/local-dispatch');
|
||||
const {
|
||||
migrateLocalSqlitePath,
|
||||
openLocalSqliteRuntimeDatabase,
|
||||
} = require('@qinglong/local-sqlite');
|
||||
const { LocalExecutionCoordinator } = require('../dist/execution');
|
||||
const { createLocalProcessDurableHandle } = require('@qinglong/local-process');
|
||||
const {
|
||||
LocalArtifactCapacityUnavailableError,
|
||||
LocalDispatchPlanMaterializer,
|
||||
LocalFileArtifactAllocator,
|
||||
LocalRunDispatcher,
|
||||
localArtifactCapacityPolicyForProfile,
|
||||
} = require('../dist/dispatch');
|
||||
|
||||
const RUN_ID = '019f7120-0000-7000-8000-000000000001';
|
||||
const ATTEMPT_ID = '019f7120-0000-7000-8000-000000000002';
|
||||
|
||||
function eventIdFactory() {
|
||||
let value = 0;
|
||||
return () => `dispatch-event-${++value}`;
|
||||
}
|
||||
|
||||
function processHandle() {
|
||||
const identity = {
|
||||
platform: 'linux',
|
||||
bootId: '11111111-2222-3333-4444-555555555555',
|
||||
pid: 4321,
|
||||
processGroupId: 4321,
|
||||
startTimeTicks: '123456',
|
||||
};
|
||||
return Object.freeze({
|
||||
handleId: 'dispatch-handle',
|
||||
pid: 4321,
|
||||
durableHandle: createLocalProcessDurableHandle('dispatch-handle', identity),
|
||||
startedAtMs: 10,
|
||||
completion: Promise.resolve({ exitCode: 0, signal: null }),
|
||||
});
|
||||
}
|
||||
|
||||
async function fixture(t) {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-dispatch-'));
|
||||
const databasePath = path.join(directory, 'qinglong3.sqlite');
|
||||
const artifactRoot = path.join(directory, 'artifacts');
|
||||
await migrateLocalSqlitePath({ databasePath, profile: 'edge' });
|
||||
const runtime = await openLocalSqliteRuntimeDatabase({
|
||||
databasePath,
|
||||
profile: 'edge',
|
||||
});
|
||||
t.after(async () => {
|
||||
await runtime.close();
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
const recipe = createLocalExecutionContextRecipe({
|
||||
environment: [
|
||||
{ name: 'PUBLIC_VALUE', kind: 'public', value: 'public-value' },
|
||||
{ name: 'SECRET_VALUE', kind: 'secret', secretRef: 'secret-ref-1' },
|
||||
],
|
||||
createdAtMs: 1,
|
||||
});
|
||||
assert.equal(
|
||||
await runtime.localDispatch.appendLocalExecutionContextRecipe(recipe),
|
||||
'inserted',
|
||||
);
|
||||
assert.equal(
|
||||
await runtime.localDispatch.appendLocalExecutionContextRecipe(recipe),
|
||||
'existing',
|
||||
);
|
||||
const revision = createLocalTaskExecutionRevision({
|
||||
projectId: 'default',
|
||||
taskId: 'task-1',
|
||||
taskRevision: 'revision-1',
|
||||
executorType: 'local_process',
|
||||
command: { kind: 'argv', file: '/bin/echo', args: ['hello'] },
|
||||
timeoutMs: 1_000,
|
||||
contextRef: recipe.contextRef,
|
||||
createdAtMs: 1,
|
||||
});
|
||||
assert.equal(
|
||||
await runtime.localDispatch.appendLocalTaskExecutionRevision(revision),
|
||||
'inserted',
|
||||
);
|
||||
assert.equal(
|
||||
await runtime.localDispatch.appendLocalTaskExecutionRevision(revision),
|
||||
'existing',
|
||||
);
|
||||
await runtime.runRepository.transaction(async (transaction) => {
|
||||
await transaction.insertRun({
|
||||
id: RUN_ID,
|
||||
projectId: 'default',
|
||||
taskId: 'task-1',
|
||||
taskRevision: 'revision-1',
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
status: 'queued',
|
||||
version: 0,
|
||||
eventSequence: 0,
|
||||
priority: 10,
|
||||
createdAtMs: 1,
|
||||
queuedAtMs: 1,
|
||||
});
|
||||
await transaction.insertAttempt({
|
||||
id: ATTEMPT_ID,
|
||||
runId: RUN_ID,
|
||||
attempt: 1,
|
||||
status: 'claimed',
|
||||
executorType: 'local_process',
|
||||
callbackSequence: 0,
|
||||
createdAtMs: 1,
|
||||
});
|
||||
});
|
||||
return { directory, databasePath, artifactRoot, runtime };
|
||||
}
|
||||
|
||||
test('materializes pinned context and atomically admits one real SQLite candidate', async (t) => {
|
||||
const value = await fixture(t);
|
||||
let launchRequest;
|
||||
const completionNotifications = [];
|
||||
const execution = new LocalExecutionCoordinator(
|
||||
value.runtime.runRepository,
|
||||
{
|
||||
async start(request) {
|
||||
launchRequest = request;
|
||||
assert.equal(request.environment.PUBLIC_VALUE, 'public-value');
|
||||
assert.equal(request.environment.SECRET_VALUE, 'top-secret');
|
||||
assert.equal(request.output.maximumBytes, 4 * 1024 * 1024);
|
||||
assert.match(request.output.logArtifactId, /^local-[0-9a-f]{30}$/);
|
||||
return processHandle();
|
||||
},
|
||||
},
|
||||
{ stop: async () => assert.fail('valid launch must not be stopped') },
|
||||
{
|
||||
clock: { now: () => 10 },
|
||||
createEventId: eventIdFactory(),
|
||||
createCallbackToken: () => 'A'.repeat(32),
|
||||
},
|
||||
);
|
||||
const dispatcher = new LocalRunDispatcher(
|
||||
value.runtime.localDispatch,
|
||||
new LocalDispatchPlanMaterializer(
|
||||
value.runtime.localDispatch,
|
||||
new LocalFileArtifactAllocator(
|
||||
value.artifactRoot,
|
||||
localArtifactCapacityPolicyForProfile('edge'),
|
||||
),
|
||||
{
|
||||
async resolveLocalSecretEnvironment(request) {
|
||||
assert.deepEqual(request.secretRefs, ['secret-ref-1']);
|
||||
assert.equal(request.candidate.projectId, 'default');
|
||||
return ['top-secret'];
|
||||
},
|
||||
},
|
||||
),
|
||||
execution,
|
||||
{
|
||||
pageSize: 4,
|
||||
maxPages: 1,
|
||||
onCompletion: (attemptId) => completionNotifications.push(attemptId),
|
||||
},
|
||||
);
|
||||
|
||||
const result = await dispatcher.dispatchOnce();
|
||||
assert.equal(result.status, 'activated');
|
||||
assert.equal(result.runId, RUN_ID);
|
||||
assert.equal(result.attemptId, ATTEMPT_ID);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.deepEqual(completionNotifications, [ATTEMPT_ID]);
|
||||
assert.equal(result.stats.candidatesScanned, 1);
|
||||
assert.equal(
|
||||
(await value.runtime.runRepository.findRunById(RUN_ID)).status,
|
||||
'running',
|
||||
);
|
||||
const attempt = await value.runtime.runRepository.findAttemptById(ATTEMPT_ID);
|
||||
assert.equal(attempt.status, 'running');
|
||||
assert.equal(attempt.logArtifactId, launchRequest.output.logArtifactId);
|
||||
assert.equal(fs.statSync(launchRequest.output.filePath).mode & 0o777, 0o600);
|
||||
assert.equal(fs.statSync(launchRequest.output.filePath).size, 0);
|
||||
assert.equal(
|
||||
fs.readFileSync(value.databasePath).includes(Buffer.from('top-secret')),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('missing Secret capability returns unavailable before Artifact allocation', async (t) => {
|
||||
const value = await fixture(t);
|
||||
const dispatcher = new LocalRunDispatcher(
|
||||
value.runtime.localDispatch,
|
||||
new LocalDispatchPlanMaterializer(
|
||||
value.runtime.localDispatch,
|
||||
new LocalFileArtifactAllocator(
|
||||
value.artifactRoot,
|
||||
localArtifactCapacityPolicyForProfile('edge'),
|
||||
),
|
||||
),
|
||||
{ start: async () => assert.fail('unavailable plan must not activate') },
|
||||
);
|
||||
const result = await dispatcher.dispatchOnce();
|
||||
assert.equal(result.status, 'idle');
|
||||
assert.equal(result.reason, 'plans_unavailable');
|
||||
assert.equal(result.stats.plansUnavailable, 1);
|
||||
assert.equal(fs.existsSync(value.artifactRoot), false);
|
||||
assert.equal(
|
||||
(await value.runtime.runRepository.findRunById(RUN_ID)).status,
|
||||
'queued',
|
||||
);
|
||||
});
|
||||
|
||||
test('capacity admission fails before creating an Artifact file', async (t) => {
|
||||
const value = await fixture(t);
|
||||
const [candidate] = (
|
||||
await value.runtime.localDispatch.listLocalDispatchCandidates({ limit: 1 })
|
||||
).candidates;
|
||||
const allocator = new LocalFileArtifactAllocator(
|
||||
value.artifactRoot,
|
||||
localArtifactCapacityPolicyForProfile('edge'),
|
||||
{ inspect: async () => 0n },
|
||||
);
|
||||
await assert.rejects(
|
||||
allocator.prepare(candidate),
|
||||
LocalArtifactCapacityUnavailableError,
|
||||
);
|
||||
assert.deepEqual(fs.readdirSync(value.artifactRoot), []);
|
||||
});
|
||||
@@ -0,0 +1,264 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
migrateLocalSqlitePath,
|
||||
openLocalSqliteRuntimeDatabase,
|
||||
} = require('@qinglong/local-sqlite');
|
||||
const {
|
||||
LocalExecutionCoordinator,
|
||||
LocalExecutionLaunchError,
|
||||
LocalExecutionOwnershipPersistenceError,
|
||||
LocalExecutionRejectedError,
|
||||
} = require('../dist/execution');
|
||||
const { createLocalProcessDurableHandle } = require('@qinglong/local-process');
|
||||
|
||||
const RUN_ID = '019f70d0-0000-7000-8000-000000000001';
|
||||
const ATTEMPT_ID = '019f70d0-0000-7000-8000-000000000002';
|
||||
const CALLBACK_TOKEN = 'A'.repeat(32);
|
||||
|
||||
async function fixture(t) {
|
||||
const directory = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), 'ql3-local-execution-'),
|
||||
);
|
||||
const options = {
|
||||
databasePath: path.join(directory, 'qinglong3.sqlite'),
|
||||
profile: 'edge',
|
||||
};
|
||||
await migrateLocalSqlitePath(options);
|
||||
const runtime = await openLocalSqliteRuntimeDatabase(options);
|
||||
t.after(async () => {
|
||||
await runtime.close();
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
await runtime.runRepository.transaction(async (transaction) => {
|
||||
await transaction.insertRun({
|
||||
id: RUN_ID,
|
||||
projectId: 'default',
|
||||
taskId: 'task-1',
|
||||
taskRevision: 'revision-1',
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
status: 'queued',
|
||||
version: 0,
|
||||
eventSequence: 0,
|
||||
priority: 0,
|
||||
createdAtMs: 1,
|
||||
queuedAtMs: 1,
|
||||
});
|
||||
await transaction.insertAttempt({
|
||||
id: ATTEMPT_ID,
|
||||
runId: RUN_ID,
|
||||
attempt: 1,
|
||||
status: 'claimed',
|
||||
executorType: 'local_process',
|
||||
callbackSequence: 0,
|
||||
createdAtMs: 1,
|
||||
});
|
||||
});
|
||||
return runtime.runRepository;
|
||||
}
|
||||
|
||||
function eventIdFactory() {
|
||||
let value = 0;
|
||||
return () => `event-${++value}`;
|
||||
}
|
||||
|
||||
function handle() {
|
||||
const identity = {
|
||||
platform: 'linux',
|
||||
bootId: '11111111-2222-3333-4444-555555555555',
|
||||
pid: 1234,
|
||||
processGroupId: 1234,
|
||||
startTimeTicks: '987654',
|
||||
};
|
||||
return Object.freeze({
|
||||
handleId: 'handle-1',
|
||||
pid: 1234,
|
||||
durableHandle: createLocalProcessDurableHandle('handle-1', identity),
|
||||
startedAtMs: 10,
|
||||
completion: Promise.resolve({ exitCode: 0, signal: null }),
|
||||
});
|
||||
}
|
||||
|
||||
function command() {
|
||||
return {
|
||||
runId: RUN_ID,
|
||||
attemptId: ATTEMPT_ID,
|
||||
command: { kind: 'argv', file: '/bin/echo', args: ['hello'] },
|
||||
timeoutMs: 1_000,
|
||||
};
|
||||
}
|
||||
|
||||
function repositoryFailingTransaction(repository, transactionNumber) {
|
||||
let transactions = 0;
|
||||
return {
|
||||
findRunById: (...args) => repository.findRunById(...args),
|
||||
findAttemptById: (...args) => repository.findAttemptById(...args),
|
||||
findLatestAttemptByRunId: (...args) =>
|
||||
repository.findLatestAttemptByRunId(...args),
|
||||
findRetryPolicyByRunId: (...args) =>
|
||||
repository.findRetryPolicyByRunId(...args),
|
||||
listEvents: (...args) => repository.listEvents(...args),
|
||||
listCancellationRequested: (...args) =>
|
||||
repository.listCancellationRequested(...args),
|
||||
transaction(work) {
|
||||
transactions += 1;
|
||||
if (transactions === transactionNumber) {
|
||||
return Promise.reject(new Error('injected transaction failure'));
|
||||
}
|
||||
return repository.transaction(work);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('atomically persists claimed -> starting -> running around launch', async (t) => {
|
||||
const repository = await fixture(t);
|
||||
let launchRequest;
|
||||
let clockReads = 0;
|
||||
const coordinator = new LocalExecutionCoordinator(
|
||||
repository,
|
||||
{
|
||||
async start(request) {
|
||||
launchRequest = request;
|
||||
const [run, attempt] = await Promise.all([
|
||||
repository.findRunById(RUN_ID),
|
||||
repository.findAttemptById(ATTEMPT_ID),
|
||||
]);
|
||||
assert.equal(run.status, 'dispatching');
|
||||
assert.equal(run.version, 2);
|
||||
assert.equal(attempt.status, 'starting');
|
||||
assert.equal(
|
||||
attempt.callbackTokenHash,
|
||||
createHash('sha256').update(CALLBACK_TOKEN).digest('hex'),
|
||||
);
|
||||
return handle();
|
||||
},
|
||||
},
|
||||
{
|
||||
stop: async () => assert.fail('controller must not stop a valid launch'),
|
||||
},
|
||||
{
|
||||
clock: { now: () => (clockReads++ === 0 ? 5 : 20) },
|
||||
createEventId: eventIdFactory(),
|
||||
createCallbackToken: () => CALLBACK_TOKEN,
|
||||
},
|
||||
);
|
||||
|
||||
const result = await coordinator.start(command());
|
||||
assert.equal(launchRequest.callbackToken, CALLBACK_TOKEN);
|
||||
assert.equal(launchRequest.callbackSequence, 1);
|
||||
assert.equal(result.run.status, 'running');
|
||||
assert.equal(result.run.version, 4);
|
||||
assert.equal(result.attempt.status, 'running');
|
||||
assert.equal(result.attempt.executorHandle, handle().durableHandle);
|
||||
assert.equal(result.attempt.pid, 1234);
|
||||
assert.equal(result.attempt.startedAtMs, handle().startedAtMs);
|
||||
assert.equal(result.attempt.callbackSequence, 0);
|
||||
assert.equal('callbackToken' in result, false);
|
||||
assert.deepEqual(
|
||||
(await repository.listEvents(RUN_ID)).map((event) => event.type),
|
||||
['run.dispatching', 'attempt.starting', 'attempt.running', 'run.running'],
|
||||
);
|
||||
});
|
||||
|
||||
test('records a pre-ownership launch failure as failed', async (t) => {
|
||||
const repository = await fixture(t);
|
||||
const coordinator = new LocalExecutionCoordinator(
|
||||
repository,
|
||||
{ start: async () => Promise.reject(new Error('spawn failed')) },
|
||||
{ stop: async () => assert.fail('no durable handle exists') },
|
||||
{
|
||||
clock: { now: () => 10 },
|
||||
createEventId: eventIdFactory(),
|
||||
createCallbackToken: () => CALLBACK_TOKEN,
|
||||
},
|
||||
);
|
||||
await assert.rejects(coordinator.start(command()), LocalExecutionLaunchError);
|
||||
assert.equal((await repository.findRunById(RUN_ID)).status, 'failed');
|
||||
assert.equal(
|
||||
(await repository.findAttemptById(ATTEMPT_ID)).errorCode,
|
||||
'EXECUTOR_START_FAILED',
|
||||
);
|
||||
});
|
||||
|
||||
test('stops exact process then records lost when running persistence fails', async (t) => {
|
||||
const repository = await fixture(t);
|
||||
const failing = repositoryFailingTransaction(repository, 2);
|
||||
const stopped = [];
|
||||
const coordinator = new LocalExecutionCoordinator(
|
||||
failing,
|
||||
{ start: async () => handle() },
|
||||
{
|
||||
async stop(durableHandle) {
|
||||
stopped.push(durableHandle);
|
||||
return { status: 'stopped', signal: 'SIGTERM' };
|
||||
},
|
||||
},
|
||||
{
|
||||
clock: { now: () => 10 },
|
||||
createEventId: eventIdFactory(),
|
||||
createCallbackToken: () => CALLBACK_TOKEN,
|
||||
},
|
||||
);
|
||||
await assert.rejects(coordinator.start(command()), (error) => {
|
||||
assert.ok(error instanceof LocalExecutionOwnershipPersistenceError);
|
||||
assert.equal(error.compensation.status, 'stopped');
|
||||
return true;
|
||||
});
|
||||
assert.deepEqual(stopped, [handle().durableHandle]);
|
||||
assert.equal((await repository.findRunById(RUN_ID)).status, 'lost');
|
||||
assert.equal((await repository.findAttemptById(ATTEMPT_ID)).status, 'lost');
|
||||
});
|
||||
|
||||
test('keeps starting authority for recovery when exact stop is inconclusive', async (t) => {
|
||||
const repository = await fixture(t);
|
||||
const failing = repositoryFailingTransaction(repository, 2);
|
||||
const coordinator = new LocalExecutionCoordinator(
|
||||
failing,
|
||||
{ start: async () => handle() },
|
||||
{
|
||||
stop: async () => ({
|
||||
status: 'unknown',
|
||||
reason: 'provider_unavailable',
|
||||
}),
|
||||
},
|
||||
{
|
||||
clock: { now: () => 10 },
|
||||
createEventId: eventIdFactory(),
|
||||
createCallbackToken: () => CALLBACK_TOKEN,
|
||||
},
|
||||
);
|
||||
await assert.rejects(
|
||||
coordinator.start(command()),
|
||||
LocalExecutionOwnershipPersistenceError,
|
||||
);
|
||||
assert.equal((await repository.findRunById(RUN_ID)).status, 'dispatching');
|
||||
assert.equal(
|
||||
(await repository.findAttemptById(ATTEMPT_ID)).status,
|
||||
'starting',
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects replay after execution authority has moved', async (t) => {
|
||||
const repository = await fixture(t);
|
||||
const coordinator = new LocalExecutionCoordinator(
|
||||
repository,
|
||||
{ start: async () => handle() },
|
||||
{ stop: async () => ({ status: 'already_exited' }) },
|
||||
{
|
||||
clock: { now: () => 10 },
|
||||
createEventId: eventIdFactory(),
|
||||
createCallbackToken: () => CALLBACK_TOKEN,
|
||||
},
|
||||
);
|
||||
await coordinator.start(command());
|
||||
await assert.rejects(
|
||||
coordinator.start(command()),
|
||||
LocalExecutionRejectedError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,554 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { test } = require('node:test');
|
||||
const { InvalidCompletionReceiptError } = require('@qinglong/local-process');
|
||||
const {
|
||||
LocalRunStartupRecoveryCoordinator,
|
||||
LocalWorkflowTaskStartupRecoveryCoordinator,
|
||||
} = require('../dist/recovery');
|
||||
|
||||
const RUN_ID = '019f70c0-0000-7000-8000-000000000001';
|
||||
const ATTEMPT_ID = '019f70c0-0000-7000-8000-000000000002';
|
||||
const TOKEN = 'A'.repeat(32);
|
||||
|
||||
function run(overrides = {}) {
|
||||
return {
|
||||
id: RUN_ID,
|
||||
projectId: 'default',
|
||||
taskId: 'task-1',
|
||||
taskRevision: 'revision-1',
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
status: 'dispatching',
|
||||
version: 0,
|
||||
eventSequence: 0,
|
||||
priority: 0,
|
||||
createdAtMs: 1,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function attempt(overrides = {}) {
|
||||
return {
|
||||
id: ATTEMPT_ID,
|
||||
runId: RUN_ID,
|
||||
attempt: 1,
|
||||
status: 'claimed',
|
||||
executorType: 'local_process',
|
||||
callbackSequence: 0,
|
||||
createdAtMs: 2,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
class MemoryRepository {
|
||||
constructor(initialRun, initialAttempt) {
|
||||
this.runs = new Map(initialRun ? [[initialRun.id, initialRun]] : []);
|
||||
this.attempts = new Map(
|
||||
initialAttempt ? [[initialAttempt.id, initialAttempt]] : [],
|
||||
);
|
||||
this.events = [];
|
||||
}
|
||||
|
||||
async transaction(work) {
|
||||
const previousRuns = structuredClone(this.runs);
|
||||
const previousAttempts = structuredClone(this.attempts);
|
||||
const previousEvents = structuredClone(this.events);
|
||||
const transaction = {
|
||||
findRunById: async (id) => structuredClone(this.runs.get(id) ?? null),
|
||||
findAttemptById: async (id) =>
|
||||
structuredClone(this.attempts.get(id) ?? null),
|
||||
findLatestAttemptByRunId: async (runId) => {
|
||||
const values = [...this.attempts.values()]
|
||||
.filter((item) => item.runId === runId)
|
||||
.sort((left, right) => right.attempt - left.attempt);
|
||||
return structuredClone(values[0] ?? null);
|
||||
},
|
||||
compareAndSetRun: async (value, expectedVersion) => {
|
||||
const current = this.runs.get(value.id);
|
||||
if (!current || current.version !== expectedVersion) return false;
|
||||
this.runs.set(value.id, structuredClone(value));
|
||||
return true;
|
||||
},
|
||||
compareAndSetAttempt: async (value, expected) => {
|
||||
const current = this.attempts.get(value.id);
|
||||
if (
|
||||
!current ||
|
||||
current.status !== expected.status ||
|
||||
current.callbackSequence !== expected.callbackSequence
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
this.attempts.set(value.id, structuredClone(value));
|
||||
return true;
|
||||
},
|
||||
appendEvent: async (value) => this.events.push(structuredClone(value)),
|
||||
};
|
||||
try {
|
||||
return await work(transaction);
|
||||
} catch (error) {
|
||||
this.runs = previousRuns;
|
||||
this.attempts = previousAttempts;
|
||||
this.events = previousEvents;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function source(repository, overrides = {}) {
|
||||
let calls = 0;
|
||||
return {
|
||||
get calls() {
|
||||
return calls;
|
||||
},
|
||||
async inspectCandidates() {
|
||||
calls += 1;
|
||||
if (overrides.page) return overrides.page;
|
||||
const candidates = [];
|
||||
for (const value of [...repository.runs.values()].sort((a, b) =>
|
||||
a.id.localeCompare(b.id),
|
||||
)) {
|
||||
if (
|
||||
value.executionOwner !== 'runtime' ||
|
||||
!['dispatching', 'running'].includes(value.status)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
candidates.push({
|
||||
runId: value.id,
|
||||
runStatus: value.status,
|
||||
activeAttemptCount: [...repository.attempts.values()].filter(
|
||||
(item) =>
|
||||
item.runId === value.id &&
|
||||
['claimed', 'starting', 'running'].includes(item.status),
|
||||
).length,
|
||||
});
|
||||
}
|
||||
return { candidates, truncated: false };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function receipts(value) {
|
||||
let readCount = 0;
|
||||
let removed = 0;
|
||||
return {
|
||||
get readCount() {
|
||||
return readCount;
|
||||
},
|
||||
get removed() {
|
||||
return removed;
|
||||
},
|
||||
async read() {
|
||||
readCount += 1;
|
||||
return value;
|
||||
},
|
||||
async remove() {
|
||||
removed += 1;
|
||||
return true;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function coordinator(
|
||||
repository,
|
||||
candidateSource,
|
||||
receiptStore,
|
||||
inspect,
|
||||
options = {},
|
||||
) {
|
||||
return new LocalRunStartupRecoveryCoordinator(
|
||||
repository,
|
||||
candidateSource,
|
||||
receiptStore,
|
||||
{ executorType: 'local_process', inspect },
|
||||
{
|
||||
clock: { now: () => 10 },
|
||||
createEventId: (() => {
|
||||
let value = 0;
|
||||
return () => `event-${++value}`;
|
||||
})(),
|
||||
...options,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
test('zero candidates pay one durable read and no receipt or process work', async () => {
|
||||
const repository = new MemoryRepository();
|
||||
const candidateSource = source(repository);
|
||||
const receiptStore = receipts(undefined);
|
||||
let inspections = 0;
|
||||
const summary = await coordinator(
|
||||
repository,
|
||||
candidateSource,
|
||||
receiptStore,
|
||||
async () => {
|
||||
inspections += 1;
|
||||
return { status: 'running', identityPid: 1 };
|
||||
},
|
||||
).recover();
|
||||
|
||||
assert.deepEqual(summary, {
|
||||
safe: true,
|
||||
scanned: 0,
|
||||
recovered: 0,
|
||||
remaining: 0,
|
||||
failed: 0,
|
||||
truncated: false,
|
||||
});
|
||||
assert.equal(candidateSource.calls, 1);
|
||||
assert.equal(receiptStore.readCount, 0);
|
||||
assert.equal(inspections, 0);
|
||||
});
|
||||
|
||||
test('truncation fails before any partial mutation or evidence read', async () => {
|
||||
const repository = new MemoryRepository(run(), attempt());
|
||||
const candidateSource = source(repository, {
|
||||
page: {
|
||||
candidates: [
|
||||
{
|
||||
runId: RUN_ID,
|
||||
runStatus: 'dispatching',
|
||||
activeAttemptCount: 1,
|
||||
},
|
||||
],
|
||||
truncated: true,
|
||||
},
|
||||
});
|
||||
const receiptStore = receipts(undefined);
|
||||
const summary = await coordinator(
|
||||
repository,
|
||||
candidateSource,
|
||||
receiptStore,
|
||||
async () => {
|
||||
throw new Error('must not inspect');
|
||||
},
|
||||
).recover();
|
||||
|
||||
assert.equal(summary.safe, false);
|
||||
assert.equal(summary.truncated, true);
|
||||
assert.equal(repository.runs.get(RUN_ID).status, 'dispatching');
|
||||
assert.equal(repository.attempts.get(ATTEMPT_ID).status, 'claimed');
|
||||
assert.equal(repository.events.length, 0);
|
||||
assert.equal(receiptStore.readCount, 0);
|
||||
});
|
||||
|
||||
test('an unstarted claimed Attempt is atomically marked lost without probing', async () => {
|
||||
const repository = new MemoryRepository(run(), attempt());
|
||||
const candidateSource = source(repository);
|
||||
const receiptStore = receipts(undefined);
|
||||
let inspections = 0;
|
||||
const summary = await coordinator(
|
||||
repository,
|
||||
candidateSource,
|
||||
receiptStore,
|
||||
async () => {
|
||||
inspections += 1;
|
||||
return { status: 'running', identityPid: 1 };
|
||||
},
|
||||
).recover();
|
||||
|
||||
assert.equal(summary.safe, true);
|
||||
assert.equal(summary.recovered, 1);
|
||||
assert.equal(repository.runs.get(RUN_ID).status, 'lost');
|
||||
assert.equal(repository.attempts.get(ATTEMPT_ID).status, 'lost');
|
||||
assert.deepEqual(
|
||||
repository.events.map((item) => item.type),
|
||||
['attempt.lost', 'run.lost'],
|
||||
);
|
||||
assert.equal(inspections, 0);
|
||||
});
|
||||
|
||||
test('a trusted completion receipt wins before process inspection', async () => {
|
||||
const callbackTokenHash = createHash('sha256').update(TOKEN).digest('hex');
|
||||
const repository = new MemoryRepository(
|
||||
run({ status: 'running', startedAtMs: 3 }),
|
||||
attempt({
|
||||
status: 'running',
|
||||
startedAtMs: 3,
|
||||
callbackTokenHash,
|
||||
executorHandle: 'unused',
|
||||
pid: 10,
|
||||
}),
|
||||
);
|
||||
const candidateSource = source(repository);
|
||||
const receiptStore = receipts({
|
||||
schemaVersion: 1,
|
||||
runId: RUN_ID,
|
||||
attemptId: ATTEMPT_ID,
|
||||
callbackSequence: 1,
|
||||
token: TOKEN,
|
||||
startedAtMs: 3,
|
||||
finishedAtMs: 8,
|
||||
exitCode: 0,
|
||||
});
|
||||
let inspections = 0;
|
||||
const resolved = [];
|
||||
const summary = await coordinator(
|
||||
repository,
|
||||
candidateSource,
|
||||
receiptStore,
|
||||
async () => {
|
||||
inspections += 1;
|
||||
return { status: 'running', identityPid: 10 };
|
||||
},
|
||||
{
|
||||
journal: {
|
||||
async markQuarantined() {
|
||||
throw new Error('must not quarantine a trusted receipt');
|
||||
},
|
||||
async resolve(attemptId) {
|
||||
resolved.push(attemptId);
|
||||
},
|
||||
},
|
||||
},
|
||||
).recover();
|
||||
|
||||
assert.equal(summary.safe, true);
|
||||
assert.equal(repository.runs.get(RUN_ID).status, 'succeeded');
|
||||
assert.equal(repository.attempts.get(ATTEMPT_ID).status, 'succeeded');
|
||||
assert.equal(repository.attempts.get(ATTEMPT_ID).callbackSequence, 1);
|
||||
assert.deepEqual(
|
||||
repository.events.map((item) => item.type),
|
||||
['attempt.succeeded', 'run.succeeded'],
|
||||
);
|
||||
assert.equal(receiptStore.removed, 1);
|
||||
assert.deepEqual(resolved, [ATTEMPT_ID]);
|
||||
assert.equal(inspections, 0);
|
||||
});
|
||||
|
||||
test('an invalid receipt records durable quarantine intent before moving the file', async () => {
|
||||
const repository = new MemoryRepository(
|
||||
run({ status: 'running', startedAtMs: 3 }),
|
||||
attempt({
|
||||
status: 'running',
|
||||
startedAtMs: 3,
|
||||
executorHandle: 'handle-1',
|
||||
pid: 10,
|
||||
}),
|
||||
);
|
||||
const operations = [];
|
||||
const receiptStore = {
|
||||
async read() {
|
||||
throw new InvalidCompletionReceiptError('invalid test receipt');
|
||||
},
|
||||
async remove() {
|
||||
throw new Error('must not remove an invalid receipt');
|
||||
},
|
||||
async publish() {
|
||||
throw new Error('must not publish during recovery');
|
||||
},
|
||||
quarantineReference(attemptId) {
|
||||
return `.quarantine/${attemptId}.json`;
|
||||
},
|
||||
async quarantine(attemptId) {
|
||||
operations.push(`file:${attemptId}`);
|
||||
return `.quarantine/${attemptId}.json`;
|
||||
},
|
||||
};
|
||||
|
||||
const summary = await coordinator(
|
||||
repository,
|
||||
source(repository),
|
||||
receiptStore,
|
||||
async () => {
|
||||
throw new Error('invalid receipt must block process inspection');
|
||||
},
|
||||
{
|
||||
quarantineRetentionMs: 100,
|
||||
journal: {
|
||||
async markQuarantined(record) {
|
||||
operations.push(`journal:${record.attemptId}`);
|
||||
assert.deepEqual(record, {
|
||||
attemptId: ATTEMPT_ID,
|
||||
quarantineRef: `.quarantine/${ATTEMPT_ID}.json`,
|
||||
updatedAtMs: 10,
|
||||
purgeAfterMs: 110,
|
||||
});
|
||||
},
|
||||
async resolve() {
|
||||
throw new Error('must not resolve invalid receipt intent');
|
||||
},
|
||||
},
|
||||
},
|
||||
).recover();
|
||||
|
||||
assert.equal(summary.safe, false);
|
||||
assert.equal(summary.remaining, 1);
|
||||
assert.deepEqual(operations, [`journal:${ATTEMPT_ID}`, `file:${ATTEMPT_ID}`]);
|
||||
});
|
||||
|
||||
test('a live exact process is verified twice without terminalizing the Run', async () => {
|
||||
const repository = new MemoryRepository(
|
||||
run({ status: 'running', startedAtMs: 3 }),
|
||||
attempt({
|
||||
status: 'running',
|
||||
startedAtMs: 3,
|
||||
executorHandle: 'handle-1',
|
||||
pid: 10,
|
||||
}),
|
||||
);
|
||||
const candidateSource = source(repository);
|
||||
const receiptStore = receipts(undefined);
|
||||
let inspections = 0;
|
||||
const summary = await coordinator(
|
||||
repository,
|
||||
candidateSource,
|
||||
receiptStore,
|
||||
async () => {
|
||||
inspections += 1;
|
||||
return { status: 'running', identityPid: 10 };
|
||||
},
|
||||
).recover();
|
||||
|
||||
assert.equal(summary.safe, true);
|
||||
assert.equal(summary.recovered, 1);
|
||||
assert.equal(repository.runs.get(RUN_ID).status, 'running');
|
||||
assert.equal(repository.attempts.get(ATTEMPT_ID).status, 'running');
|
||||
assert.equal(repository.events.length, 0);
|
||||
assert.equal(inspections, 2);
|
||||
assert.equal(receiptStore.readCount, 2);
|
||||
});
|
||||
|
||||
test('a process change during final verification revokes startup safety', async () => {
|
||||
const repository = new MemoryRepository(
|
||||
run({ status: 'running', startedAtMs: 3 }),
|
||||
attempt({
|
||||
status: 'running',
|
||||
startedAtMs: 3,
|
||||
executorHandle: 'handle-1',
|
||||
pid: 10,
|
||||
}),
|
||||
);
|
||||
let inspections = 0;
|
||||
const summary = await coordinator(
|
||||
repository,
|
||||
source(repository),
|
||||
receipts(undefined),
|
||||
async () => {
|
||||
inspections += 1;
|
||||
return inspections === 1
|
||||
? { status: 'running', identityPid: 10 }
|
||||
: { status: 'not_running', identityPid: 10 };
|
||||
},
|
||||
).recover();
|
||||
|
||||
assert.deepEqual(summary, {
|
||||
safe: false,
|
||||
scanned: 1,
|
||||
recovered: 0,
|
||||
remaining: 1,
|
||||
failed: 0,
|
||||
truncated: false,
|
||||
});
|
||||
assert.equal(repository.runs.get(RUN_ID).status, 'running');
|
||||
});
|
||||
|
||||
test('trusted not-running evidence marks the aggregate lost but unknown evidence blocks', async () => {
|
||||
const activeRun = run({ status: 'running', startedAtMs: 3 });
|
||||
const activeAttempt = attempt({
|
||||
status: 'running',
|
||||
startedAtMs: 3,
|
||||
executorHandle: 'handle-1',
|
||||
pid: 10,
|
||||
});
|
||||
const repository = new MemoryRepository(activeRun, activeAttempt);
|
||||
const summary = await coordinator(
|
||||
repository,
|
||||
source(repository),
|
||||
receipts(undefined),
|
||||
async () => ({ status: 'not_running', identityPid: 10 }),
|
||||
).recover();
|
||||
assert.equal(summary.safe, true);
|
||||
assert.equal(repository.runs.get(RUN_ID).status, 'lost');
|
||||
|
||||
const unknownRepository = new MemoryRepository(activeRun, activeAttempt);
|
||||
const unknown = await coordinator(
|
||||
unknownRepository,
|
||||
source(unknownRepository),
|
||||
receipts(undefined),
|
||||
async () => ({ status: 'unknown', reason: 'invalid_handle' }),
|
||||
).recover();
|
||||
assert.equal(unknown.safe, false);
|
||||
assert.equal(unknown.remaining, 1);
|
||||
assert.equal(unknownRepository.runs.get(RUN_ID).status, 'running');
|
||||
assert.equal(unknownRepository.events.length, 0);
|
||||
});
|
||||
|
||||
test('recovers an orphaned claimed Workflow Task without terminalizing its parent', async () => {
|
||||
const workflowRun = run({
|
||||
taskId: 'workflow',
|
||||
triggerType: 'plugin_package_workflow',
|
||||
executionOrigin: 'system',
|
||||
status: 'running',
|
||||
version: 4,
|
||||
eventSequence: 4,
|
||||
startedAtMs: 2,
|
||||
});
|
||||
const workflowAttempt = attempt({
|
||||
stepRunId: 'workflow-step',
|
||||
status: 'claimed',
|
||||
createdAtMs: 5,
|
||||
});
|
||||
const repository = new MemoryRepository(workflowRun, workflowAttempt);
|
||||
let recovered = false;
|
||||
let inspections = 0;
|
||||
const recovery = {
|
||||
async listRecoveryCandidates() {
|
||||
return {
|
||||
candidates: recovered
|
||||
? []
|
||||
: [
|
||||
{
|
||||
runId: RUN_ID,
|
||||
attemptId: ATTEMPT_ID,
|
||||
attemptCreatedAtMs: 5,
|
||||
},
|
||||
],
|
||||
truncated: false,
|
||||
};
|
||||
},
|
||||
async recover(command) {
|
||||
assert.equal(command.reason, 'unstarted_claim_expired');
|
||||
repository.attempts.set(ATTEMPT_ID, {
|
||||
...workflowAttempt,
|
||||
status: 'lost',
|
||||
finishedAtMs: command.observedAtMs,
|
||||
});
|
||||
recovered = true;
|
||||
return 'requeued';
|
||||
},
|
||||
};
|
||||
const coordinator = new LocalWorkflowTaskStartupRecoveryCoordinator(
|
||||
repository,
|
||||
recovery,
|
||||
{
|
||||
async recordRunning() {
|
||||
throw new Error('claimed recovery must not mark running');
|
||||
},
|
||||
},
|
||||
{ process: async () => 'missing' },
|
||||
{
|
||||
executorType: 'local_process',
|
||||
async inspect() {
|
||||
inspections += 1;
|
||||
throw new Error('claimed recovery must not inspect a process');
|
||||
},
|
||||
},
|
||||
{ clock: { now: () => 10 } },
|
||||
);
|
||||
|
||||
assert.deepEqual(await coordinator.recover(), {
|
||||
safe: true,
|
||||
scanned: 1,
|
||||
recovered: 1,
|
||||
verified: 0,
|
||||
remaining: 0,
|
||||
failed: 0,
|
||||
truncated: false,
|
||||
});
|
||||
assert.equal(repository.runs.get(RUN_ID).status, 'running');
|
||||
assert.equal(repository.attempts.get(ATTEMPT_ID).status, 'lost');
|
||||
assert.equal(inspections, 0);
|
||||
});
|
||||
@@ -0,0 +1,377 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
LocalSchedulerCoordinator,
|
||||
LocalSchedulerLifecycle,
|
||||
LocalWorkflowSchedulerCoordinator,
|
||||
} = require('../dist/scheduler');
|
||||
|
||||
function nextMinute(schedule, afterMs) {
|
||||
if (schedule.expression !== '* * * * *' || schedule.timezone !== 'UTC') {
|
||||
throw new Error('unsupported test schedule');
|
||||
}
|
||||
return Math.floor(afterMs / 60_000 + 1) * 60_000;
|
||||
}
|
||||
|
||||
function candidate(overrides = {}) {
|
||||
return {
|
||||
projectId: 'default',
|
||||
triggerId: 'trigger-1',
|
||||
triggerRevision: 1,
|
||||
triggerContentDigest: 'a'.repeat(64),
|
||||
triggerUpdatedAtMs: 1,
|
||||
taskId: 'task-1',
|
||||
taskRevision: 1,
|
||||
taskContentDigest: 'b'.repeat(64),
|
||||
expression: '* * * * *',
|
||||
timezone: 'UTC',
|
||||
misfirePolicy: 'skip',
|
||||
stateVersion: 0,
|
||||
nextFireAtMs: 60_000,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('coordinates one bounded page and notifies only committed admissions', async () => {
|
||||
const committed = [];
|
||||
const notified = [];
|
||||
let sequence = 0;
|
||||
const coordinator = new LocalSchedulerCoordinator(
|
||||
{
|
||||
async listLocalScheduleCandidates(options) {
|
||||
assert.deepEqual(options, { observedAtMs: 61_000, limit: 4 });
|
||||
return {
|
||||
candidates: [candidate(), candidate({ triggerId: 'trigger-2' })],
|
||||
truncated: true,
|
||||
};
|
||||
},
|
||||
async commitLocalScheduleDecision(command) {
|
||||
committed.push(command);
|
||||
if (committed.length === 2) return { status: 'raced' };
|
||||
return {
|
||||
status: 'admitted',
|
||||
disposition: 'admit',
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
pageSize: 4,
|
||||
misfireGraceMs: 5_000,
|
||||
clock: () => 61_000,
|
||||
nextOccurrence: nextMinute,
|
||||
createId: () =>
|
||||
`019f7500-0000-4000-8000-${String(++sequence).padStart(12, '0')}`,
|
||||
onAdmitted: (runId, attemptId) => notified.push([runId, attemptId]),
|
||||
},
|
||||
);
|
||||
|
||||
assert.deepEqual(await coordinator.scheduleOnce(), {
|
||||
observedAtMs: 61_000,
|
||||
scanned: 2,
|
||||
initialized: 0,
|
||||
skipped: 0,
|
||||
admitted: 1,
|
||||
raced: 1,
|
||||
truncated: true,
|
||||
});
|
||||
assert.equal(committed.length, 2);
|
||||
assert.deepEqual(notified, [[committed[0].runId, committed[0].attemptId]]);
|
||||
});
|
||||
|
||||
test('does not allocate Run identities for initialization or skip decisions', async () => {
|
||||
let allocations = 0;
|
||||
const commands = [];
|
||||
const coordinator = new LocalSchedulerCoordinator(
|
||||
{
|
||||
async listLocalScheduleCandidates() {
|
||||
return {
|
||||
candidates: [
|
||||
candidate({ nextFireAtMs: null, triggerUpdatedAtMs: 90_000 }),
|
||||
candidate({ triggerId: 'trigger-2', nextFireAtMs: 60_000 }),
|
||||
],
|
||||
truncated: false,
|
||||
};
|
||||
},
|
||||
async commitLocalScheduleDecision(command) {
|
||||
commands.push(command);
|
||||
return {
|
||||
status: 'advanced',
|
||||
disposition: command.decision.disposition,
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
clock: () => 100_000,
|
||||
misfireGraceMs: 5_000,
|
||||
nextOccurrence: nextMinute,
|
||||
createId() {
|
||||
allocations += 1;
|
||||
return 'unused';
|
||||
},
|
||||
},
|
||||
);
|
||||
const summary = await coordinator.scheduleOnce();
|
||||
assert.equal(summary.initialized, 1);
|
||||
assert.equal(summary.skipped, 1);
|
||||
assert.equal(allocations, 0);
|
||||
assert.equal(
|
||||
commands.every((command) => command.runId === undefined),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('reuses one scheduler cycle for cancellation, frontier, Task admission and dispatch', async () => {
|
||||
const calls = [];
|
||||
let dispatches = 0;
|
||||
const schedulerSummary = {
|
||||
observedAtMs: 10,
|
||||
scanned: 0,
|
||||
initialized: 0,
|
||||
skipped: 0,
|
||||
admitted: 0,
|
||||
raced: 0,
|
||||
truncated: false,
|
||||
};
|
||||
const coordinator = new LocalWorkflowSchedulerCoordinator(
|
||||
{
|
||||
async scheduleOnce() {
|
||||
calls.push('schedule');
|
||||
return schedulerSummary;
|
||||
},
|
||||
},
|
||||
{
|
||||
async convergePage(command) {
|
||||
calls.push(`cancel:${command.limit}`);
|
||||
return {
|
||||
scanned: 0,
|
||||
settledRuns: 0,
|
||||
settledAttempts: 0,
|
||||
blocked: 0,
|
||||
hasMore: false,
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
async listCandidates(command) {
|
||||
calls.push(`frontier-list:${command.limit}`);
|
||||
return {
|
||||
candidates: [
|
||||
{
|
||||
runId: 'workflow-run',
|
||||
planDigest: 'a'.repeat(64),
|
||||
admittedAtMs: 1,
|
||||
},
|
||||
],
|
||||
truncated: false,
|
||||
};
|
||||
},
|
||||
async advance(runId) {
|
||||
calls.push(`frontier-advance:${runId}`);
|
||||
return {};
|
||||
},
|
||||
},
|
||||
{
|
||||
async listCandidates(command) {
|
||||
calls.push(`task-list:${command.limit}`);
|
||||
return {
|
||||
candidates: [
|
||||
{
|
||||
runId: 'workflow-run',
|
||||
stepRunId: 'workflow-step',
|
||||
readyAtMs: 2,
|
||||
planDigest: 'a'.repeat(64),
|
||||
},
|
||||
],
|
||||
truncated: false,
|
||||
};
|
||||
},
|
||||
async admit(runId, stepRunId) {
|
||||
calls.push(`task-admit:${runId}:${stepRunId}`);
|
||||
return { status: 'created', receipt: {} };
|
||||
},
|
||||
},
|
||||
{
|
||||
async dispatchOnce() {
|
||||
dispatches += 1;
|
||||
calls.push(`dispatch:${dispatches}`);
|
||||
const stats = {
|
||||
pages: 1,
|
||||
candidatesScanned: dispatches === 1 ? 1 : 0,
|
||||
plansUnavailable: 0,
|
||||
activationRaces: 0,
|
||||
};
|
||||
return dispatches === 1
|
||||
? {
|
||||
status: 'activated',
|
||||
runId: 'workflow-run',
|
||||
attemptId: 'workflow-attempt',
|
||||
stats,
|
||||
truncated: false,
|
||||
}
|
||||
: {
|
||||
status: 'idle',
|
||||
reason: 'no_candidates',
|
||||
stats,
|
||||
truncated: false,
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
cancellationPageSize: 1,
|
||||
cancellationMaxPages: 1,
|
||||
frontierPageSize: 1,
|
||||
frontierMaxPages: 1,
|
||||
taskAttemptPageSize: 1,
|
||||
taskAttemptMaxPages: 1,
|
||||
maxDispatches: 2,
|
||||
},
|
||||
);
|
||||
|
||||
assert.deepEqual(await coordinator.scheduleOnce(), schedulerSummary);
|
||||
assert.deepEqual(calls, [
|
||||
'cancel:1',
|
||||
'schedule',
|
||||
'frontier-list:1',
|
||||
'frontier-advance:workflow-run',
|
||||
'task-list:1',
|
||||
'task-admit:workflow-run:workflow-step',
|
||||
'dispatch:1',
|
||||
'dispatch:2',
|
||||
]);
|
||||
assert.deepEqual(coordinator.latestWorkflowSummary(), {
|
||||
cancellation: {
|
||||
pages: 1,
|
||||
scanned: 0,
|
||||
settledRuns: 0,
|
||||
settledAttempts: 0,
|
||||
blocked: 0,
|
||||
hasMore: false,
|
||||
remaining: false,
|
||||
stopReason: 'complete',
|
||||
},
|
||||
frontierPages: 1,
|
||||
frontierScanned: 1,
|
||||
frontierAdvanced: 1,
|
||||
frontierTruncated: false,
|
||||
taskAttemptPages: 1,
|
||||
taskAttemptsScanned: 1,
|
||||
taskAttemptsCreated: 1,
|
||||
taskAttemptsExisting: 0,
|
||||
taskAttemptsTruncated: false,
|
||||
dispatches: 2,
|
||||
activated: 1,
|
||||
activationFailed: 0,
|
||||
dispatchIdle: true,
|
||||
});
|
||||
});
|
||||
|
||||
test('lifecycle coalesces cycles and stops without leaving a scheduler timer', async () => {
|
||||
let calls = 0;
|
||||
let release;
|
||||
const pending = new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const summary = {
|
||||
observedAtMs: 1,
|
||||
scanned: 0,
|
||||
initialized: 0,
|
||||
skipped: 0,
|
||||
admitted: 0,
|
||||
raced: 0,
|
||||
truncated: false,
|
||||
};
|
||||
const lifecycle = new LocalSchedulerLifecycle(
|
||||
{
|
||||
async scheduleOnce() {
|
||||
calls += 1;
|
||||
await pending;
|
||||
return summary;
|
||||
},
|
||||
},
|
||||
{ intervalMs: 250, stopTimeoutMs: 1_000 },
|
||||
);
|
||||
|
||||
assert.equal(lifecycle.start(), 'started');
|
||||
const first = lifecycle.runOnce();
|
||||
const second = lifecycle.runOnce();
|
||||
assert.equal(first, second);
|
||||
const stopping = lifecycle.stopAndDrain();
|
||||
release();
|
||||
assert.deepEqual(await first, summary);
|
||||
assert.deepEqual(await stopping, { status: 'stopped' });
|
||||
assert.equal(calls, 1);
|
||||
await assert.rejects(lifecycle.runOnce(), /stopping/);
|
||||
});
|
||||
|
||||
test('lifecycle bounds shutdown when a schedule transaction does not settle', async () => {
|
||||
const lifecycle = new LocalSchedulerLifecycle(
|
||||
{ scheduleOnce: () => new Promise(() => {}) },
|
||||
{ intervalMs: 250, stopTimeoutMs: 100 },
|
||||
);
|
||||
void lifecycle.runOnce();
|
||||
assert.deepEqual(await lifecycle.stopAndDrain(), { status: 'timed_out' });
|
||||
});
|
||||
|
||||
test('lifecycle shutdown absorbs an already isolated cycle failure', async () => {
|
||||
let rejectCycle;
|
||||
const lifecycle = new LocalSchedulerLifecycle(
|
||||
{
|
||||
scheduleOnce: () =>
|
||||
new Promise((resolve, reject) => {
|
||||
rejectCycle = reject;
|
||||
}),
|
||||
},
|
||||
{ intervalMs: 250, stopTimeoutMs: 1_000 },
|
||||
);
|
||||
const cycle = lifecycle.runOnce();
|
||||
const stopping = lifecycle.stopAndDrain();
|
||||
rejectCycle(new Error('schedule storage unavailable'));
|
||||
await assert.rejects(cycle, /schedule storage unavailable/);
|
||||
assert.deepEqual(await stopping, { status: 'stopped' });
|
||||
});
|
||||
|
||||
test('lifecycle runs an unrefed non-overlapping cadence and isolates diagnostics', async () => {
|
||||
let active = 0;
|
||||
let maximumActive = 0;
|
||||
let calls = 0;
|
||||
const diagnostics = [];
|
||||
const lifecycle = new LocalSchedulerLifecycle(
|
||||
{
|
||||
async scheduleOnce() {
|
||||
active += 1;
|
||||
maximumActive = Math.max(maximumActive, active);
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
active -= 1;
|
||||
calls += 1;
|
||||
if (calls === 1) throw new Error('temporary schedule failure');
|
||||
return {
|
||||
observedAtMs: calls,
|
||||
scanned: 0,
|
||||
initialized: 0,
|
||||
skipped: 0,
|
||||
admitted: 0,
|
||||
raced: 0,
|
||||
truncated: false,
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
intervalMs: 250,
|
||||
stopTimeoutMs: 1_000,
|
||||
onDiagnostic(error, summary) {
|
||||
diagnostics.push({ error, summary });
|
||||
if (error === undefined) throw new Error('diagnostic sink unavailable');
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
lifecycle.start();
|
||||
await new Promise((resolve) => setTimeout(resolve, 650));
|
||||
assert.deepEqual(await lifecycle.stopAndDrain(), { status: 'stopped' });
|
||||
assert.ok(calls >= 2);
|
||||
assert.equal(maximumActive, 1);
|
||||
assert.ok(diagnostics.some(({ error }) => error instanceof Error));
|
||||
assert.ok(diagnostics.some(({ summary }) => summary?.observedAtMs >= 2));
|
||||
});
|
||||
Reference in New Issue
Block a user