mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 01:00:24 +08:00
460 lines
13 KiB
JavaScript
460 lines
13 KiB
JavaScript
const assert = require('node:assert/strict');
|
|
const { createHash } = require('node:crypto');
|
|
const fs = require('node:fs');
|
|
const os = require('node:os');
|
|
const path = require('node:path');
|
|
const { test } = require('node:test');
|
|
const {
|
|
migrateLocalSqlitePath,
|
|
openLocalSqliteRuntimeDatabase,
|
|
} = require('@qinglong/local-sqlite');
|
|
const {
|
|
LocalCompletionReceiptProcessor,
|
|
LocalExecutionControlCoordinator,
|
|
LocalExecutionControlLifecycle,
|
|
LocalExecutionControlScanner,
|
|
} = require('../dist/control');
|
|
|
|
const IDS = Object.freeze({
|
|
completionRun: '019f7130-0000-7000-8000-000000000001',
|
|
completionAttempt: '019f7130-0000-7000-8000-000000000002',
|
|
deadlineRun: '019f7130-0000-7000-8000-000000000003',
|
|
deadlineAttempt: '019f7130-0000-7000-8000-000000000004',
|
|
cancelRun: '019f7130-0000-7000-8000-000000000005',
|
|
cancelAttempt: '019f7130-0000-7000-8000-000000000006',
|
|
shutdownRun: '019f7130-0000-7000-8000-000000000007',
|
|
shutdownAttempt: '019f7130-0000-7000-8000-000000000008',
|
|
});
|
|
const TOKEN = 'A'.repeat(32);
|
|
|
|
function eventIds() {
|
|
let value = 0;
|
|
return () => `control-event-${++value}`;
|
|
}
|
|
|
|
function receiptStore() {
|
|
const receipts = new Map();
|
|
return {
|
|
receipts,
|
|
removed: [],
|
|
async publish(receipt) {
|
|
receipts.set(receipt.attemptId, receipt);
|
|
},
|
|
async read(attemptId) {
|
|
return receipts.get(attemptId);
|
|
},
|
|
async remove(attemptId) {
|
|
this.removed.push(attemptId);
|
|
return receipts.delete(attemptId);
|
|
},
|
|
};
|
|
}
|
|
|
|
async function fixture(t) {
|
|
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-control-'));
|
|
const databasePath = path.join(directory, 'qinglong3.sqlite');
|
|
await migrateLocalSqlitePath({ databasePath, profile: 'edge' });
|
|
const runtime = await openLocalSqliteRuntimeDatabase({
|
|
databasePath,
|
|
profile: 'edge',
|
|
});
|
|
t.after(async () => {
|
|
await runtime.close();
|
|
fs.rmSync(directory, { recursive: true, force: true });
|
|
});
|
|
return runtime;
|
|
}
|
|
|
|
async function insertActive(runtime, options) {
|
|
await runtime.runRepository.transaction(async (transaction) => {
|
|
await transaction.insertRun({
|
|
id: options.runId,
|
|
projectId: 'default',
|
|
taskId: 'task-1',
|
|
taskRevision: 'revision-1',
|
|
triggerType: 'manual',
|
|
executionOrigin: 'manual',
|
|
executionOwner: 'runtime',
|
|
status: options.runStatus || 'running',
|
|
version: 0,
|
|
eventSequence: 0,
|
|
priority: 0,
|
|
createdAtMs: 1,
|
|
startedAtMs: 2,
|
|
...(options.cancelRequestedAtMs === undefined
|
|
? {}
|
|
: {
|
|
cancelRequestedAtMs: options.cancelRequestedAtMs,
|
|
cancelReason: options.cancelReason,
|
|
}),
|
|
});
|
|
await transaction.insertAttempt({
|
|
id: options.attemptId,
|
|
runId: options.runId,
|
|
attempt: 1,
|
|
status: options.attemptStatus || 'running',
|
|
executorType: 'local_process',
|
|
executorHandle: `handle:${options.attemptId}`,
|
|
pid: 1234,
|
|
callbackTokenHash: createHash('sha256').update(TOKEN).digest('hex'),
|
|
callbackSequence: options.callbackSequence || 0,
|
|
createdAtMs: 1,
|
|
startedAtMs: 2,
|
|
...(options.deadlineAtMs === undefined
|
|
? {}
|
|
: { deadlineAtMs: options.deadlineAtMs }),
|
|
});
|
|
});
|
|
}
|
|
|
|
test('authenticates one receipt and commits Attempt then Run terminal facts', async (t) => {
|
|
const runtime = await fixture(t);
|
|
const store = receiptStore();
|
|
await insertActive(runtime, {
|
|
runId: IDS.completionRun,
|
|
attemptId: IDS.completionAttempt,
|
|
});
|
|
store.receipts.set(IDS.completionAttempt, {
|
|
schemaVersion: 1,
|
|
runId: IDS.completionRun,
|
|
attemptId: IDS.completionAttempt,
|
|
callbackSequence: 1,
|
|
token: TOKEN,
|
|
startedAtMs: 2,
|
|
finishedAtMs: 10,
|
|
exitCode: 0,
|
|
});
|
|
const processor = new LocalCompletionReceiptProcessor(
|
|
runtime.runRepository,
|
|
store,
|
|
{ clock: { now: () => 10 }, createEventId: eventIds() },
|
|
);
|
|
assert.equal(await processor.process(IDS.completionAttempt), 'completed');
|
|
assert.equal(
|
|
(await runtime.runRepository.findRunById(IDS.completionRun)).status,
|
|
'succeeded',
|
|
);
|
|
assert.equal(
|
|
(await runtime.runRepository.findAttemptById(IDS.completionAttempt)).status,
|
|
'succeeded',
|
|
);
|
|
assert.deepEqual(
|
|
(await runtime.runRepository.listEvents(IDS.completionRun)).map(
|
|
(event) => event.type,
|
|
),
|
|
['attempt.succeeded', 'run.succeeded'],
|
|
);
|
|
assert.deepEqual(store.removed, [IDS.completionAttempt]);
|
|
});
|
|
|
|
test('discovers due deadlines and cancellation intents through stable SQLite pages', async (t) => {
|
|
const runtime = await fixture(t);
|
|
await insertActive(runtime, {
|
|
runId: IDS.deadlineRun,
|
|
attemptId: IDS.deadlineAttempt,
|
|
deadlineAtMs: 50,
|
|
});
|
|
await insertActive(runtime, {
|
|
runId: IDS.cancelRun,
|
|
attemptId: IDS.cancelAttempt,
|
|
cancelRequestedAtMs: 40,
|
|
cancelReason: 'user',
|
|
deadlineAtMs: 30,
|
|
});
|
|
const first =
|
|
await runtime.executionControl.listLocalExecutionControlCandidates({
|
|
observedAtMs: 100,
|
|
limit: 1,
|
|
});
|
|
assert.equal(first.truncated, true);
|
|
assert.deepEqual(first.candidates[0], {
|
|
kind: 'cancellation',
|
|
runId: IDS.cancelRun,
|
|
attemptId: IDS.cancelAttempt,
|
|
dueAtMs: 40,
|
|
cancelReason: 'user',
|
|
});
|
|
const second =
|
|
await runtime.executionControl.listLocalExecutionControlCandidates({
|
|
observedAtMs: 100,
|
|
limit: 1,
|
|
after: first.nextCursor,
|
|
});
|
|
assert.equal(second.truncated, false);
|
|
assert.deepEqual(second.candidates[0], {
|
|
kind: 'deadline',
|
|
runId: IDS.deadlineRun,
|
|
attemptId: IDS.deadlineAttempt,
|
|
dueAtMs: 50,
|
|
});
|
|
});
|
|
|
|
test('turns deadline and user cancellation into exact terminal aggregates', async (t) => {
|
|
const runtime = await fixture(t);
|
|
await insertActive(runtime, {
|
|
runId: IDS.deadlineRun,
|
|
attemptId: IDS.deadlineAttempt,
|
|
deadlineAtMs: 50,
|
|
});
|
|
await insertActive(runtime, {
|
|
runId: IDS.cancelRun,
|
|
attemptId: IDS.cancelAttempt,
|
|
cancelRequestedAtMs: 40,
|
|
cancelReason: 'user',
|
|
});
|
|
const stopped = [];
|
|
const store = receiptStore();
|
|
const completion = new LocalCompletionReceiptProcessor(
|
|
runtime.runRepository,
|
|
store,
|
|
{ clock: { now: () => 100 }, createEventId: eventIds() },
|
|
);
|
|
const coordinator = new LocalExecutionControlCoordinator(
|
|
runtime.runRepository,
|
|
completion,
|
|
{
|
|
async stop(handle) {
|
|
stopped.push(handle);
|
|
return { status: 'stopped', signal: 'SIGTERM' };
|
|
},
|
|
},
|
|
{ clock: { now: () => 100 }, createEventId: eventIds() },
|
|
);
|
|
const scanner = new LocalExecutionControlScanner(
|
|
runtime.executionControl,
|
|
coordinator,
|
|
{ now: () => 100 },
|
|
);
|
|
const summary = await scanner.scan({ limit: 8 });
|
|
assert.equal(summary.scanned, 2);
|
|
assert.equal(summary.terminal, 2);
|
|
assert.equal(
|
|
(await runtime.runRepository.findRunById(IDS.deadlineRun)).status,
|
|
'timed_out',
|
|
);
|
|
assert.equal(
|
|
(await runtime.runRepository.findRunById(IDS.cancelRun)).status,
|
|
'cancelled',
|
|
);
|
|
assert.equal(stopped.length, 2);
|
|
});
|
|
|
|
test('routes a Workflow Task deadline to Step scope without cancelling its parent', async () => {
|
|
const run = {
|
|
id: 'workflow-run',
|
|
projectId: 'default',
|
|
taskId: 'workflow',
|
|
taskRevision: 'revision-1',
|
|
triggerType: 'system',
|
|
executionOrigin: 'system',
|
|
executionOwner: 'runtime',
|
|
status: 'running',
|
|
version: 7,
|
|
eventSequence: 7,
|
|
priority: 0,
|
|
createdAtMs: 1,
|
|
startedAtMs: 2,
|
|
};
|
|
const attempt = {
|
|
id: 'workflow-attempt',
|
|
runId: run.id,
|
|
stepRunId: 'workflow-step',
|
|
attempt: 1,
|
|
status: 'running',
|
|
executorType: 'local_process',
|
|
executorHandle: 'workflow-handle',
|
|
pid: 321,
|
|
callbackTokenHash: 'a'.repeat(64),
|
|
callbackSequence: 0,
|
|
deadlineAtMs: 50,
|
|
createdAtMs: 1,
|
|
startedAtMs: 2,
|
|
};
|
|
let latestReads = 0;
|
|
const repository = {
|
|
transaction(work) {
|
|
return work({
|
|
findRunById: async (runId) => (runId === run.id ? run : null),
|
|
findAttemptById: async (attemptId) =>
|
|
attemptId === attempt.id ? attempt : null,
|
|
findLatestAttemptByRunId: async () => {
|
|
latestReads += 1;
|
|
return attempt;
|
|
},
|
|
});
|
|
},
|
|
};
|
|
const workflowCalls = [];
|
|
const coordinator = new LocalExecutionControlCoordinator(
|
|
repository,
|
|
{ process: async () => 'missing' },
|
|
{ stop: async () => ({ status: 'stopped', signal: 'SIGTERM' }) },
|
|
{
|
|
clock: { now: () => 100 },
|
|
createEventId: eventIds(),
|
|
workflowTasks: {
|
|
async requestTimeout(command) {
|
|
workflowCalls.push(['timeout', command]);
|
|
return 'requested';
|
|
},
|
|
async recordControlTerminal(command) {
|
|
workflowCalls.push(['terminal', command]);
|
|
return 'terminal';
|
|
},
|
|
},
|
|
},
|
|
);
|
|
|
|
assert.equal(
|
|
await coordinator.process({
|
|
kind: 'deadline',
|
|
runId: run.id,
|
|
attemptId: attempt.id,
|
|
dueAtMs: 50,
|
|
}),
|
|
'terminal',
|
|
);
|
|
assert.equal(latestReads, 0);
|
|
assert.equal(workflowCalls[0][0], 'timeout');
|
|
assert.equal(workflowCalls[1][0], 'terminal');
|
|
assert.equal(workflowCalls[1][1].terminalStatus, 'timed_out');
|
|
assert.equal(run.cancelRequestedAtMs, undefined);
|
|
});
|
|
|
|
test('shutdown drain requests shutdown cancellation before stopping work', async (t) => {
|
|
const runtime = await fixture(t);
|
|
await insertActive(runtime, {
|
|
runId: IDS.shutdownRun,
|
|
attemptId: IDS.shutdownAttempt,
|
|
});
|
|
const completion = new LocalCompletionReceiptProcessor(
|
|
runtime.runRepository,
|
|
receiptStore(),
|
|
{ clock: { now: () => 100 }, createEventId: eventIds() },
|
|
);
|
|
const coordinator = new LocalExecutionControlCoordinator(
|
|
runtime.runRepository,
|
|
completion,
|
|
{ stop: async () => ({ status: 'already_exited' }) },
|
|
{ clock: { now: () => 100 }, createEventId: eventIds() },
|
|
);
|
|
const scanner = new LocalExecutionControlScanner(
|
|
runtime.executionControl,
|
|
coordinator,
|
|
{ now: () => 100 },
|
|
);
|
|
const summary = await scanner.drain({ limit: 4, maxPages: 2 });
|
|
assert.deepEqual(summary, {
|
|
scanned: 1,
|
|
terminal: 1,
|
|
remaining: 0,
|
|
failed: 0,
|
|
truncated: false,
|
|
});
|
|
const run = await runtime.runRepository.findRunById(IDS.shutdownRun);
|
|
assert.equal(run.status, 'cancelled');
|
|
assert.equal(run.cancelReason, 'shutdown');
|
|
assert.deepEqual(
|
|
(await runtime.runRepository.listEvents(IDS.shutdownRun)).map(
|
|
(event) => event.type,
|
|
),
|
|
['run.cancel_requested', 'attempt.cancelled', 'run.cancelled'],
|
|
);
|
|
});
|
|
|
|
test('coalesces completion notifications and owns one idempotent shutdown drain', async () => {
|
|
const completions = [];
|
|
let scans = 0;
|
|
let drains = 0;
|
|
let cleanups = 0;
|
|
let retentionSweeps = 0;
|
|
const lifecycle = new LocalExecutionControlLifecycle(
|
|
{
|
|
async process(attemptId) {
|
|
completions.push(attemptId);
|
|
return 'missing';
|
|
},
|
|
},
|
|
{
|
|
async scan() {
|
|
scans += 1;
|
|
return {
|
|
scanned: 0,
|
|
terminal: 0,
|
|
cancelRequested: 0,
|
|
stale: 0,
|
|
remaining: 0,
|
|
failed: 0,
|
|
truncated: false,
|
|
};
|
|
},
|
|
async drain() {
|
|
drains += 1;
|
|
return {
|
|
scanned: 0,
|
|
terminal: 0,
|
|
remaining: 0,
|
|
failed: 0,
|
|
truncated: false,
|
|
};
|
|
},
|
|
},
|
|
{
|
|
async scan() {
|
|
cleanups += 1;
|
|
return {
|
|
scanned: 0,
|
|
removed: 0,
|
|
expiredMissing: 0,
|
|
purgedQuarantines: 0,
|
|
remaining: 0,
|
|
failed: 0,
|
|
truncated: false,
|
|
};
|
|
},
|
|
},
|
|
{
|
|
intervalMs: 60_000,
|
|
pageSize: 4,
|
|
cleanupIntervalMs: 60_000,
|
|
cleanupPageSize: 4,
|
|
stopTimeoutMs: 1_000,
|
|
maxDrainPages: 1,
|
|
clock: { now: () => 100 },
|
|
artifactRetention: {
|
|
async sweep() {
|
|
retentionSweeps += 1;
|
|
return {
|
|
status: 'complete',
|
|
pressure: false,
|
|
observedAtMs: 100,
|
|
retentionMs: 60_000,
|
|
availableBytes: '100',
|
|
totalBytes: '100',
|
|
candidatesScanned: 0,
|
|
deletionsAttempted: 0,
|
|
recordsWritten: 0,
|
|
failedCandidates: 0,
|
|
bytesReclaimed: 0,
|
|
entries: [],
|
|
};
|
|
},
|
|
},
|
|
},
|
|
);
|
|
assert.equal(lifecycle.notifyCompletion(IDS.completionAttempt), true);
|
|
assert.equal(lifecycle.notifyCompletion(IDS.completionAttempt), true);
|
|
await new Promise((resolve) => setImmediate(resolve));
|
|
assert.deepEqual(completions, [IDS.completionAttempt]);
|
|
assert.equal(scans, 1);
|
|
assert.equal(cleanups, 1);
|
|
assert.equal(retentionSweeps, 1);
|
|
const first = lifecycle.stopAndDrain();
|
|
const second = lifecycle.stopAndDrain();
|
|
assert.equal(first, second);
|
|
assert.equal((await first).status, 'stopped');
|
|
assert.equal(drains, 1);
|
|
assert.equal(cleanups, 2);
|
|
assert.equal(retentionSweeps, 1);
|
|
});
|