mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): reconcile legacy shadow runs on startup
This commit is contained in:
whyour
@@ -0,0 +1,149 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
bootstrapLegacyShadowStartupReconciliation,
|
||||
} = require('../../back/runtime/adapters/legacy/bootstrapLegacyShadowStartupReconciliation');
|
||||
|
||||
function summary(overrides = {}) {
|
||||
return {
|
||||
pages: 1,
|
||||
scanned: 0,
|
||||
completed: 0,
|
||||
cancelled: 0,
|
||||
abandoned: 0,
|
||||
markedLost: 0,
|
||||
repaired: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('disabled bootstrap remains fully lazy', async () => {
|
||||
const calls = [];
|
||||
const result = await bootstrapLegacyShadowStartupReconciliation({
|
||||
origins: [],
|
||||
profile: 'edge',
|
||||
async execute() {
|
||||
calls.push('execute');
|
||||
throw new Error('must remain lazy');
|
||||
},
|
||||
audit(record) {
|
||||
calls.push(record.state);
|
||||
},
|
||||
});
|
||||
|
||||
assert.deepEqual(result, { state: 'disabled' });
|
||||
assert.deepEqual(calls, ['disabled']);
|
||||
});
|
||||
|
||||
test('cluster profiles reject local Shadow recovery without loading storage', async () => {
|
||||
for (const profile of ['cluster-control', 'worker']) {
|
||||
const result = await bootstrapLegacyShadowStartupReconciliation({
|
||||
origins: ['manual'],
|
||||
profile,
|
||||
async execute() {
|
||||
throw new Error('cluster profile must not load local storage');
|
||||
},
|
||||
audit() {},
|
||||
});
|
||||
assert.deepEqual(result, { state: 'profile_rejected', profile });
|
||||
}
|
||||
});
|
||||
|
||||
test('applies distinct one-shot edge and standalone budgets', async () => {
|
||||
const requests = [];
|
||||
for (const profile of ['edge', 'standalone']) {
|
||||
const result = await bootstrapLegacyShadowStartupReconciliation({
|
||||
origins: ['manual', 'scheduled_system'],
|
||||
profile,
|
||||
async execute(request) {
|
||||
requests.push(request);
|
||||
return summary();
|
||||
},
|
||||
audit() {},
|
||||
});
|
||||
assert.equal(result.state, 'reconciled');
|
||||
}
|
||||
|
||||
assert.deepEqual(
|
||||
requests.map(({ profile, pageSize, maxPages }) => ({
|
||||
profile,
|
||||
pageSize,
|
||||
maxPages,
|
||||
})),
|
||||
[
|
||||
{ profile: 'edge', pageSize: 8, maxPages: 1 },
|
||||
{ profile: 'standalone', pageSize: 32, maxPages: 4 },
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('fails open with a low-sensitivity error type', async () => {
|
||||
const result = await bootstrapLegacyShadowStartupReconciliation({
|
||||
origins: ['manual'],
|
||||
profile: 'edge',
|
||||
async execute() {
|
||||
throw new RangeError('secret command must not be reported');
|
||||
},
|
||||
audit() {},
|
||||
});
|
||||
|
||||
assert.deepEqual(result, { state: 'failed', errorType: 'RangeError' });
|
||||
assert.equal(JSON.stringify(result).includes('secret command'), false);
|
||||
});
|
||||
|
||||
test('redacts the resume cursor Run identity from startup audit output', async () => {
|
||||
const records = [];
|
||||
const result = await bootstrapLegacyShadowStartupReconciliation({
|
||||
origins: ['manual'],
|
||||
profile: 'edge',
|
||||
async execute() {
|
||||
return summary({
|
||||
stopReason: 'page_limit',
|
||||
remaining: true,
|
||||
nextCursor: { createdAtMs: 10, runId: 'run-secret-identity' },
|
||||
});
|
||||
},
|
||||
audit(record) {
|
||||
records.push(JSON.stringify(record));
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.state, 'incomplete');
|
||||
assert.equal(result.summary.resumeAvailable, true);
|
||||
assert.equal('nextCursor' in result.summary, false);
|
||||
assert.equal(records.join('').includes('run-secret-identity'), false);
|
||||
});
|
||||
|
||||
test('HTTP startup orders Shadow recovery after Legacy normalization and before Primary/listen', () => {
|
||||
const source = fs.readFileSync(
|
||||
path.resolve(__dirname, '../../back/app.ts'),
|
||||
'utf8',
|
||||
);
|
||||
const legacyNormalization = source.indexOf(
|
||||
'await appLoader.default({ app: this.app })',
|
||||
);
|
||||
const shadowRecovery = source.indexOf(
|
||||
'await bootstrapLegacyShadowStartupReconciliation()',
|
||||
);
|
||||
const primaryActivation = source.indexOf(
|
||||
'await bootstrapDefaultManualPrimaryRuntime()',
|
||||
);
|
||||
const listen = source.indexOf(
|
||||
'this.httpServerService.initialize(this.app, config.port)',
|
||||
);
|
||||
|
||||
assert.equal(legacyNormalization >= 0, true);
|
||||
assert.equal(legacyNormalization < shadowRecovery, true);
|
||||
assert.equal(shadowRecovery < primaryActivation, true);
|
||||
assert.equal(primaryActivation < listen, true);
|
||||
});
|
||||
@@ -0,0 +1,368 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const { DataTypes, Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const { runSchemaMigration } = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
runCancellationRequestMigration,
|
||||
} = require('../../back/migrations/0004-run-cancellation-request');
|
||||
const {
|
||||
runAttemptDeadlineMigration,
|
||||
} = require('../../back/migrations/0006-run-attempt-deadline');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeRunRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/runRepository');
|
||||
const {
|
||||
LegacySequelizeShadowStartupRecoverySource,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/legacyShadowStartupRecoverySource');
|
||||
const {
|
||||
LegacyShadowRunWriter,
|
||||
} = require('../../back/runtime/application/legacyShadowRunWriter');
|
||||
const {
|
||||
LegacyShadowStartupReconciler,
|
||||
LegacyShadowStartupSupervisor,
|
||||
} = require('../../back/runtime/application/legacyShadowStartupReconciler');
|
||||
const {
|
||||
RunCommandService,
|
||||
} = require('../../back/runtime/application/runCommandService');
|
||||
const {
|
||||
createLegacyLogArtifactId,
|
||||
} = require('../../back/runtime/compatibility/legacyTaskRevision');
|
||||
|
||||
const databases = [];
|
||||
let idSequence = 1_000;
|
||||
let timeSequence = 1_750_001_000_000;
|
||||
|
||||
function nextId() {
|
||||
idSequence += 1;
|
||||
return `019f7200-0000-7000-8000-${String(idSequence).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
function nextTime() {
|
||||
timeSequence += 100;
|
||||
return timeSequence;
|
||||
}
|
||||
|
||||
async function createStack() {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
runCancellationRequestMigration,
|
||||
runAttemptDeadlineMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
await database.getQueryInterface().createTable('RunningInstances', {
|
||||
id: {
|
||||
type: DataTypes.INTEGER,
|
||||
primaryKey: true,
|
||||
autoIncrement: true,
|
||||
},
|
||||
cron_id: { type: DataTypes.INTEGER, allowNull: false },
|
||||
pid: { type: DataTypes.INTEGER, allowNull: true },
|
||||
log_path: { type: DataTypes.STRING, allowNull: true },
|
||||
started_at: { type: DataTypes.INTEGER, allowNull: false },
|
||||
finished_at: { type: DataTypes.INTEGER, allowNull: true },
|
||||
status: { type: DataTypes.INTEGER, allowNull: false },
|
||||
exit_code: { type: DataTypes.INTEGER, allowNull: true },
|
||||
});
|
||||
databases.push(database);
|
||||
const repository = new LegacySequelizeRunRepository(database);
|
||||
const writer = new LegacyShadowRunWriter(repository, nextId);
|
||||
const source = new LegacySequelizeShadowStartupRecoverySource(
|
||||
database,
|
||||
createLegacyLogArtifactId,
|
||||
);
|
||||
const reconciler = new LegacyShadowStartupReconciler(
|
||||
repository,
|
||||
source,
|
||||
writer,
|
||||
{ clock: { now: () => nextTime() }, createEventId: nextId },
|
||||
);
|
||||
return { database, repository, writer, source, reconciler };
|
||||
}
|
||||
|
||||
async function activeShadow(writer, overrides = {}) {
|
||||
const acceptedAtMs = nextTime();
|
||||
const origin = overrides.origin ?? 'manual';
|
||||
const reference = await writer.accept({
|
||||
origin,
|
||||
projectId: 'default',
|
||||
taskId: `legacy-cron:${overrides.legacyCronId ?? 15}`,
|
||||
taskRevision: 'sha256:startup-reconciliation',
|
||||
legacyCronId: overrides.legacyCronId ?? 15,
|
||||
triggerType: origin,
|
||||
acceptedAtMs,
|
||||
});
|
||||
if (overrides.queuedOnly) return { reference, acceptedAtMs };
|
||||
await writer.spawned(reference, {
|
||||
atMs: acceptedAtMs + 1,
|
||||
...(overrides.pid === undefined ? {} : { pid: overrides.pid }),
|
||||
...(overrides.logPath === undefined
|
||||
? {}
|
||||
: { logArtifactId: createLegacyLogArtifactId(overrides.logPath) }),
|
||||
});
|
||||
await writer.running(reference, acceptedAtMs + 2);
|
||||
return { reference, acceptedAtMs };
|
||||
}
|
||||
|
||||
async function insertInstance(database, values) {
|
||||
await database.getQueryInterface().bulkInsert('RunningInstances', [
|
||||
{
|
||||
cron_id: 15,
|
||||
pid: null,
|
||||
log_path: null,
|
||||
started_at: 1_750_001_000,
|
||||
finished_at: null,
|
||||
status: 0,
|
||||
exit_code: null,
|
||||
...values,
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(databases.splice(0).map((database) => database.close()));
|
||||
});
|
||||
|
||||
test('completes an active Shadow Run from unique terminal RunningInstance evidence', async () => {
|
||||
const { database, repository, writer, reconciler } = await createStack();
|
||||
const logPath = 'cron/2026-08-18-01.log';
|
||||
const { reference } = await activeShadow(writer, {
|
||||
pid: 4101,
|
||||
logPath,
|
||||
});
|
||||
await insertInstance(database, {
|
||||
pid: 4101,
|
||||
log_path: logPath,
|
||||
finished_at: 1_750_001_010,
|
||||
status: 1,
|
||||
exit_code: 0,
|
||||
});
|
||||
|
||||
const summary = await reconciler.reconcileBatch({ origins: ['manual'] });
|
||||
|
||||
assert.equal(summary.completed, 1);
|
||||
assert.equal(summary.failed, 0);
|
||||
assert.equal(
|
||||
(await repository.findRunById(reference.runId)).status,
|
||||
'succeeded',
|
||||
);
|
||||
assert.equal(
|
||||
(await repository.findAttemptById(reference.attemptId)).status,
|
||||
'succeeded',
|
||||
);
|
||||
});
|
||||
|
||||
test('maps an explicitly finished stopped instance to reconciled cancellation', async () => {
|
||||
const { database, repository, writer, reconciler } = await createStack();
|
||||
const { reference } = await activeShadow(writer, { pid: 4151 });
|
||||
await insertInstance(database, {
|
||||
pid: 4151,
|
||||
finished_at: 1_750_001_011,
|
||||
status: 2,
|
||||
exit_code: 143,
|
||||
});
|
||||
|
||||
const summary = await reconciler.reconcileBatch({ origins: ['manual'] });
|
||||
|
||||
assert.equal(summary.cancelled, 1);
|
||||
assert.equal(
|
||||
(await repository.findRunById(reference.runId)).status,
|
||||
'cancelled',
|
||||
);
|
||||
assert.equal(
|
||||
(await repository.findAttemptById(reference.attemptId)).status,
|
||||
'cancelled',
|
||||
);
|
||||
});
|
||||
|
||||
test('pages active Shadow candidates with a stable keyset cursor', async () => {
|
||||
const { writer, source } = await createStack();
|
||||
for (const legacyCronId of [21, 22, 23]) {
|
||||
await activeShadow(writer, { legacyCronId, queuedOnly: true });
|
||||
}
|
||||
|
||||
const first = await source.listCandidates({ origins: ['manual'], limit: 2 });
|
||||
const second = await source.listCandidates({
|
||||
origins: ['manual'],
|
||||
cursor: first.nextCursor,
|
||||
limit: 2,
|
||||
});
|
||||
|
||||
assert.equal(first.truncated, true);
|
||||
assert.equal(first.candidates.length, 2);
|
||||
assert.equal(second.truncated, false);
|
||||
assert.equal(second.candidates.length, 1);
|
||||
assert.equal(
|
||||
new Set(
|
||||
[...first.candidates, ...second.candidates].map(({ runId }) => runId),
|
||||
).size,
|
||||
3,
|
||||
);
|
||||
});
|
||||
|
||||
test('marks a worker-owned execution lost when startup reset has no terminal evidence', async () => {
|
||||
const { database, repository, writer, reconciler } = await createStack();
|
||||
const { reference } = await activeShadow(writer, { pid: 4201 });
|
||||
await insertInstance(database, {
|
||||
pid: 4201,
|
||||
status: 2,
|
||||
finished_at: null,
|
||||
});
|
||||
|
||||
const summary = await reconciler.reconcileBatch({ origins: ['manual'] });
|
||||
|
||||
assert.equal(summary.markedLost, 1);
|
||||
assert.equal((await repository.findRunById(reference.runId)).status, 'lost');
|
||||
assert.equal(
|
||||
(await repository.findAttemptById(reference.attemptId)).status,
|
||||
'lost',
|
||||
);
|
||||
assert.equal(
|
||||
(await repository.listEvents(reference.runId)).at(-1).actorType,
|
||||
'reconciler',
|
||||
);
|
||||
});
|
||||
|
||||
test('abandons an accepted execution that never produced spawn evidence', async () => {
|
||||
const { repository, writer, reconciler } = await createStack();
|
||||
const { reference } = await activeShadow(writer, { queuedOnly: true });
|
||||
|
||||
const summary = await reconciler.reconcileBatch({ origins: ['manual'] });
|
||||
|
||||
assert.equal(summary.abandoned, 1);
|
||||
assert.equal(
|
||||
(await repository.findRunById(reference.runId)).status,
|
||||
'cancelled',
|
||||
);
|
||||
assert.equal(
|
||||
(await repository.findAttemptById(reference.attemptId)).status,
|
||||
'cancelled',
|
||||
);
|
||||
});
|
||||
|
||||
test('keeps system-crond pending without terminal callback evidence', async () => {
|
||||
const { database, repository, writer, reconciler } = await createStack();
|
||||
const { reference } = await activeShadow(writer, {
|
||||
origin: 'scheduled_system',
|
||||
pid: 4301,
|
||||
});
|
||||
await insertInstance(database, {
|
||||
pid: 4301,
|
||||
status: 2,
|
||||
finished_at: null,
|
||||
});
|
||||
|
||||
const summary = await reconciler.reconcileBatch({
|
||||
origins: ['scheduled_system'],
|
||||
});
|
||||
|
||||
assert.equal(summary.pending, 1);
|
||||
assert.equal(
|
||||
(await repository.findRunById(reference.runId)).status,
|
||||
'running',
|
||||
);
|
||||
assert.equal(
|
||||
(await repository.findAttemptById(reference.attemptId)).status,
|
||||
'running',
|
||||
);
|
||||
});
|
||||
|
||||
test('refuses ambiguous duplicate identity evidence instead of guessing a terminal result', async () => {
|
||||
const { database, repository, writer, reconciler } = await createStack();
|
||||
const logPath = 'cron/ambiguous.log';
|
||||
const { reference } = await activeShadow(writer, {
|
||||
pid: 4351,
|
||||
logPath,
|
||||
});
|
||||
for (const pid of [4352, 4353]) {
|
||||
await insertInstance(database, {
|
||||
pid,
|
||||
log_path: logPath,
|
||||
finished_at: 1_750_001_020,
|
||||
status: 1,
|
||||
exit_code: 0,
|
||||
});
|
||||
}
|
||||
|
||||
const summary = await reconciler.reconcileBatch({ origins: ['manual'] });
|
||||
|
||||
assert.equal(summary.ambiguous, 1);
|
||||
assert.equal(
|
||||
(await repository.findRunById(reference.runId)).status,
|
||||
'running',
|
||||
);
|
||||
assert.equal(
|
||||
(await repository.findAttemptById(reference.attemptId)).status,
|
||||
'running',
|
||||
);
|
||||
});
|
||||
|
||||
test('repairs a Run left active after its lost Attempt transaction committed', async () => {
|
||||
const { repository, writer, reconciler } = await createStack();
|
||||
const { reference } = await activeShadow(writer, { pid: 4401 });
|
||||
const run = await repository.findRunById(reference.runId);
|
||||
const commands = new RunCommandService(repository, nextId);
|
||||
await commands.transitionRunAttempt({
|
||||
runId: reference.runId,
|
||||
attemptId: reference.attemptId,
|
||||
to: 'lost',
|
||||
expectedRunVersion: run.version,
|
||||
atMs: nextTime(),
|
||||
errorCode: 'LEGACY_RECONCILE_OWNER_LOST',
|
||||
errorSummary: 'simulated response loss',
|
||||
actor: { type: 'reconciler' },
|
||||
});
|
||||
|
||||
const summary = await reconciler.reconcileBatch({ origins: ['manual'] });
|
||||
|
||||
assert.equal(summary.repaired, 1);
|
||||
assert.equal((await repository.findRunById(reference.runId)).status, 'lost');
|
||||
});
|
||||
|
||||
test('supervisor preserves a stable cursor when its Profile budget is exhausted', async () => {
|
||||
const calls = [];
|
||||
const supervisor = new LegacyShadowStartupSupervisor({
|
||||
async reconcileBatch(options) {
|
||||
calls.push(options);
|
||||
return {
|
||||
scanned: 1,
|
||||
completed: 0,
|
||||
cancelled: 0,
|
||||
abandoned: 0,
|
||||
markedLost: 1,
|
||||
repaired: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
truncated: true,
|
||||
nextCursor: { createdAtMs: 10, runId: 'run-10' },
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
const summary = await supervisor.run({
|
||||
origins: ['manual'],
|
||||
pageSize: 8,
|
||||
maxPages: 1,
|
||||
});
|
||||
|
||||
assert.equal(summary.stopReason, 'page_limit');
|
||||
assert.equal(summary.remaining, true);
|
||||
assert.deepEqual(summary.nextCursor, { createdAtMs: 10, runId: 'run-10' });
|
||||
assert.equal(calls[0].limit, 8);
|
||||
});
|
||||
Reference in New Issue
Block a user