mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,438 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PostgresClusterControlRecoveryResolutionRepository,
|
||||
} = require('../dist');
|
||||
const {
|
||||
pluginPackageWorkflowTaskAttemptAdmissionReceiptDigest,
|
||||
} = require('@qinglong/runtime-core/plugin-package-workflow-task-attempt-admission');
|
||||
const {
|
||||
createStepRunRecord,
|
||||
} = require('@qinglong/runtime-core/step-run');
|
||||
|
||||
const RUN_ID = '019f70b0-0000-7000-8000-000000000101';
|
||||
const ATTEMPT_ID = '019f70b0-0000-7000-8000-000000000102';
|
||||
const STEP_RUN_ID = '019f70b0-0000-7000-8000-000000000103';
|
||||
|
||||
function workflowStepRun() {
|
||||
return createStepRunRecord({
|
||||
id: STEP_RUN_ID,
|
||||
runId: RUN_ID,
|
||||
stepKey: 'collect',
|
||||
kind: 'task',
|
||||
definitionRef: 'pkg:demo:alpha',
|
||||
definitionDigest: 'a'.repeat(64),
|
||||
required: true,
|
||||
initialStatus: 'ready',
|
||||
mutationId: '019f70b0-0000-7000-8000-000000000104',
|
||||
createdAtMs: 300,
|
||||
});
|
||||
}
|
||||
|
||||
function workflowAdmission(stepRun) {
|
||||
const unsigned = {
|
||||
schema:
|
||||
'qinglong/plugin-package-workflow-task-attempt-admission@v1',
|
||||
attemptId: ATTEMPT_ID,
|
||||
planDigest: 'c'.repeat(64),
|
||||
runId: RUN_ID,
|
||||
stepRunId: STEP_RUN_ID,
|
||||
stepRunVersion: stepRun.version,
|
||||
stepRunDigest: stepRun.stepRunDigest,
|
||||
resourceTaskId: 'alpha',
|
||||
taskReconciliationReceiptDigest: 'd'.repeat(64),
|
||||
taskId: 'pkg:demo:alpha',
|
||||
taskRevision: `qltd:v1:1:${'a'.repeat(64)}`,
|
||||
taskDefinitionDigest: 'a'.repeat(64),
|
||||
executorType: 'remote_worker',
|
||||
executionDigest: 'e'.repeat(64),
|
||||
attemptNumber: 1,
|
||||
eventId: '019f70b0-0000-7000-8000-000000000105',
|
||||
runVersion: 3,
|
||||
runEventSequence: 3,
|
||||
admittedAtMs: 400,
|
||||
};
|
||||
return {
|
||||
...unsigned,
|
||||
receiptDigest:
|
||||
pluginPackageWorkflowTaskAttemptAdmissionReceiptDigest(unsigned),
|
||||
};
|
||||
}
|
||||
|
||||
function claim() {
|
||||
return {
|
||||
candidate: {
|
||||
kind: 'attempt',
|
||||
id: ATTEMPT_ID,
|
||||
runId: RUN_ID,
|
||||
status: 'claimed',
|
||||
createdAtMs: 200,
|
||||
},
|
||||
observedAtMs: 1000,
|
||||
ownerId: 'replica-a',
|
||||
token: '00000000-0000-4000-8000-000000000001',
|
||||
version: 1,
|
||||
expiresAtMs: 31000,
|
||||
};
|
||||
}
|
||||
|
||||
function runRow(overrides = {}) {
|
||||
return {
|
||||
id: RUN_ID,
|
||||
projectId: 'default',
|
||||
taskId: 'task-1',
|
||||
taskRevision: 'v1',
|
||||
taskName: null,
|
||||
taskSnapshotRef: null,
|
||||
legacyCronId: null,
|
||||
parentRunId: null,
|
||||
retryOfRunId: null,
|
||||
triggerId: null,
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
triggeredBy: null,
|
||||
requestId: null,
|
||||
scheduledForMs: null,
|
||||
status: 'dispatching',
|
||||
version: 1,
|
||||
eventSequence: 0,
|
||||
priority: 0,
|
||||
idempotencyKey: null,
|
||||
inputRef: null,
|
||||
outputRef: null,
|
||||
createdAtMs: '100',
|
||||
queuedAtMs: null,
|
||||
startedAtMs: null,
|
||||
finishedAtMs: null,
|
||||
cancelRequestedAtMs: null,
|
||||
cancelReason: null,
|
||||
errorCode: null,
|
||||
errorSummary: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function attemptRow(overrides = {}) {
|
||||
return {
|
||||
id: ATTEMPT_ID,
|
||||
runId: RUN_ID,
|
||||
stepRunId: null,
|
||||
attempt: 1,
|
||||
status: 'claimed',
|
||||
executorType: 'worker',
|
||||
workerId: 'worker-1',
|
||||
executorHandle: null,
|
||||
pid: null,
|
||||
logArtifactId: null,
|
||||
leaseToken: 'lease-token',
|
||||
leaseExpiresAtMs: '900',
|
||||
deadlineAtMs: null,
|
||||
callbackTokenHash: null,
|
||||
callbackSequence: 0,
|
||||
createdAtMs: '200',
|
||||
startedAtMs: null,
|
||||
finishedAtMs: null,
|
||||
exitCode: null,
|
||||
errorCode: null,
|
||||
errorSummary: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function harness(options = {}) {
|
||||
const queries = [];
|
||||
let released = 0;
|
||||
let runUpdates = 0;
|
||||
const client = {
|
||||
async query(text, values = []) {
|
||||
queries.push({ text, values });
|
||||
if (text.includes('run_recovery_controls')) {
|
||||
return options.fenced
|
||||
? { rows: [], rowCount: 0 }
|
||||
: { rows: [{ observedAtMs: '1000' }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."runs"')) {
|
||||
return {
|
||||
rows: [
|
||||
runRow(
|
||||
options.workflowTask
|
||||
? {
|
||||
taskId: 'workflow-alpha',
|
||||
taskRevision: 'b'.repeat(64),
|
||||
triggerType: 'plugin_package_workflow',
|
||||
executionOrigin: 'system',
|
||||
requestId: 'workflow-plan-1',
|
||||
idempotencyKey:
|
||||
'plugin-package-workflow:workflow-plan-1',
|
||||
status: 'running',
|
||||
version: 5,
|
||||
eventSequence: 5,
|
||||
startedAtMs: '200',
|
||||
}
|
||||
: {},
|
||||
),
|
||||
],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."run_attempts"')) {
|
||||
return {
|
||||
rows: [
|
||||
attemptRow(
|
||||
{
|
||||
...(options.workflowTask
|
||||
? {
|
||||
stepRunId: STEP_RUN_ID,
|
||||
executorType: 'remote_worker',
|
||||
createdAtMs: '400',
|
||||
}
|
||||
: {}),
|
||||
...(options.leased
|
||||
? {
|
||||
workerSessionId:
|
||||
'019f70b0-0000-7000-8000-000000000201',
|
||||
workerGeneration: 2,
|
||||
leaseTokenDigest: 'f'.repeat(64),
|
||||
leaseGeneration: 3,
|
||||
leaseVersion: 7,
|
||||
offerId: 'offer-1',
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
),
|
||||
],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'plugin_package_workflow_task_attempt_admissions',
|
||||
)
|
||||
) {
|
||||
return options.workflowTask
|
||||
? {
|
||||
rows: [{
|
||||
admissionJson: options.workflowTask.admission,
|
||||
stepRunJson: options.workflowTask.stepRun,
|
||||
}],
|
||||
rowCount: 1,
|
||||
}
|
||||
: { rows: [], rowCount: 0 };
|
||||
}
|
||||
if (text.startsWith('UPDATE "ql3"."runs"')) {
|
||||
runUpdates += 1;
|
||||
return { rows: [{ id: RUN_ID }], rowCount: 1 };
|
||||
}
|
||||
if (text.startsWith('UPDATE "ql3"."run_attempts"')) {
|
||||
return options.attemptConflict
|
||||
? { rows: [], rowCount: 0 }
|
||||
: { rows: [{ id: ATTEMPT_ID }], rowCount: 1 };
|
||||
}
|
||||
if (text.startsWith('UPDATE "ql3"."run_dispatch_leases"')) {
|
||||
return options.leaseConflict
|
||||
? { rows: [], rowCount: 0 }
|
||||
: { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text.startsWith('UPDATE "ql3"."step_runs"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text.startsWith('INSERT INTO "ql3"."run_events"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
text.startsWith(
|
||||
'INSERT INTO "ql3"."step_run_mutations"',
|
||||
)
|
||||
) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
return { rows: [], rowCount: 0 };
|
||||
},
|
||||
release() {
|
||||
released += 1;
|
||||
},
|
||||
};
|
||||
return {
|
||||
queries,
|
||||
released: () => released,
|
||||
runUpdates: () => runUpdates,
|
||||
pool: {
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
query: (...args) => client.query(...args),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('loads only under a live claim fence and returns a normalized snapshot', async () => {
|
||||
const database = harness();
|
||||
const repository = new PostgresClusterControlRecoveryResolutionRepository(
|
||||
database.pool,
|
||||
);
|
||||
const loaded = await repository.load(claim());
|
||||
|
||||
assert.equal(loaded.observedAtMs, 1000);
|
||||
assert.equal(loaded.run.status, 'dispatching');
|
||||
assert.equal(loaded.attempt.status, 'claimed');
|
||||
assert.equal(loaded.attempt.leaseExpiresAtMs, 900);
|
||||
assert.equal(Object.isFrozen(loaded), true);
|
||||
assert.equal(database.queries[4].text.includes('FOR UPDATE'), false);
|
||||
assert.equal(database.queries.at(-1).text, 'COMMIT');
|
||||
assert.equal(database.released(), 1);
|
||||
});
|
||||
|
||||
test('locks the claim and commits both lost transitions and events atomically', async () => {
|
||||
const database = harness();
|
||||
let event = 0;
|
||||
const repository = new PostgresClusterControlRecoveryResolutionRepository(
|
||||
database.pool,
|
||||
() => `00000000-0000-4000-8000-${String(++event).padStart(12, '0')}`,
|
||||
);
|
||||
const loaded = await repository.load(claim());
|
||||
assert.equal(
|
||||
await repository.applyLost(claim(), loaded, {
|
||||
kind: 'mark_attempt_and_run_lost',
|
||||
reason: 'unstarted_claim_expired',
|
||||
}),
|
||||
'applied',
|
||||
);
|
||||
|
||||
const applyQueries = database.queries.slice(8);
|
||||
assert.equal(
|
||||
applyQueries.some(({ text }) => text.includes('FOR UPDATE OF control')),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
applyQueries.filter(({ text }) => text.startsWith('UPDATE "ql3"."runs"'))
|
||||
.length,
|
||||
2,
|
||||
);
|
||||
assert.equal(
|
||||
applyQueries.filter(({ text }) =>
|
||||
text.startsWith('UPDATE "ql3"."run_attempts"'),
|
||||
).length,
|
||||
1,
|
||||
);
|
||||
const events = applyQueries.filter(({ text }) =>
|
||||
text.startsWith('INSERT INTO "ql3"."run_events"'),
|
||||
);
|
||||
assert.equal(events.length, 2);
|
||||
assert.equal(events[0].values.includes('reconciler'), true);
|
||||
assert.equal(events[0].values.includes('replica-a'), true);
|
||||
assert.equal(applyQueries.at(-1).text, 'COMMIT');
|
||||
});
|
||||
|
||||
test('releases an expired dispatch lease under the same Attempt authority transaction', async () => {
|
||||
const database = harness({ leased: true });
|
||||
const repository = new PostgresClusterControlRecoveryResolutionRepository(
|
||||
database.pool,
|
||||
);
|
||||
const loaded = await repository.load(claim());
|
||||
|
||||
assert.equal(
|
||||
await repository.applyLost(claim(), loaded, {
|
||||
kind: 'mark_attempt_and_run_lost',
|
||||
reason: 'unstarted_claim_expired',
|
||||
}),
|
||||
'applied',
|
||||
);
|
||||
const leaseUpdate = database.queries.find(({ text }) =>
|
||||
text.startsWith('UPDATE "ql3"."run_dispatch_leases"'),
|
||||
);
|
||||
assert.ok(leaseUpdate);
|
||||
assert.deepEqual(leaseUpdate.values, [
|
||||
ATTEMPT_ID,
|
||||
8,
|
||||
1000,
|
||||
7,
|
||||
3,
|
||||
]);
|
||||
const attemptUpdate = database.queries.find(({ text }) =>
|
||||
text.startsWith('UPDATE "ql3"."run_attempts"'),
|
||||
);
|
||||
assert.equal(attemptUpdate.values.includes(8), true);
|
||||
assert.equal(database.queries.at(-1).text, 'COMMIT');
|
||||
});
|
||||
|
||||
test('atomically loses and requeues one admission-bound Workflow Task epoch', async () => {
|
||||
const stepRun = workflowStepRun();
|
||||
const database = harness({
|
||||
workflowTask: {
|
||||
admission: workflowAdmission(stepRun),
|
||||
stepRun,
|
||||
},
|
||||
});
|
||||
const repository =
|
||||
new PostgresClusterControlRecoveryResolutionRepository(database.pool);
|
||||
const loaded = await repository.load(claim());
|
||||
assert.equal(loaded.workflowTask.admission.attemptId, ATTEMPT_ID);
|
||||
assert.equal(loaded.workflowTask.stepRun.status, 'ready');
|
||||
|
||||
assert.equal(
|
||||
await repository.applyLost(claim(), loaded, {
|
||||
kind: 'recover_workflow_task',
|
||||
reason: 'unstarted_claim_expired',
|
||||
}),
|
||||
'applied',
|
||||
);
|
||||
|
||||
const runUpdate = database.queries.find(
|
||||
({ text, values }) =>
|
||||
text.startsWith('UPDATE "ql3"."runs"') &&
|
||||
values[0] === RUN_ID &&
|
||||
values.length === 5,
|
||||
);
|
||||
assert.ok(runUpdate);
|
||||
assert.deepEqual(runUpdate.values.slice(1), [7, 7, 5, 5]);
|
||||
const attemptUpdate = database.queries.find(({ text }) =>
|
||||
text.startsWith('UPDATE "ql3"."run_attempts"'));
|
||||
assert.ok(attemptUpdate);
|
||||
assert.equal(attemptUpdate.values.includes('lost'), true);
|
||||
const stepUpdate = database.queries.find(({ text }) =>
|
||||
text.startsWith('UPDATE "ql3"."step_runs"'));
|
||||
assert.ok(stepUpdate);
|
||||
assert.equal(stepUpdate.values[0], 'ready');
|
||||
assert.equal(stepUpdate.values[1], stepRun.version + 1);
|
||||
assert.notEqual(stepUpdate.values[12], stepRun.stepRunDigest);
|
||||
const events = database.queries.filter(({ text }) =>
|
||||
text.startsWith('INSERT INTO "ql3"."run_events"'));
|
||||
assert.equal(events.length, 2);
|
||||
assert.equal(
|
||||
database.queries.filter(({ text }) =>
|
||||
text.startsWith('INSERT INTO "ql3"."step_run_mutations"'),
|
||||
).length,
|
||||
1,
|
||||
);
|
||||
assert.equal(database.queries.at(-1).text, 'COMMIT');
|
||||
});
|
||||
|
||||
test('rolls back partial CAS work and reports a stale aggregate', async () => {
|
||||
const database = harness({ attemptConflict: true });
|
||||
const repository = new PostgresClusterControlRecoveryResolutionRepository(
|
||||
database.pool,
|
||||
);
|
||||
const loaded = await repository.load(claim());
|
||||
assert.equal(
|
||||
await repository.applyLost(claim(), loaded, {
|
||||
kind: 'mark_attempt_and_run_lost',
|
||||
reason: 'unstarted_claim_expired',
|
||||
}),
|
||||
'stale',
|
||||
);
|
||||
assert.equal(database.runUpdates(), 1);
|
||||
assert.equal(database.queries.at(-1).text, 'ROLLBACK');
|
||||
});
|
||||
|
||||
test('does not read or mutate Run state after the recovery fence is lost', async () => {
|
||||
const database = harness({ fenced: true });
|
||||
const repository = new PostgresClusterControlRecoveryResolutionRepository(
|
||||
database.pool,
|
||||
);
|
||||
assert.equal(await repository.load(claim()), 'fenced');
|
||||
assert.equal(
|
||||
database.queries.some(({ text }) => text.includes('FROM "ql3"."runs"')),
|
||||
false,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user