mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
Vendored
+407
@@ -0,0 +1,407 @@
|
||||
const fs = require('node:fs');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
|
||||
const {
|
||||
LocalSqliteOperationAuthority,
|
||||
} = require('../../dist/authority/operationAuthority');
|
||||
const {
|
||||
LocalSqlitePluginPackageTaskReconciliationRepository,
|
||||
} = require('../../dist/plugin-package/pluginPackageTaskReconciliationRepository');
|
||||
const {
|
||||
LocalSqlitePluginPackageWorkflowAdmissionRepository,
|
||||
} = require('../../dist/plugin-package/workflow/pluginPackageWorkflowAdmissionRepository');
|
||||
const {
|
||||
LocalSqlitePluginPackageWorkflowTaskAttemptAdmissionRepository,
|
||||
} = require('../../dist/plugin-package/workflow/pluginPackageWorkflowTaskAttemptAdmissionRepository');
|
||||
const {
|
||||
LocalSqlitePluginPackageWorkflowCancellationConvergenceRepository,
|
||||
} = require('../../dist/plugin-package/workflow/pluginPackageWorkflowCancellationConvergenceRepository');
|
||||
const { LocalSqliteRunRepository } = require('../../dist/run/runRepository');
|
||||
const {
|
||||
LocalSqliteWorkflowTaskExecutionRepository,
|
||||
} = require('../../dist/plugin-package/workflow/workflowTaskExecutionRepository');
|
||||
const { auditLocalSqliteReadiness } = require('../../dist/readiness/readiness');
|
||||
const {
|
||||
executionPlan,
|
||||
fixture,
|
||||
setupScenario: setupWorkflowAdmissionScenario,
|
||||
} = require('./pluginPackageWorkflowAdmissionCrashMatrixFixture.cjs');
|
||||
|
||||
const CRASH_POINTS = Object.freeze({
|
||||
after_conclusive_stop_before_begin: Object.freeze({
|
||||
timing: 'beforeExec',
|
||||
sql: 'BEGIN IMMEDIATE',
|
||||
durable: false,
|
||||
}),
|
||||
after_attempt_terminal: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'UPDATE "RunAttempts"',
|
||||
durable: false,
|
||||
}),
|
||||
after_run_cas: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'UPDATE "Runs"',
|
||||
durable: false,
|
||||
}),
|
||||
after_step_terminal: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'UPDATE "StepRuns"',
|
||||
durable: false,
|
||||
}),
|
||||
after_attempt_event: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "RunEvents"',
|
||||
durable: false,
|
||||
}),
|
||||
after_step_mutation: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "StepRunMutations"',
|
||||
durable: false,
|
||||
}),
|
||||
before_commit: Object.freeze({
|
||||
timing: 'beforeExec',
|
||||
sql: 'COMMIT',
|
||||
durable: false,
|
||||
}),
|
||||
after_commit: Object.freeze({
|
||||
timing: 'afterExec',
|
||||
sql: 'COMMIT',
|
||||
durable: true,
|
||||
}),
|
||||
});
|
||||
|
||||
function client(databasePath) {
|
||||
const database = new DatabaseSync(databasePath);
|
||||
database.exec('PRAGMA foreign_keys = ON');
|
||||
return database;
|
||||
}
|
||||
|
||||
function writeCrashMarker(markerPath, pointName) {
|
||||
const descriptor = fs.openSync(markerPath, 'wx', 0o600);
|
||||
try {
|
||||
fs.writeSync(
|
||||
descriptor,
|
||||
JSON.stringify({
|
||||
schema:
|
||||
'qinglong/sqlite-plugin-package-workflow-task-control-crash-marker@v1',
|
||||
point: pointName,
|
||||
conclusiveStopObserved: true,
|
||||
pid: process.pid,
|
||||
}),
|
||||
);
|
||||
fs.fsyncSync(descriptor);
|
||||
} finally {
|
||||
fs.closeSync(descriptor);
|
||||
}
|
||||
}
|
||||
|
||||
function crashClient(database, pointName, markerPath) {
|
||||
const point = CRASH_POINTS[pointName];
|
||||
if (!point) throw new Error(`unknown crash point ${pointName}`);
|
||||
let triggered = false;
|
||||
const crash = () => {
|
||||
if (triggered) return;
|
||||
triggered = true;
|
||||
writeCrashMarker(markerPath, pointName);
|
||||
process.kill(process.pid, 'SIGKILL');
|
||||
throw new Error(`SIGKILL did not terminate ${pointName}`);
|
||||
};
|
||||
const matches = (timing, sql) =>
|
||||
!triggered && point.timing === timing && sql.trim().includes(point.sql);
|
||||
return new Proxy(database, {
|
||||
get(target, property) {
|
||||
if (property === 'exec') {
|
||||
return (sql) => {
|
||||
if (matches('beforeExec', sql)) crash();
|
||||
const result = target.exec(sql);
|
||||
if (matches('afterExec', sql)) crash();
|
||||
return result;
|
||||
};
|
||||
}
|
||||
if (property === 'prepare') {
|
||||
return (sql) => {
|
||||
const statement = target.prepare(sql);
|
||||
return new Proxy(statement, {
|
||||
get(statementTarget, statementProperty) {
|
||||
const value = Reflect.get(
|
||||
statementTarget,
|
||||
statementProperty,
|
||||
statementTarget,
|
||||
);
|
||||
if (statementProperty === 'run') {
|
||||
return (...values) => {
|
||||
const result = value.apply(statementTarget, values);
|
||||
if (matches('afterRun', sql)) crash();
|
||||
return result;
|
||||
};
|
||||
}
|
||||
return typeof value === 'function'
|
||||
? value.bind(statementTarget)
|
||||
: value;
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
const value = Reflect.get(target, property, target);
|
||||
return typeof value === 'function' ? value.bind(target) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function command(run, attempt, pointName, profile) {
|
||||
const identity = createHash('sha256')
|
||||
.update(profile)
|
||||
.update('\0')
|
||||
.update(pointName)
|
||||
.digest('hex')
|
||||
.slice(0, 16);
|
||||
return {
|
||||
run,
|
||||
attempt,
|
||||
reason: 'user',
|
||||
terminalStatus: 'cancelled',
|
||||
errorCode: 'EXECUTION_CANCELLED',
|
||||
errorSummary: 'Execution was cancelled',
|
||||
finishedAtMs: Math.max(
|
||||
Date.now(),
|
||||
run.updatedAtMs ?? 0,
|
||||
attempt.startedAtMs ?? 0,
|
||||
),
|
||||
attemptEventId: `wfc-a-${identity}`,
|
||||
stepMutationId: `wfc-s-${identity}`,
|
||||
};
|
||||
}
|
||||
|
||||
async function setupScenario({ databasePath, profile }) {
|
||||
await setupWorkflowAdmissionScenario({ databasePath, profile });
|
||||
const value = fixture(profile);
|
||||
const plan = executionPlan(value);
|
||||
const database = client(databasePath);
|
||||
const authority = new LocalSqliteOperationAuthority(database);
|
||||
try {
|
||||
await new LocalSqlitePluginPackageTaskReconciliationRepository(
|
||||
authority,
|
||||
value.registry,
|
||||
).reconcile(value.revision, {
|
||||
async findActiveResourceGeneration() {
|
||||
return value.revision.generation;
|
||||
},
|
||||
});
|
||||
await new LocalSqlitePluginPackageWorkflowAdmissionRepository(
|
||||
authority,
|
||||
).admit(plan);
|
||||
const collect = plan.steps.find(({ stepKey }) => stepKey === 'collect');
|
||||
if (!collect) throw new Error('Workflow collect Step is missing');
|
||||
const admitted =
|
||||
await new LocalSqlitePluginPackageWorkflowTaskAttemptAdmissionRepository(
|
||||
authority,
|
||||
).admit(plan.runId, collect.stepRunId);
|
||||
const runs = new LocalSqliteRunRepository(database);
|
||||
const execution = new LocalSqliteWorkflowTaskExecutionRepository(authority);
|
||||
const callbackTokenHash = 'c'.repeat(64);
|
||||
const startingAtMs = admitted.receipt.admittedAtMs + 1;
|
||||
const runningAtMs = startingAtMs + 1;
|
||||
const prepared = await execution.prepare({
|
||||
runId: plan.runId,
|
||||
attemptId: admitted.receipt.attemptId,
|
||||
stepRunId: collect.stepRunId,
|
||||
callbackTokenHash,
|
||||
deadlineAtMs: startingAtMs + 60_000,
|
||||
logArtifactId: 'local-0123456789abcdef0123456789abcd',
|
||||
atMs: startingAtMs,
|
||||
eventId: `wfc-start-${profile}`,
|
||||
});
|
||||
if (prepared.status !== 'applied') {
|
||||
throw new Error('Workflow Task did not enter starting');
|
||||
}
|
||||
const run = await runs.findRunById(plan.runId);
|
||||
const attempt = await runs.findAttemptById(admitted.receipt.attemptId);
|
||||
if (!run || !attempt) {
|
||||
throw new Error('Workflow Task starting authority is missing');
|
||||
}
|
||||
const running = await execution.recordRunning({
|
||||
run,
|
||||
attempt,
|
||||
callbackTokenHash,
|
||||
executorHandle: `qlp:v1:workflow-control-${profile}`,
|
||||
pid: 321,
|
||||
startedAtMs: runningAtMs,
|
||||
attemptEventId: `wfc-running-a-${profile}`,
|
||||
stepMutationId: `wfc-running-s-${profile}`,
|
||||
});
|
||||
if (running.status !== 'applied') {
|
||||
throw new Error('Workflow Task did not enter running');
|
||||
}
|
||||
const cancellation = database
|
||||
.prepare(
|
||||
`UPDATE "Runs"
|
||||
SET cancel_requested_at_ms = ?,
|
||||
cancel_reason = 'user'
|
||||
WHERE id = ? AND status = 'running'
|
||||
AND cancel_requested_at_ms IS NULL`,
|
||||
)
|
||||
.run(runningAtMs + 1, plan.runId);
|
||||
if (cancellation.changes !== 1) {
|
||||
throw new Error('Workflow cancellation intent was not recorded');
|
||||
}
|
||||
return Object.freeze({
|
||||
runId: plan.runId,
|
||||
attemptId: admitted.receipt.attemptId,
|
||||
stepRunId: collect.stepRunId,
|
||||
});
|
||||
} finally {
|
||||
await authority.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function runCrashScenario({
|
||||
databasePath,
|
||||
markerPath,
|
||||
pointName,
|
||||
profile,
|
||||
}) {
|
||||
const database = client(databasePath);
|
||||
const runs = new LocalSqliteRunRepository(database);
|
||||
const value = fixture(profile);
|
||||
const plan = executionPlan(value);
|
||||
const run = await runs.findRunById(plan.runId);
|
||||
const attempt = await runs.findLatestAttemptByRunId(plan.runId);
|
||||
if (!run || !attempt) {
|
||||
throw new Error('Workflow Task control authority is missing');
|
||||
}
|
||||
const authority = new LocalSqliteOperationAuthority(
|
||||
crashClient(database, pointName, markerPath),
|
||||
);
|
||||
await new LocalSqliteWorkflowTaskExecutionRepository(
|
||||
authority,
|
||||
).recordControlTerminal(command(run, attempt, pointName, profile));
|
||||
throw new Error(`crash point ${pointName} was not reached`);
|
||||
}
|
||||
|
||||
async function verifyScenario({ databasePath, pointName, profile }) {
|
||||
const point = CRASH_POINTS[pointName];
|
||||
if (!point) throw new Error(`unknown crash point ${pointName}`);
|
||||
const database = client(databasePath);
|
||||
const authority = new LocalSqliteOperationAuthority(database);
|
||||
try {
|
||||
const value = fixture(profile);
|
||||
const plan = executionPlan(value);
|
||||
const runs = new LocalSqliteRunRepository(database);
|
||||
const execution = new LocalSqliteWorkflowTaskExecutionRepository(authority);
|
||||
let run = await runs.findRunById(plan.runId);
|
||||
let attempt = await runs.findLatestAttemptByRunId(plan.runId);
|
||||
if (!run || !attempt) {
|
||||
throw new Error('Workflow Task recovery authority is missing');
|
||||
}
|
||||
if (point.durable !== (attempt.status === 'cancelled')) {
|
||||
throw new Error(`${profile}/${pointName} durability is inconsistent`);
|
||||
}
|
||||
const recovered = await execution.recordControlTerminal(
|
||||
command(run, attempt, pointName, profile),
|
||||
);
|
||||
if (recovered !== (point.durable ? 'already_terminal' : 'terminal')) {
|
||||
throw new Error(`${profile}/${pointName} replay is inconsistent`);
|
||||
}
|
||||
const cancellation =
|
||||
new LocalSqlitePluginPackageWorkflowCancellationConvergenceRepository(
|
||||
authority,
|
||||
);
|
||||
const converged = await cancellation.convergePage({ limit: 8 });
|
||||
if (
|
||||
converged.settledRuns !== 1 ||
|
||||
converged.settledAttempts !== 0 ||
|
||||
converged.blocked !== 0
|
||||
) {
|
||||
throw new Error(`${profile}/${pointName} parent did not converge`);
|
||||
}
|
||||
run = await runs.findRunById(plan.runId);
|
||||
attempt = await runs.findLatestAttemptByRunId(plan.runId);
|
||||
const facts = database
|
||||
.prepare(
|
||||
`SELECT
|
||||
(SELECT COUNT(*) FROM "RunAttempts"
|
||||
WHERE run_id = ?) AS attempts,
|
||||
(SELECT COUNT(*) FROM "RunEvents"
|
||||
WHERE run_id = ? AND type =
|
||||
'workflow.task_attempt.cancelled') AS attemptEvents,
|
||||
(SELECT COUNT(*) FROM "RunEvents"
|
||||
WHERE run_id = ? AND type = 'step.cancelled') AS stepEvents,
|
||||
(SELECT COUNT(*) FROM "RunEvents"
|
||||
WHERE run_id = ? AND type =
|
||||
'workflow.cancelled') AS workflowEvents,
|
||||
(SELECT COUNT(*) FROM "StepRuns"
|
||||
WHERE run_id = ? AND status = 'cancelled') AS cancelledSteps`,
|
||||
)
|
||||
.get(plan.runId, plan.runId, plan.runId, plan.runId, plan.runId);
|
||||
if (
|
||||
run?.status !== 'cancelled' ||
|
||||
run.version !== run.eventSequence ||
|
||||
attempt?.status !== 'cancelled' ||
|
||||
facts.attempts !== 1 ||
|
||||
facts.attemptEvents !== 1 ||
|
||||
facts.stepEvents !== 2 ||
|
||||
facts.workflowEvents !== 1 ||
|
||||
facts.cancelledSteps !== 2
|
||||
) {
|
||||
throw new Error(`${profile}/${pointName} terminal facts are incomplete`);
|
||||
}
|
||||
const replay = await execution.recordControlTerminal(
|
||||
command(run, attempt, pointName, profile),
|
||||
);
|
||||
if (replay !== 'already_terminal') {
|
||||
throw new Error(`${profile}/${pointName} terminal replay drifted`);
|
||||
}
|
||||
const empty = await cancellation.convergePage({ limit: 8 });
|
||||
if (
|
||||
empty.scanned !== 0 ||
|
||||
empty.settledRuns !== 0 ||
|
||||
empty.settledAttempts !== 0
|
||||
) {
|
||||
throw new Error(`${profile}/${pointName} cancellation replay drifted`);
|
||||
}
|
||||
await auditLocalSqliteReadiness(database);
|
||||
const integrity = database.prepare('PRAGMA integrity_check').get();
|
||||
const foreignKey = database
|
||||
.prepare('SELECT * FROM pragma_foreign_key_check LIMIT 1')
|
||||
.get();
|
||||
const journal = database.prepare('PRAGMA journal_mode').get();
|
||||
return Object.freeze({
|
||||
profile,
|
||||
pointName,
|
||||
crashAfterConclusiveStop: true,
|
||||
crashBeforeCommit: !point.durable,
|
||||
durableAfterCrash: point.durable,
|
||||
exactTerminalReplay: replay === 'already_terminal',
|
||||
parentConverged: run.status === 'cancelled',
|
||||
integrityCheck: Object.values(integrity)[0],
|
||||
foreignKeyCheck: foreignKey === undefined ? 'ok' : 'failed',
|
||||
journalMode: journal.journal_mode,
|
||||
});
|
||||
} finally {
|
||||
await authority.close();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
CRASH_POINTS,
|
||||
setupScenario,
|
||||
verifyScenario,
|
||||
};
|
||||
|
||||
if (require.main === module) {
|
||||
const [, , action, databasePath, markerPath, pointName, profile] =
|
||||
process.argv;
|
||||
if (action !== 'crash') {
|
||||
throw new Error('fixture action must be crash');
|
||||
}
|
||||
runCrashScenario({
|
||||
databasePath,
|
||||
markerPath,
|
||||
pointName,
|
||||
profile,
|
||||
}).catch((error) => {
|
||||
process.stderr.write(`${error.stack ?? error}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user