mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-22 10:32:40 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
+667
@@ -0,0 +1,667 @@
|
||||
const fs = require('node:fs');
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
|
||||
const {
|
||||
createApprovalRequest,
|
||||
} = require('@qinglong/runtime-core/approved-action');
|
||||
const {
|
||||
createPluginPackageLifecycleEvent,
|
||||
pluginPackageLifecycleActionDigest,
|
||||
} = require('@qinglong/runtime-core/plugin-package-lifecycle');
|
||||
const {
|
||||
createProjectToolDefinitionSnapshot,
|
||||
projectToolDefinitionSnapshotContribution,
|
||||
} = require('@qinglong/runtime-core/project-tool-definition-snapshot');
|
||||
const {
|
||||
activateInstall,
|
||||
pluginPackageTaskReconciliationFixture,
|
||||
} = require('../../../../test/contracts/pluginPackageTaskReconciliationRepositoryContract.cjs');
|
||||
const {
|
||||
LocalSqliteApprovalRequestRepository,
|
||||
} = require('../../dist/approved-action/approvalRequestRepository');
|
||||
const {
|
||||
LocalSqliteOperationAuthority,
|
||||
} = require('../../dist/authority/operationAuthority');
|
||||
const {
|
||||
LocalSqlitePluginPackageInstallRepository,
|
||||
} = require('../../dist/plugin-package/pluginPackageInstallRepository');
|
||||
const {
|
||||
EDGE_PLUGIN_PACKAGE_LIFECYCLE_ACTIVE_SOURCE_LIMIT,
|
||||
STANDALONE_PLUGIN_PACKAGE_LIFECYCLE_ACTIVE_SOURCE_LIMIT,
|
||||
LocalSqlitePluginPackageLifecycleRepository,
|
||||
} = require('../../dist/plugin-package/pluginPackageLifecycleRepository');
|
||||
const {
|
||||
LocalSqlitePluginPackageMaterializedRevisionRepository,
|
||||
} = require('../../dist/plugin-package/pluginPackageMaterializedRevisionRepository');
|
||||
const {
|
||||
LocalSqlitePluginPackageTaskReconciliationRepository,
|
||||
} = require('../../dist/plugin-package/pluginPackageTaskReconciliationRepository');
|
||||
const {
|
||||
LocalSqliteProjectToolDefinitionSnapshotRepository,
|
||||
} = require('../../dist/tool-execution/projectToolDefinitionSnapshotRepository');
|
||||
const { migrateLocalSqlitePath } = require('../../dist/migration/migration');
|
||||
const { auditLocalSqliteReadiness } = require('../../dist/readiness/readiness');
|
||||
|
||||
const OWNER = Object.freeze({ type: 'user', id: 'owner-001' });
|
||||
const SYSTEM = Object.freeze({ type: 'system', id: 'lifecycle-dispatcher' });
|
||||
const FENCE = Object.freeze({ projectVersion: 1, bindingVersion: 1 });
|
||||
|
||||
const CRASH_POINTS = Object.freeze({
|
||||
after_task_revision: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "QingLong3TaskDefinitionRevisions"',
|
||||
durable: false,
|
||||
}),
|
||||
after_tool_snapshot: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "QingLong3ProjectToolDefinitionSnapshots"',
|
||||
durable: false,
|
||||
}),
|
||||
after_lifecycle_event: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "QingLong3PluginPackageLifecycleEvents"',
|
||||
durable: false,
|
||||
}),
|
||||
after_lifecycle_receipt: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "QingLong3PluginPackageLifecycleReceipts"',
|
||||
durable: false,
|
||||
}),
|
||||
after_lifecycle_task: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "QingLong3PluginPackageLifecycleTasks"',
|
||||
durable: false,
|
||||
}),
|
||||
after_lifecycle_head: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "QingLong3PluginPackageLifecycleHeads"',
|
||||
durable: false,
|
||||
}),
|
||||
before_commit: Object.freeze({
|
||||
timing: 'beforeExec',
|
||||
sql: 'COMMIT',
|
||||
durable: false,
|
||||
}),
|
||||
after_commit: Object.freeze({
|
||||
timing: 'afterExec',
|
||||
sql: 'COMMIT',
|
||||
durable: true,
|
||||
}),
|
||||
});
|
||||
|
||||
function fixture(profile) {
|
||||
return pluginPackageTaskReconciliationFixture(
|
||||
`lifecycle-crash-${profile}`,
|
||||
{ profile },
|
||||
);
|
||||
}
|
||||
|
||||
function client(databasePath) {
|
||||
const database = new DatabaseSync(databasePath);
|
||||
database.exec('PRAGMA foreign_keys = ON');
|
||||
return database;
|
||||
}
|
||||
|
||||
function activeSourceLimit(profile) {
|
||||
return profile === 'edge'
|
||||
? EDGE_PLUGIN_PACKAGE_LIFECYCLE_ACTIVE_SOURCE_LIMIT
|
||||
: STANDALONE_PLUGIN_PACKAGE_LIFECYCLE_ACTIVE_SOURCE_LIMIT;
|
||||
}
|
||||
|
||||
function repositories(authority, value, profile) {
|
||||
return {
|
||||
approval: new LocalSqliteApprovalRequestRepository(authority),
|
||||
install: new LocalSqlitePluginPackageInstallRepository(authority),
|
||||
lifecycle: new LocalSqlitePluginPackageLifecycleRepository(authority, {
|
||||
registry: value.registry,
|
||||
activeSourceLimit: activeSourceLimit(profile),
|
||||
}),
|
||||
materialized:
|
||||
new LocalSqlitePluginPackageMaterializedRevisionRepository(
|
||||
authority,
|
||||
value.registry,
|
||||
),
|
||||
reconciliation:
|
||||
new LocalSqlitePluginPackageTaskReconciliationRepository(
|
||||
authority,
|
||||
value.registry,
|
||||
),
|
||||
snapshots: new LocalSqliteProjectToolDefinitionSnapshotRepository(
|
||||
authority,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function audit(
|
||||
eventId,
|
||||
requestId,
|
||||
operationId,
|
||||
subject,
|
||||
authenticationId,
|
||||
outcome,
|
||||
projectId,
|
||||
occurredAtMs,
|
||||
) {
|
||||
return {
|
||||
eventId,
|
||||
requestId,
|
||||
operationId,
|
||||
projectId,
|
||||
subject,
|
||||
authenticationId,
|
||||
outcome,
|
||||
reasons: [outcome === 'approval_required' ? 'package_review' : 'role_grant'],
|
||||
fence: FENCE,
|
||||
occurredAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
function auditId(sequence, offset) {
|
||||
return `91000000-0000-4000-8000-${String(sequence * 10 + offset).padStart(
|
||||
12,
|
||||
'0',
|
||||
)}`;
|
||||
}
|
||||
|
||||
function approvalSequence(action) {
|
||||
return action === 'enable' ? 2 : 1;
|
||||
}
|
||||
|
||||
async function approveLifecycleImpact(
|
||||
approval,
|
||||
value,
|
||||
impact,
|
||||
sequence,
|
||||
) {
|
||||
const requestId = `lifecycle-crash-approval-${sequence}`;
|
||||
const dispatchId = `lifecycle-crash-dispatch-${sequence}`;
|
||||
const requestedAtMs = 10_000 * sequence + 1;
|
||||
const decidedAtMs = requestedAtMs + 1;
|
||||
const consumedAtMs = requestedAtMs + 2;
|
||||
const expiresAtMs = requestedAtMs + 1_000;
|
||||
const action = {
|
||||
permission: 'package.manage',
|
||||
actionType: `plugin_package.lifecycle.${impact.action}`,
|
||||
actionRef: `lifecycle:${impact.impactDigest}`,
|
||||
actionDigest: pluginPackageLifecycleActionDigest(impact),
|
||||
previewDigest: impact.impactDigest,
|
||||
};
|
||||
await approval.create({
|
||||
request: createApprovalRequest({
|
||||
id: requestId,
|
||||
projectId: value.projectId,
|
||||
action,
|
||||
risk: 'high',
|
||||
decisionMode: 'human_confirmation',
|
||||
requestedBy: OWNER,
|
||||
requestedAtMs,
|
||||
expiresAtMs,
|
||||
requestFence: FENCE,
|
||||
}),
|
||||
audit: audit(
|
||||
auditId(sequence, 1),
|
||||
`lifecycle-crash-http-${sequence}`,
|
||||
'approval.request',
|
||||
OWNER,
|
||||
`auth-request-${sequence}`,
|
||||
'approval_required',
|
||||
value.projectId,
|
||||
requestedAtMs,
|
||||
),
|
||||
});
|
||||
await approval.decide({
|
||||
requestId,
|
||||
expectedVersion: 1,
|
||||
decisionId: `lifecycle-crash-decision-${sequence}`,
|
||||
decision: 'approved',
|
||||
reasonCode: 'reviewed',
|
||||
principal: {
|
||||
subject: OWNER,
|
||||
authenticationId: `auth-approve-${sequence}`,
|
||||
authenticatedAtMs: decidedAtMs - 1,
|
||||
expiresAtMs,
|
||||
assurance: 'local_console',
|
||||
},
|
||||
decidedAtMs,
|
||||
authorizationFence: FENCE,
|
||||
audit: audit(
|
||||
auditId(sequence, 2),
|
||||
`lifecycle-crash-http-${sequence}`,
|
||||
'approval.decide',
|
||||
OWNER,
|
||||
`auth-approve-${sequence}`,
|
||||
'allowed',
|
||||
value.projectId,
|
||||
decidedAtMs,
|
||||
),
|
||||
});
|
||||
return approval.consume({
|
||||
requestId,
|
||||
expectedVersion: 2,
|
||||
consumptionId: `lifecycle-crash-consume-${sequence}`,
|
||||
dispatchId,
|
||||
action,
|
||||
requestedBy: OWNER,
|
||||
consumedBy: SYSTEM,
|
||||
consumedAtMs,
|
||||
authorizationFence: FENCE,
|
||||
audit: audit(
|
||||
auditId(sequence, 3),
|
||||
`lifecycle-crash-dispatch-cycle-${sequence}`,
|
||||
'approval.consume',
|
||||
SYSTEM,
|
||||
`auth-dispatch-${sequence}`,
|
||||
'allowed',
|
||||
value.projectId,
|
||||
consumedAtMs,
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
function lifecycleEvent(impact, action) {
|
||||
const sequence = approvalSequence(action);
|
||||
return createPluginPackageLifecycleEvent({
|
||||
dispatchId: `lifecycle-crash-dispatch-${sequence}`,
|
||||
impact,
|
||||
requestedBy: OWNER,
|
||||
approvedBy: OWNER,
|
||||
authorizationMode: 'human_confirmation',
|
||||
occurredAtMs: 10_000 * sequence + 4,
|
||||
});
|
||||
}
|
||||
|
||||
async function publishActivePackage(repositoriesValue, value) {
|
||||
await activateInstall(repositoriesValue.install, value);
|
||||
await repositoriesValue.materialized.publish(value.revision);
|
||||
await repositoriesValue.reconciliation.reconcile(value.revision, {
|
||||
async findActiveResourceGeneration() {
|
||||
return value.revision.generation;
|
||||
},
|
||||
});
|
||||
await repositoriesValue.snapshots.publish(
|
||||
createProjectToolDefinitionSnapshot({
|
||||
projectId: value.projectId,
|
||||
contributions: [
|
||||
projectToolDefinitionSnapshotContribution(
|
||||
value.revision,
|
||||
value.registry,
|
||||
),
|
||||
],
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function setupScenario({ action, databasePath, profile }) {
|
||||
await migrateLocalSqlitePath({ databasePath, profile });
|
||||
const value = fixture(profile);
|
||||
const database = client(databasePath);
|
||||
database
|
||||
.prepare(
|
||||
`INSERT INTO "QingLong3Projects"
|
||||
(id, name, slug, status, version, created_at_ms, updated_at_ms)
|
||||
VALUES (?, ?, ?, 'active', 1, 1, 1)`,
|
||||
)
|
||||
.run(value.projectId, value.projectId, value.projectId);
|
||||
database
|
||||
.prepare(
|
||||
`INSERT INTO "QingLong3ProjectRoleBindings" (
|
||||
project_id, subject_type, subject_id, version, state, role,
|
||||
mutation_id, changed_by_type, changed_by_id, created_at_ms
|
||||
) VALUES (?, 'user', ?, 1, 'active', 'owner', ?, 'user', ?, 1)`,
|
||||
)
|
||||
.run(
|
||||
value.projectId,
|
||||
OWNER.id,
|
||||
`grant-lifecycle-crash-${profile}`,
|
||||
OWNER.id,
|
||||
);
|
||||
const authority = new LocalSqliteOperationAuthority(database);
|
||||
try {
|
||||
const repository = repositories(authority, value, profile);
|
||||
await publishActivePackage(repository, value);
|
||||
if (action === 'enable') {
|
||||
const disableImpact = await repository.lifecycle.plan(
|
||||
'disable',
|
||||
value.projectId,
|
||||
value.packageName,
|
||||
);
|
||||
await approveLifecycleImpact(
|
||||
repository.approval,
|
||||
value,
|
||||
disableImpact,
|
||||
1,
|
||||
);
|
||||
await repository.lifecycle.transition(
|
||||
lifecycleEvent(disableImpact, 'disable'),
|
||||
() => {},
|
||||
);
|
||||
}
|
||||
const impact = await repository.lifecycle.plan(
|
||||
action,
|
||||
value.projectId,
|
||||
value.packageName,
|
||||
);
|
||||
await approveLifecycleImpact(
|
||||
repository.approval,
|
||||
value,
|
||||
impact,
|
||||
approvalSequence(action),
|
||||
);
|
||||
return lifecycleEvent(impact, action);
|
||||
} finally {
|
||||
await authority.close();
|
||||
}
|
||||
}
|
||||
|
||||
function writeCrashMarker(markerPath, pointName, action) {
|
||||
const descriptor = fs.openSync(markerPath, 'wx', 0o600);
|
||||
try {
|
||||
fs.writeSync(
|
||||
descriptor,
|
||||
JSON.stringify({
|
||||
schema: 'qinglong/sqlite-plugin-package-lifecycle-crash-marker@v1',
|
||||
action,
|
||||
point: pointName,
|
||||
pid: process.pid,
|
||||
}),
|
||||
);
|
||||
fs.fsyncSync(descriptor);
|
||||
} finally {
|
||||
fs.closeSync(descriptor);
|
||||
}
|
||||
}
|
||||
|
||||
function crashClient(database, pointName, markerPath, action) {
|
||||
const point = CRASH_POINTS[pointName];
|
||||
if (!point) throw new Error(`unknown crash point ${pointName}`);
|
||||
let triggered = false;
|
||||
const crash = () => {
|
||||
if (triggered) return;
|
||||
triggered = true;
|
||||
writeCrashMarker(markerPath, pointName, action);
|
||||
process.kill(process.pid, 'SIGKILL');
|
||||
throw new Error(`SIGKILL did not terminate ${action}/${pointName}`);
|
||||
};
|
||||
const matches = (timing, sql) =>
|
||||
!triggered && point.timing === timing && sql.trim().includes(point.sql);
|
||||
return new Proxy(database, {
|
||||
get(target, property) {
|
||||
if (property === 'exec') {
|
||||
return (sql) => {
|
||||
if (matches('beforeExec', sql)) crash();
|
||||
const result = target.exec(sql);
|
||||
if (matches('afterExec', sql)) crash();
|
||||
return result;
|
||||
};
|
||||
}
|
||||
if (property === 'prepare') {
|
||||
return (sql) => {
|
||||
const statement = target.prepare(sql);
|
||||
return new Proxy(statement, {
|
||||
get(statementTarget, statementProperty) {
|
||||
const value = Reflect.get(
|
||||
statementTarget,
|
||||
statementProperty,
|
||||
statementTarget,
|
||||
);
|
||||
if (statementProperty === 'run') {
|
||||
return (...values) => {
|
||||
const result = value.apply(statementTarget, values);
|
||||
if (matches('afterRun', sql)) crash();
|
||||
return result;
|
||||
};
|
||||
}
|
||||
return typeof value === 'function'
|
||||
? value.bind(statementTarget)
|
||||
: value;
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
const value = Reflect.get(target, property, target);
|
||||
return typeof value === 'function' ? value.bind(target) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function runCrashScenario({
|
||||
action,
|
||||
databasePath,
|
||||
markerPath,
|
||||
pointName,
|
||||
profile,
|
||||
}) {
|
||||
const value = fixture(profile);
|
||||
const planningAuthority = new LocalSqliteOperationAuthority(
|
||||
client(databasePath),
|
||||
);
|
||||
const impact = await repositories(
|
||||
planningAuthority,
|
||||
value,
|
||||
profile,
|
||||
).lifecycle.plan(action, value.projectId, value.packageName);
|
||||
await planningAuthority.close();
|
||||
const database = client(databasePath);
|
||||
const authority = new LocalSqliteOperationAuthority(
|
||||
crashClient(database, pointName, markerPath, action),
|
||||
);
|
||||
const repository = repositories(authority, value, profile);
|
||||
await repository.lifecycle.transition(lifecycleEvent(impact, action), () => {});
|
||||
throw new Error(`crash point ${action}/${pointName} was not reached`);
|
||||
}
|
||||
|
||||
function eventFacts(database, eventDigest) {
|
||||
return {
|
||||
events: database
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS count
|
||||
FROM "QingLong3PluginPackageLifecycleEvents"
|
||||
WHERE event_digest = ?`,
|
||||
)
|
||||
.get(eventDigest).count,
|
||||
receipts: database
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS count
|
||||
FROM "QingLong3PluginPackageLifecycleReceipts"
|
||||
WHERE event_digest = ?`,
|
||||
)
|
||||
.get(eventDigest).count,
|
||||
tasks: database
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS count
|
||||
FROM "QingLong3PluginPackageLifecycleTasks"
|
||||
WHERE event_digest = ?`,
|
||||
)
|
||||
.get(eventDigest).count,
|
||||
heads: database
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS count
|
||||
FROM "QingLong3PluginPackageLifecycleHeads"
|
||||
WHERE event_digest = ?`,
|
||||
)
|
||||
.get(eventDigest).count,
|
||||
};
|
||||
}
|
||||
|
||||
function taskFacts(database, projectId) {
|
||||
return database
|
||||
.prepare(
|
||||
`SELECT revision.enabled,
|
||||
head.current_revision AS "currentRevision"
|
||||
FROM "QingLong3TaskDefinitions" AS head
|
||||
JOIN "QingLong3TaskDefinitionRevisions" AS revision
|
||||
ON revision.project_id = head.project_id
|
||||
AND revision.task_id = head.task_id
|
||||
AND revision.revision = head.current_revision
|
||||
WHERE head.project_id = ?
|
||||
ORDER BY head.task_id`,
|
||||
)
|
||||
.all(projectId);
|
||||
}
|
||||
|
||||
function assertTaskState(facts, revision, enabled, label) {
|
||||
if (
|
||||
facts.length !== 2 ||
|
||||
facts.some(
|
||||
(fact) =>
|
||||
fact.enabled !== enabled || fact.currentRevision !== revision,
|
||||
)
|
||||
) {
|
||||
throw new Error(`${label} Task state is incomplete`);
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyScenario({
|
||||
action,
|
||||
databasePath,
|
||||
event,
|
||||
pointName,
|
||||
profile,
|
||||
}) {
|
||||
const point = CRASH_POINTS[pointName];
|
||||
const value = fixture(profile);
|
||||
const database = client(databasePath);
|
||||
const authority = new LocalSqliteOperationAuthority(database);
|
||||
try {
|
||||
const repository = repositories(authority, value, profile);
|
||||
const beforeRecovery = await repository.lifecycle.findByEventDigest(
|
||||
event.eventDigest,
|
||||
);
|
||||
if (point.durable !== (beforeRecovery !== null)) {
|
||||
throw new Error(
|
||||
`${profile}/${action}/${pointName} durability is inconsistent`,
|
||||
);
|
||||
}
|
||||
const beforeEventFacts = eventFacts(database, event.eventDigest);
|
||||
const expectedBeforeFacts = point.durable
|
||||
? { events: 1, receipts: 1, tasks: 2, heads: 1 }
|
||||
: { events: 0, receipts: 0, tasks: 0, heads: 0 };
|
||||
if (JSON.stringify(beforeEventFacts) !== JSON.stringify(expectedBeforeFacts)) {
|
||||
throw new Error(
|
||||
`${profile}/${action}/${pointName} left partial lifecycle facts`,
|
||||
);
|
||||
}
|
||||
const beforeRevision = action === 'disable' ? 1 : 2;
|
||||
const beforeEnabled = action === 'disable' ? 1 : 0;
|
||||
const finalRevision = action === 'disable' ? 2 : 3;
|
||||
const finalEnabled = action === 'disable' ? 0 : 1;
|
||||
assertTaskState(
|
||||
taskFacts(database, value.projectId),
|
||||
point.durable ? finalRevision : beforeRevision,
|
||||
point.durable ? finalEnabled : beforeEnabled,
|
||||
`${profile}/${action}/${pointName} pre-recovery`,
|
||||
);
|
||||
const beforeSnapshot = await repository.snapshots.findCurrent(
|
||||
value.projectId,
|
||||
);
|
||||
const expectedBeforeSourceCount = point.durable
|
||||
? action === 'enable'
|
||||
? 1
|
||||
: 0
|
||||
: action === 'enable'
|
||||
? 0
|
||||
: 1;
|
||||
if (beforeSnapshot.snapshot.sources.length !== expectedBeforeSourceCount) {
|
||||
throw new Error(
|
||||
`${profile}/${action}/${pointName} Tool snapshot is partial`,
|
||||
);
|
||||
}
|
||||
const recovered = await repository.lifecycle.transition(event, () => {});
|
||||
if (recovered.status !== (point.durable ? 'existing' : 'created')) {
|
||||
throw new Error(
|
||||
`${profile}/${action}/${pointName} replay status is inconsistent`,
|
||||
);
|
||||
}
|
||||
assertTaskState(
|
||||
taskFacts(database, value.projectId),
|
||||
finalRevision,
|
||||
finalEnabled,
|
||||
`${profile}/${action}/${pointName} recovered`,
|
||||
);
|
||||
const recoveredSnapshot = await repository.snapshots.findCurrent(
|
||||
value.projectId,
|
||||
);
|
||||
const finalSourceCount = action === 'enable' ? 1 : 0;
|
||||
if (recoveredSnapshot.snapshot.sources.length !== finalSourceCount) {
|
||||
throw new Error(
|
||||
`${profile}/${action}/${pointName} recovered Tool snapshot is invalid`,
|
||||
);
|
||||
}
|
||||
const recoveredFacts = eventFacts(database, event.eventDigest);
|
||||
if (
|
||||
JSON.stringify(recoveredFacts) !==
|
||||
JSON.stringify({ events: 1, receipts: 1, tasks: 2, heads: 1 })
|
||||
) {
|
||||
throw new Error(
|
||||
`${profile}/${action}/${pointName} replay is not exactly once`,
|
||||
);
|
||||
}
|
||||
const head = await repository.lifecycle.findHead(
|
||||
value.projectId,
|
||||
value.packageName,
|
||||
);
|
||||
const expectedDisposition = action === 'enable' ? 'active' : 'disabled';
|
||||
if (
|
||||
!head ||
|
||||
head.disposition !== expectedDisposition ||
|
||||
head.eventDigest !== event.eventDigest
|
||||
) {
|
||||
throw new Error(
|
||||
`${profile}/${action}/${pointName} lifecycle head is invalid`,
|
||||
);
|
||||
}
|
||||
await auditLocalSqliteReadiness(database);
|
||||
const integrity = database.prepare('PRAGMA integrity_check').get();
|
||||
const foreignKey = database
|
||||
.prepare('SELECT * FROM pragma_foreign_key_check LIMIT 1')
|
||||
.get();
|
||||
const journal = database.prepare('PRAGMA journal_mode').get();
|
||||
const synchronous = database.prepare('PRAGMA synchronous').get();
|
||||
return Object.freeze({
|
||||
profile,
|
||||
action,
|
||||
pointName,
|
||||
crashBeforeCommit: !point.durable,
|
||||
durableAfterCrash: point.durable,
|
||||
exactReplay: true,
|
||||
integrityCheck: Object.values(integrity)[0],
|
||||
foreignKeyCheck: foreignKey === undefined ? 'ok' : 'failed',
|
||||
journalMode: journal.journal_mode,
|
||||
synchronous: synchronous.synchronous,
|
||||
});
|
||||
} finally {
|
||||
await authority.close();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
CRASH_POINTS,
|
||||
setupScenario,
|
||||
verifyScenario,
|
||||
};
|
||||
|
||||
if (require.main === module) {
|
||||
const [
|
||||
,
|
||||
,
|
||||
command,
|
||||
databasePath,
|
||||
markerPath,
|
||||
pointName,
|
||||
profile,
|
||||
action,
|
||||
] = process.argv;
|
||||
if (command !== 'crash') {
|
||||
throw new Error('fixture command must be crash');
|
||||
}
|
||||
runCrashScenario({
|
||||
action,
|
||||
databasePath,
|
||||
markerPath,
|
||||
pointName,
|
||||
profile,
|
||||
}).catch((error) => {
|
||||
process.stderr.write(`${error.stack ?? error}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
+303
@@ -0,0 +1,303 @@
|
||||
const fs = require('node:fs');
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
|
||||
const {
|
||||
createPluginPackageQuarantineEvent,
|
||||
} = require('@qinglong/runtime-core/plugin-package-quarantine');
|
||||
const {
|
||||
activateInstall,
|
||||
pluginPackageTaskReconciliationFixture,
|
||||
} = require('../../../../test/contracts/pluginPackageTaskReconciliationRepositoryContract.cjs');
|
||||
const {
|
||||
LocalSqliteOperationAuthority,
|
||||
} = require('../../dist/authority/operationAuthority');
|
||||
const {
|
||||
LocalSqlitePluginPackageInstallRepository,
|
||||
} = require('../../dist/plugin-package/pluginPackageInstallRepository');
|
||||
const {
|
||||
LocalSqlitePluginPackageMaterializedRevisionRepository,
|
||||
} = require('../../dist/plugin-package/pluginPackageMaterializedRevisionRepository');
|
||||
const {
|
||||
LocalSqlitePluginPackageQuarantineRepository,
|
||||
} = require('../../dist/plugin-package/pluginPackageQuarantineRepository');
|
||||
const {
|
||||
LocalSqlitePluginPackageTaskReconciliationRepository,
|
||||
} = require('../../dist/plugin-package/pluginPackageTaskReconciliationRepository');
|
||||
const { auditLocalSqliteReadiness } = require('../../dist/readiness/readiness');
|
||||
const { migrateLocalSqlitePath } = require('../../dist/migration/migration');
|
||||
|
||||
const DIGEST_D = 'd'.repeat(64);
|
||||
const DIGEST_E = 'e'.repeat(64);
|
||||
|
||||
const CRASH_POINTS = Object.freeze({
|
||||
after_task_disable: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "QingLong3TaskDefinitionRevisions"',
|
||||
durable: false,
|
||||
}),
|
||||
after_quarantine_event: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "QingLong3PluginPackageQuarantineEvents"',
|
||||
durable: false,
|
||||
}),
|
||||
after_withdrawal_receipt: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "QingLong3PluginPackageWithdrawalReceipts"',
|
||||
durable: false,
|
||||
}),
|
||||
before_commit: Object.freeze({
|
||||
timing: 'beforeExec',
|
||||
sql: 'COMMIT',
|
||||
durable: false,
|
||||
}),
|
||||
after_commit: Object.freeze({
|
||||
timing: 'afterExec',
|
||||
sql: 'COMMIT',
|
||||
durable: true,
|
||||
}),
|
||||
});
|
||||
|
||||
function fixture(profile) {
|
||||
return pluginPackageTaskReconciliationFixture(`quarantine-crash-${profile}`, {
|
||||
profile,
|
||||
});
|
||||
}
|
||||
|
||||
function event(value) {
|
||||
const record = value.install.active;
|
||||
return createPluginPackageQuarantineEvent({
|
||||
mutationId: `quarantine-crash-${value.profile}`,
|
||||
revocationReceiptDigest: DIGEST_D,
|
||||
impactDigest: DIGEST_E,
|
||||
target: {
|
||||
projectId: record.projectId,
|
||||
packageName: record.packageName,
|
||||
installationId: record.installationId,
|
||||
lockDigest: record.lockDigest,
|
||||
installState: record.state,
|
||||
installVersion: record.version,
|
||||
installRecordDigest: record.recordDigest,
|
||||
activeLockDigest: record.activeLockDigest,
|
||||
},
|
||||
proposer: { type: 'user', id: 'owner-a' },
|
||||
confirmer: { type: 'user', id: 'owner-b' },
|
||||
authorizationMode: 'dual_control',
|
||||
reasonCode: 'confirmed_key_compromise',
|
||||
occurredAtMs: record.updatedAtMs + 1,
|
||||
});
|
||||
}
|
||||
|
||||
function client(databasePath) {
|
||||
const database = new DatabaseSync(databasePath);
|
||||
database.exec('PRAGMA foreign_keys = ON');
|
||||
return database;
|
||||
}
|
||||
|
||||
async function setupScenario({ databasePath, profile }) {
|
||||
await migrateLocalSqlitePath({ databasePath, profile });
|
||||
const value = fixture(profile);
|
||||
const database = client(databasePath);
|
||||
database
|
||||
.prepare(
|
||||
`INSERT INTO "QingLong3Projects"
|
||||
(id, name, slug, status, version, created_at_ms, updated_at_ms)
|
||||
VALUES (?, ?, ?, 'active', 1, 1, 1)`,
|
||||
)
|
||||
.run(value.projectId, value.projectId, value.projectId);
|
||||
const authority = new LocalSqliteOperationAuthority(database);
|
||||
try {
|
||||
const install = new LocalSqlitePluginPackageInstallRepository(authority);
|
||||
const materialized =
|
||||
new LocalSqlitePluginPackageMaterializedRevisionRepository(
|
||||
authority,
|
||||
value.registry,
|
||||
);
|
||||
const reconciliation =
|
||||
new LocalSqlitePluginPackageTaskReconciliationRepository(
|
||||
authority,
|
||||
value.registry,
|
||||
);
|
||||
await activateInstall(install, value);
|
||||
await materialized.publish(value.revision);
|
||||
await reconciliation.reconcile(value.revision, {
|
||||
async findActiveResourceGeneration() {
|
||||
return value.revision.generation;
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
await authority.close();
|
||||
}
|
||||
}
|
||||
|
||||
function writeCrashMarker(markerPath, pointName) {
|
||||
const descriptor = fs.openSync(markerPath, 'wx', 0o600);
|
||||
try {
|
||||
fs.writeSync(
|
||||
descriptor,
|
||||
JSON.stringify({
|
||||
schema: 'qinglong/sqlite-plugin-package-quarantine-crash-marker@v1',
|
||||
point: pointName,
|
||||
pid: process.pid,
|
||||
}),
|
||||
);
|
||||
fs.fsyncSync(descriptor);
|
||||
} finally {
|
||||
fs.closeSync(descriptor);
|
||||
}
|
||||
}
|
||||
|
||||
function crashClient(database, pointName, markerPath) {
|
||||
const point = CRASH_POINTS[pointName];
|
||||
if (!point) throw new Error(`unknown crash point ${pointName}`);
|
||||
let triggered = false;
|
||||
const crash = () => {
|
||||
if (triggered) return;
|
||||
triggered = true;
|
||||
writeCrashMarker(markerPath, pointName);
|
||||
process.kill(process.pid, 'SIGKILL');
|
||||
throw new Error(`SIGKILL did not terminate ${pointName}`);
|
||||
};
|
||||
const matches = (timing, sql) =>
|
||||
!triggered && point.timing === timing && sql.trim().includes(point.sql);
|
||||
return new Proxy(database, {
|
||||
get(target, property) {
|
||||
if (property === 'exec') {
|
||||
return (sql) => {
|
||||
if (matches('beforeExec', sql)) crash();
|
||||
const result = target.exec(sql);
|
||||
if (matches('afterExec', sql)) crash();
|
||||
return result;
|
||||
};
|
||||
}
|
||||
if (property === 'prepare') {
|
||||
return (sql) => {
|
||||
const statement = target.prepare(sql);
|
||||
return new Proxy(statement, {
|
||||
get(statementTarget, statementProperty) {
|
||||
const value = Reflect.get(
|
||||
statementTarget,
|
||||
statementProperty,
|
||||
statementTarget,
|
||||
);
|
||||
if (statementProperty === 'run') {
|
||||
return (...values) => {
|
||||
const result = value.apply(statementTarget, values);
|
||||
if (matches('afterRun', sql)) crash();
|
||||
return result;
|
||||
};
|
||||
}
|
||||
return typeof value === 'function'
|
||||
? value.bind(statementTarget)
|
||||
: value;
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
const value = Reflect.get(target, property, target);
|
||||
return typeof value === 'function' ? value.bind(target) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function runCrashScenario({
|
||||
databasePath,
|
||||
markerPath,
|
||||
pointName,
|
||||
profile,
|
||||
}) {
|
||||
const database = client(databasePath);
|
||||
const authority = new LocalSqliteOperationAuthority(
|
||||
crashClient(database, pointName, markerPath),
|
||||
);
|
||||
const value = fixture(profile);
|
||||
await new LocalSqlitePluginPackageQuarantineRepository(authority, {
|
||||
registry: value.registry,
|
||||
activeSourceLimit: profile === 'edge' ? 4 : 16,
|
||||
}).quarantine(event(value), () => {});
|
||||
throw new Error(`crash point ${pointName} was not reached`);
|
||||
}
|
||||
|
||||
async function verifyScenario({ databasePath, pointName, profile }) {
|
||||
const point = CRASH_POINTS[pointName];
|
||||
const value = fixture(profile);
|
||||
const quarantineEvent = event(value);
|
||||
const database = client(databasePath);
|
||||
const authority = new LocalSqliteOperationAuthority(database);
|
||||
try {
|
||||
const repository = new LocalSqlitePluginPackageQuarantineRepository(
|
||||
authority,
|
||||
{
|
||||
registry: value.registry,
|
||||
activeSourceLimit: profile === 'edge' ? 4 : 16,
|
||||
},
|
||||
);
|
||||
const beforeRecovery = await repository.findByEventDigest(
|
||||
quarantineEvent.eventDigest,
|
||||
);
|
||||
if (point.durable !== (beforeRecovery !== null)) {
|
||||
throw new Error(`${profile}/${pointName} durability is inconsistent`);
|
||||
}
|
||||
const recovered = await repository.quarantine(quarantineEvent, () => {});
|
||||
if (recovered.status !== (point.durable ? 'existing' : 'created')) {
|
||||
throw new Error(`${profile}/${pointName} replay status is inconsistent`);
|
||||
}
|
||||
const taskFacts = database
|
||||
.prepare(
|
||||
`SELECT revision.enabled, head.current_revision AS "currentRevision"
|
||||
FROM "QingLong3TaskDefinitions" AS head
|
||||
JOIN "QingLong3TaskDefinitionRevisions" AS revision
|
||||
ON revision.project_id = head.project_id
|
||||
AND revision.task_id = head.task_id
|
||||
AND revision.revision = head.current_revision
|
||||
WHERE head.project_id = ?
|
||||
ORDER BY head.task_id`,
|
||||
)
|
||||
.all(value.projectId);
|
||||
if (
|
||||
taskFacts.length !== 2 ||
|
||||
taskFacts.some((fact) => fact.enabled !== 0 || fact.currentRevision !== 2)
|
||||
) {
|
||||
throw new Error(`${profile}/${pointName} Task withdrawal is incomplete`);
|
||||
}
|
||||
await auditLocalSqliteReadiness(database);
|
||||
const integrity = database.prepare('PRAGMA integrity_check').get();
|
||||
const foreignKey = database
|
||||
.prepare('SELECT * FROM pragma_foreign_key_check LIMIT 1')
|
||||
.get();
|
||||
const journal = database.prepare('PRAGMA journal_mode').get();
|
||||
return Object.freeze({
|
||||
profile,
|
||||
pointName,
|
||||
crashBeforeCommit: !point.durable,
|
||||
durableAfterCrash: point.durable,
|
||||
integrityCheck: Object.values(integrity)[0],
|
||||
foreignKeyCheck: foreignKey === undefined ? 'ok' : 'failed',
|
||||
journalMode: journal.journal_mode,
|
||||
});
|
||||
} finally {
|
||||
await authority.close();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
CRASH_POINTS,
|
||||
setupScenario,
|
||||
verifyScenario,
|
||||
};
|
||||
|
||||
if (require.main === module) {
|
||||
const [, , action, databasePath, markerPath, pointName, profile] =
|
||||
process.argv;
|
||||
if (action !== 'crash') {
|
||||
throw new Error('fixture action must be crash');
|
||||
}
|
||||
runCrashScenario({
|
||||
databasePath,
|
||||
markerPath,
|
||||
pointName,
|
||||
profile,
|
||||
}).catch((error) => {
|
||||
process.stderr.write(`${error.stack ?? error}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
Vendored
+488
@@ -0,0 +1,488 @@
|
||||
const fs = require('node:fs');
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
const { performance } = require('node:perf_hooks');
|
||||
|
||||
const {
|
||||
createInitialPluginPackageAutomationPublication,
|
||||
} = require('@qinglong/runtime-core/plugin-package-automation-publication');
|
||||
const {
|
||||
createPluginPackageWorkflowExecutionPlan,
|
||||
} = require('@qinglong/runtime-core/plugin-package-workflow-execution-plan');
|
||||
const {
|
||||
activateInstall,
|
||||
pluginPackageTaskReconciliationFixture,
|
||||
} = require('../../../../test/contracts/pluginPackageTaskReconciliationRepositoryContract.cjs');
|
||||
const {
|
||||
LocalSqliteOperationAuthority,
|
||||
} = require('../../dist/authority/operationAuthority');
|
||||
const {
|
||||
LocalSqlitePluginPackageAutomationPublicationRepository,
|
||||
} = require('../../dist/plugin-package/pluginPackageAutomationPublicationRepository');
|
||||
const {
|
||||
LocalSqlitePluginPackageInstallRepository,
|
||||
} = require('../../dist/plugin-package/pluginPackageInstallRepository');
|
||||
const {
|
||||
LocalSqlitePluginPackageMaterializedRevisionRepository,
|
||||
} = require('../../dist/plugin-package/pluginPackageMaterializedRevisionRepository');
|
||||
const {
|
||||
LocalSqlitePluginPackageWorkflowAdmissionRepository,
|
||||
} = require('../../dist/plugin-package/workflow/pluginPackageWorkflowAdmissionRepository');
|
||||
const { auditLocalSqliteReadiness } = require('../../dist/readiness/readiness');
|
||||
const { migrateLocalSqlitePath } = require('../../dist/migration/migration');
|
||||
|
||||
const CRASH_POINTS = Object.freeze({
|
||||
after_run: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "Runs"',
|
||||
durable: false,
|
||||
}),
|
||||
after_admission_event: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "RunEvents"',
|
||||
durable: false,
|
||||
}),
|
||||
after_first_step_run: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "StepRuns"',
|
||||
durable: false,
|
||||
}),
|
||||
after_first_step_mutation: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "StepRunMutations"',
|
||||
durable: false,
|
||||
}),
|
||||
after_admission: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "QingLong3PluginPackageWorkflowAdmissions"',
|
||||
durable: false,
|
||||
}),
|
||||
after_first_admission_step: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "QingLong3PluginPackageWorkflowAdmissionSteps"',
|
||||
durable: false,
|
||||
}),
|
||||
before_commit: Object.freeze({
|
||||
timing: 'beforeExec',
|
||||
sql: 'COMMIT',
|
||||
durable: false,
|
||||
}),
|
||||
after_commit: Object.freeze({
|
||||
timing: 'afterExec',
|
||||
sql: 'COMMIT',
|
||||
durable: true,
|
||||
}),
|
||||
});
|
||||
|
||||
function fixture(profile) {
|
||||
const value = pluginPackageTaskReconciliationFixture(
|
||||
`workflow-admission-crash-${profile}`,
|
||||
{
|
||||
profile,
|
||||
workflows: [
|
||||
{
|
||||
schema: 'qinglong/plugin-package-workflow-resource@v1',
|
||||
id: 'daily',
|
||||
name: 'Daily workflow',
|
||||
enabled: true,
|
||||
steps: [
|
||||
{ id: 'collect', task: 'alpha', needs: [] },
|
||||
{ id: 'summarize', task: 'beta', needs: ['collect'] },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
return {
|
||||
...value,
|
||||
publication: createInitialPluginPackageAutomationPublication(
|
||||
value.revision,
|
||||
value.registry,
|
||||
2_000,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function executionPlan(value) {
|
||||
return createPluginPackageWorkflowExecutionPlan({
|
||||
planId: `workflow-admission-crash-plan-${value.profile}`,
|
||||
runId: `wfa-crash-run-${value.profile}`,
|
||||
workflowId: 'daily',
|
||||
stepRunIds: {
|
||||
collect: `workflow-admission-crash-collect-${value.profile}`,
|
||||
summarize: `workflow-admission-crash-summary-${value.profile}`,
|
||||
},
|
||||
publication: value.publication,
|
||||
revision: value.revision,
|
||||
taskSpecSemanticRegistry: value.registry,
|
||||
plannedAtMs: 3_000,
|
||||
});
|
||||
}
|
||||
|
||||
function client(databasePath) {
|
||||
const database = new DatabaseSync(databasePath);
|
||||
database.exec('PRAGMA foreign_keys = ON');
|
||||
return database;
|
||||
}
|
||||
|
||||
async function setupScenario({ databasePath, profile }) {
|
||||
await migrateLocalSqlitePath({ databasePath, profile });
|
||||
const value = fixture(profile);
|
||||
const database = client(databasePath);
|
||||
database
|
||||
.prepare(
|
||||
`INSERT INTO "QingLong3Projects"
|
||||
(id, name, slug, status, version, created_at_ms, updated_at_ms)
|
||||
VALUES (?, ?, ?, 'active', 1, 1, 1)`,
|
||||
)
|
||||
.run(value.projectId, value.projectId, value.projectId);
|
||||
const authority = new LocalSqliteOperationAuthority(database);
|
||||
try {
|
||||
await activateInstall(
|
||||
new LocalSqlitePluginPackageInstallRepository(authority),
|
||||
value,
|
||||
);
|
||||
await new LocalSqlitePluginPackageMaterializedRevisionRepository(
|
||||
authority,
|
||||
value.registry,
|
||||
).publish(value.revision);
|
||||
await new LocalSqlitePluginPackageAutomationPublicationRepository(
|
||||
authority,
|
||||
).publish(value.publication);
|
||||
} finally {
|
||||
await authority.close();
|
||||
}
|
||||
}
|
||||
|
||||
function writeCrashMarker(markerPath, pointName) {
|
||||
const descriptor = fs.openSync(markerPath, 'wx', 0o600);
|
||||
try {
|
||||
fs.writeSync(
|
||||
descriptor,
|
||||
JSON.stringify({
|
||||
schema:
|
||||
'qinglong/sqlite-plugin-package-workflow-admission-crash-marker@v1',
|
||||
point: pointName,
|
||||
pid: process.pid,
|
||||
}),
|
||||
);
|
||||
fs.fsyncSync(descriptor);
|
||||
} finally {
|
||||
fs.closeSync(descriptor);
|
||||
}
|
||||
}
|
||||
|
||||
function crashClient(database, pointName, markerPath) {
|
||||
const point = CRASH_POINTS[pointName];
|
||||
if (!point) throw new Error(`unknown crash point ${pointName}`);
|
||||
let triggered = false;
|
||||
const crash = () => {
|
||||
if (triggered) return;
|
||||
triggered = true;
|
||||
writeCrashMarker(markerPath, pointName);
|
||||
process.kill(process.pid, 'SIGKILL');
|
||||
throw new Error(`SIGKILL did not terminate ${pointName}`);
|
||||
};
|
||||
const matches = (timing, sql) =>
|
||||
!triggered && point.timing === timing && sql.trim().includes(point.sql);
|
||||
return new Proxy(database, {
|
||||
get(target, property) {
|
||||
if (property === 'exec') {
|
||||
return (sql) => {
|
||||
if (matches('beforeExec', sql)) crash();
|
||||
const result = target.exec(sql);
|
||||
if (matches('afterExec', sql)) crash();
|
||||
return result;
|
||||
};
|
||||
}
|
||||
if (property === 'prepare') {
|
||||
return (sql) => {
|
||||
const statement = target.prepare(sql);
|
||||
return new Proxy(statement, {
|
||||
get(statementTarget, statementProperty) {
|
||||
const value = Reflect.get(
|
||||
statementTarget,
|
||||
statementProperty,
|
||||
statementTarget,
|
||||
);
|
||||
if (statementProperty === 'run') {
|
||||
return (...values) => {
|
||||
const result = value.apply(statementTarget, values);
|
||||
if (matches('afterRun', sql)) crash();
|
||||
return result;
|
||||
};
|
||||
}
|
||||
return typeof value === 'function'
|
||||
? value.bind(statementTarget)
|
||||
: value;
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
const value = Reflect.get(target, property, target);
|
||||
return typeof value === 'function' ? value.bind(target) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function runCrashScenario({
|
||||
databasePath,
|
||||
markerPath,
|
||||
pointName,
|
||||
profile,
|
||||
}) {
|
||||
const database = client(databasePath);
|
||||
const authority = new LocalSqliteOperationAuthority(
|
||||
crashClient(database, pointName, markerPath),
|
||||
);
|
||||
const value = fixture(profile);
|
||||
await new LocalSqlitePluginPackageWorkflowAdmissionRepository(
|
||||
authority,
|
||||
).admit(executionPlan(value));
|
||||
throw new Error(`crash point ${pointName} was not reached`);
|
||||
}
|
||||
|
||||
async function verifyScenario({ databasePath, pointName, profile }) {
|
||||
const point = CRASH_POINTS[pointName];
|
||||
const value = fixture(profile);
|
||||
const plan = executionPlan(value);
|
||||
const database = client(databasePath);
|
||||
const authority = new LocalSqliteOperationAuthority(database);
|
||||
try {
|
||||
const repository = new LocalSqlitePluginPackageWorkflowAdmissionRepository(
|
||||
authority,
|
||||
);
|
||||
const beforeRecovery = await repository.findByPlanId(plan.planId);
|
||||
if (point.durable !== (beforeRecovery !== null)) {
|
||||
throw new Error(`${profile}/${pointName} durability is inconsistent`);
|
||||
}
|
||||
const recovered = await repository.admit(plan);
|
||||
if (recovered.status !== (point.durable ? 'existing' : 'created')) {
|
||||
throw new Error(`${profile}/${pointName} replay status is inconsistent`);
|
||||
}
|
||||
const counts = database
|
||||
.prepare(
|
||||
`SELECT
|
||||
(SELECT COUNT(*) FROM "Runs") AS runs,
|
||||
(SELECT COUNT(*) FROM "StepRuns") AS steps,
|
||||
(SELECT COUNT(*) FROM "RunEvents") AS events,
|
||||
(SELECT COUNT(*) FROM "StepRunMutations") AS mutations,
|
||||
(SELECT COUNT(*)
|
||||
FROM "QingLong3PluginPackageWorkflowAdmissions") AS admissions,
|
||||
(SELECT COUNT(*)
|
||||
FROM "QingLong3PluginPackageWorkflowAdmissionSteps")
|
||||
AS admissionSteps`,
|
||||
)
|
||||
.get();
|
||||
if (
|
||||
counts.runs !== 1 ||
|
||||
counts.steps !== 2 ||
|
||||
counts.events !== 3 ||
|
||||
counts.mutations !== 2 ||
|
||||
counts.admissions !== 1 ||
|
||||
counts.admissionSteps !== 2
|
||||
) {
|
||||
throw new Error(`${profile}/${pointName} evidence is incomplete`);
|
||||
}
|
||||
await auditLocalSqliteReadiness(database);
|
||||
const integrity = database.prepare('PRAGMA integrity_check').get();
|
||||
const foreignKey = database
|
||||
.prepare('SELECT * FROM pragma_foreign_key_check LIMIT 1')
|
||||
.get();
|
||||
const journal = database.prepare('PRAGMA journal_mode').get();
|
||||
const synchronous = database.prepare('PRAGMA synchronous').get();
|
||||
return Object.freeze({
|
||||
profile,
|
||||
pointName,
|
||||
crashBeforeCommit: !point.durable,
|
||||
durableAfterCrash: point.durable,
|
||||
exactReplay:
|
||||
recovered.status === (point.durable ? 'existing' : 'created'),
|
||||
integrityCheck: Object.values(integrity)[0],
|
||||
foreignKeyCheck: foreignKey === undefined ? 'ok' : 'failed',
|
||||
journalMode: journal.journal_mode,
|
||||
synchronous: synchronous.synchronous,
|
||||
});
|
||||
} finally {
|
||||
await authority.close();
|
||||
}
|
||||
}
|
||||
|
||||
function measuredClient(database, measurement) {
|
||||
let lockStartedAt;
|
||||
return new Proxy(database, {
|
||||
get(target, property) {
|
||||
if (property === 'exec') {
|
||||
return (sql) => {
|
||||
const normalized = sql.trim().toUpperCase();
|
||||
const result = target.exec(sql);
|
||||
if (normalized === 'BEGIN IMMEDIATE') {
|
||||
if (lockStartedAt !== undefined) {
|
||||
throw new Error('nested Workflow admission write lock');
|
||||
}
|
||||
measurement.beginImmediateCount += 1;
|
||||
lockStartedAt = performance.now();
|
||||
} else if (normalized === 'COMMIT') {
|
||||
if (lockStartedAt === undefined) {
|
||||
throw new Error(
|
||||
'Workflow admission committed without a write lock',
|
||||
);
|
||||
}
|
||||
measurement.commitCount += 1;
|
||||
measurement.lockDurationsMs.push(performance.now() - lockStartedAt);
|
||||
lockStartedAt = undefined;
|
||||
} else if (normalized === 'ROLLBACK') {
|
||||
measurement.rollbackCount += 1;
|
||||
lockStartedAt = undefined;
|
||||
}
|
||||
return result;
|
||||
};
|
||||
}
|
||||
const value = Reflect.get(target, property, target);
|
||||
return typeof value === 'function' ? value.bind(target) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function measuredExecutionPlan(value, sequence) {
|
||||
const suffix = String(sequence).padStart(4, '0');
|
||||
return createPluginPackageWorkflowExecutionPlan({
|
||||
planId: `wfl-plan-${value.profile}-${suffix}`,
|
||||
runId: `wfl-run-${value.profile}-${suffix}`,
|
||||
workflowId: 'daily',
|
||||
stepRunIds: {
|
||||
collect: `wfl-collect-${value.profile}-${suffix}`,
|
||||
summarize: `wfl-summary-${value.profile}-${suffix}`,
|
||||
},
|
||||
publication: value.publication,
|
||||
revision: value.revision,
|
||||
taskSpecSemanticRegistry: value.registry,
|
||||
plannedAtMs: 4_000 + sequence,
|
||||
});
|
||||
}
|
||||
|
||||
function rounded(value) {
|
||||
return Math.round(value * 1_000) / 1_000;
|
||||
}
|
||||
|
||||
function percentile(sortedValues, percentileValue) {
|
||||
const index = Math.min(
|
||||
sortedValues.length - 1,
|
||||
Math.ceil((percentileValue / 100) * sortedValues.length) - 1,
|
||||
);
|
||||
return sortedValues[Math.max(0, index)];
|
||||
}
|
||||
|
||||
async function measureWorkflowAdmissionTransactions({
|
||||
databasePath,
|
||||
profile,
|
||||
samples,
|
||||
}) {
|
||||
if (!Number.isSafeInteger(samples) || samples < 1 || samples > 1_000) {
|
||||
throw new RangeError('Workflow admission samples are out of range');
|
||||
}
|
||||
const value = fixture(profile);
|
||||
const database = client(databasePath);
|
||||
const measurement = {
|
||||
beginImmediateCount: 0,
|
||||
commitCount: 0,
|
||||
rollbackCount: 0,
|
||||
lockDurationsMs: [],
|
||||
};
|
||||
const authority = new LocalSqliteOperationAuthority(
|
||||
measuredClient(database, measurement),
|
||||
);
|
||||
try {
|
||||
const repository = new LocalSqlitePluginPackageWorkflowAdmissionRepository(
|
||||
authority,
|
||||
);
|
||||
for (let sequence = 1; sequence <= samples; sequence += 1) {
|
||||
const result = await repository.admit(
|
||||
measuredExecutionPlan(value, sequence),
|
||||
);
|
||||
if (result.status !== 'created') {
|
||||
throw new Error(
|
||||
`Workflow admission sample ${sequence} was not newly committed`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const counts = database
|
||||
.prepare(
|
||||
`SELECT
|
||||
(SELECT COUNT(*) FROM "Runs") AS runs,
|
||||
(SELECT COUNT(*) FROM "StepRuns") AS steps,
|
||||
(SELECT COUNT(*)
|
||||
FROM "QingLong3PluginPackageWorkflowAdmissions") AS admissions,
|
||||
(SELECT COUNT(*)
|
||||
FROM "QingLong3PluginPackageWorkflowAdmissionSteps")
|
||||
AS admissionSteps`,
|
||||
)
|
||||
.get();
|
||||
if (
|
||||
counts.runs !== samples ||
|
||||
counts.steps !== samples * 2 ||
|
||||
counts.admissions !== samples ||
|
||||
counts.admissionSteps !== samples * 2
|
||||
) {
|
||||
throw new Error('Workflow admission measurement facts are incomplete');
|
||||
}
|
||||
await auditLocalSqliteReadiness(database);
|
||||
const integrity = database.prepare('PRAGMA integrity_check').get();
|
||||
const foreignKey = database
|
||||
.prepare('SELECT * FROM pragma_foreign_key_check LIMIT 1')
|
||||
.get();
|
||||
const journal = database.prepare('PRAGMA journal_mode').get();
|
||||
const synchronous = database.prepare('PRAGMA synchronous').get();
|
||||
const sorted = [...measurement.lockDurationsMs].sort(
|
||||
(left, right) => left - right,
|
||||
);
|
||||
return Object.freeze({
|
||||
profile,
|
||||
samples,
|
||||
beginImmediateCount: measurement.beginImmediateCount,
|
||||
commitCount: measurement.commitCount,
|
||||
rollbackCount: measurement.rollbackCount,
|
||||
oneWriteTransactionPerWorkflow:
|
||||
measurement.beginImmediateCount === samples &&
|
||||
measurement.commitCount === samples &&
|
||||
measurement.rollbackCount === 0,
|
||||
lockDurationMs: Object.freeze({
|
||||
p50: rounded(percentile(sorted, 50)),
|
||||
p95: rounded(percentile(sorted, 95)),
|
||||
p99: rounded(percentile(sorted, 99)),
|
||||
max: rounded(sorted.at(-1)),
|
||||
}),
|
||||
integrityCheck: Object.values(integrity)[0],
|
||||
foreignKeyCheck: foreignKey === undefined ? 'ok' : 'failed',
|
||||
journalMode: journal.journal_mode,
|
||||
synchronous: synchronous.synchronous,
|
||||
});
|
||||
} finally {
|
||||
await authority.close();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
CRASH_POINTS,
|
||||
executionPlan,
|
||||
fixture,
|
||||
measureWorkflowAdmissionTransactions,
|
||||
setupScenario,
|
||||
verifyScenario,
|
||||
};
|
||||
|
||||
if (require.main === module) {
|
||||
const [, , action, databasePath, markerPath, pointName, profile] =
|
||||
process.argv;
|
||||
if (action !== 'crash') {
|
||||
throw new Error('fixture action must be crash');
|
||||
}
|
||||
runCrashScenario({
|
||||
databasePath,
|
||||
markerPath,
|
||||
pointName,
|
||||
profile,
|
||||
}).catch((error) => {
|
||||
process.stderr.write(`${error.stack ?? error}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
Vendored
+407
@@ -0,0 +1,407 @@
|
||||
const fs = require('node:fs');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
|
||||
const {
|
||||
LocalSqliteOperationAuthority,
|
||||
} = require('../../dist/authority/operationAuthority');
|
||||
const {
|
||||
LocalSqlitePluginPackageTaskReconciliationRepository,
|
||||
} = require('../../dist/plugin-package/pluginPackageTaskReconciliationRepository');
|
||||
const {
|
||||
LocalSqlitePluginPackageWorkflowAdmissionRepository,
|
||||
} = require('../../dist/plugin-package/workflow/pluginPackageWorkflowAdmissionRepository');
|
||||
const {
|
||||
LocalSqlitePluginPackageWorkflowTaskAttemptAdmissionRepository,
|
||||
} = require('../../dist/plugin-package/workflow/pluginPackageWorkflowTaskAttemptAdmissionRepository');
|
||||
const {
|
||||
LocalSqlitePluginPackageWorkflowCancellationConvergenceRepository,
|
||||
} = require('../../dist/plugin-package/workflow/pluginPackageWorkflowCancellationConvergenceRepository');
|
||||
const { LocalSqliteRunRepository } = require('../../dist/run/runRepository');
|
||||
const {
|
||||
LocalSqliteWorkflowTaskExecutionRepository,
|
||||
} = require('../../dist/plugin-package/workflow/workflowTaskExecutionRepository');
|
||||
const { auditLocalSqliteReadiness } = require('../../dist/readiness/readiness');
|
||||
const {
|
||||
executionPlan,
|
||||
fixture,
|
||||
setupScenario: setupWorkflowAdmissionScenario,
|
||||
} = require('./pluginPackageWorkflowAdmissionCrashMatrixFixture.cjs');
|
||||
|
||||
const CRASH_POINTS = Object.freeze({
|
||||
after_conclusive_stop_before_begin: Object.freeze({
|
||||
timing: 'beforeExec',
|
||||
sql: 'BEGIN IMMEDIATE',
|
||||
durable: false,
|
||||
}),
|
||||
after_attempt_terminal: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'UPDATE "RunAttempts"',
|
||||
durable: false,
|
||||
}),
|
||||
after_run_cas: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'UPDATE "Runs"',
|
||||
durable: false,
|
||||
}),
|
||||
after_step_terminal: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'UPDATE "StepRuns"',
|
||||
durable: false,
|
||||
}),
|
||||
after_attempt_event: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "RunEvents"',
|
||||
durable: false,
|
||||
}),
|
||||
after_step_mutation: Object.freeze({
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "StepRunMutations"',
|
||||
durable: false,
|
||||
}),
|
||||
before_commit: Object.freeze({
|
||||
timing: 'beforeExec',
|
||||
sql: 'COMMIT',
|
||||
durable: false,
|
||||
}),
|
||||
after_commit: Object.freeze({
|
||||
timing: 'afterExec',
|
||||
sql: 'COMMIT',
|
||||
durable: true,
|
||||
}),
|
||||
});
|
||||
|
||||
function client(databasePath) {
|
||||
const database = new DatabaseSync(databasePath);
|
||||
database.exec('PRAGMA foreign_keys = ON');
|
||||
return database;
|
||||
}
|
||||
|
||||
function writeCrashMarker(markerPath, pointName) {
|
||||
const descriptor = fs.openSync(markerPath, 'wx', 0o600);
|
||||
try {
|
||||
fs.writeSync(
|
||||
descriptor,
|
||||
JSON.stringify({
|
||||
schema:
|
||||
'qinglong/sqlite-plugin-package-workflow-task-control-crash-marker@v1',
|
||||
point: pointName,
|
||||
conclusiveStopObserved: true,
|
||||
pid: process.pid,
|
||||
}),
|
||||
);
|
||||
fs.fsyncSync(descriptor);
|
||||
} finally {
|
||||
fs.closeSync(descriptor);
|
||||
}
|
||||
}
|
||||
|
||||
function crashClient(database, pointName, markerPath) {
|
||||
const point = CRASH_POINTS[pointName];
|
||||
if (!point) throw new Error(`unknown crash point ${pointName}`);
|
||||
let triggered = false;
|
||||
const crash = () => {
|
||||
if (triggered) return;
|
||||
triggered = true;
|
||||
writeCrashMarker(markerPath, pointName);
|
||||
process.kill(process.pid, 'SIGKILL');
|
||||
throw new Error(`SIGKILL did not terminate ${pointName}`);
|
||||
};
|
||||
const matches = (timing, sql) =>
|
||||
!triggered && point.timing === timing && sql.trim().includes(point.sql);
|
||||
return new Proxy(database, {
|
||||
get(target, property) {
|
||||
if (property === 'exec') {
|
||||
return (sql) => {
|
||||
if (matches('beforeExec', sql)) crash();
|
||||
const result = target.exec(sql);
|
||||
if (matches('afterExec', sql)) crash();
|
||||
return result;
|
||||
};
|
||||
}
|
||||
if (property === 'prepare') {
|
||||
return (sql) => {
|
||||
const statement = target.prepare(sql);
|
||||
return new Proxy(statement, {
|
||||
get(statementTarget, statementProperty) {
|
||||
const value = Reflect.get(
|
||||
statementTarget,
|
||||
statementProperty,
|
||||
statementTarget,
|
||||
);
|
||||
if (statementProperty === 'run') {
|
||||
return (...values) => {
|
||||
const result = value.apply(statementTarget, values);
|
||||
if (matches('afterRun', sql)) crash();
|
||||
return result;
|
||||
};
|
||||
}
|
||||
return typeof value === 'function'
|
||||
? value.bind(statementTarget)
|
||||
: value;
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
const value = Reflect.get(target, property, target);
|
||||
return typeof value === 'function' ? value.bind(target) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function command(run, attempt, pointName, profile) {
|
||||
const identity = createHash('sha256')
|
||||
.update(profile)
|
||||
.update('\0')
|
||||
.update(pointName)
|
||||
.digest('hex')
|
||||
.slice(0, 16);
|
||||
return {
|
||||
run,
|
||||
attempt,
|
||||
reason: 'user',
|
||||
terminalStatus: 'cancelled',
|
||||
errorCode: 'EXECUTION_CANCELLED',
|
||||
errorSummary: 'Execution was cancelled',
|
||||
finishedAtMs: Math.max(
|
||||
Date.now(),
|
||||
run.updatedAtMs ?? 0,
|
||||
attempt.startedAtMs ?? 0,
|
||||
),
|
||||
attemptEventId: `wfc-a-${identity}`,
|
||||
stepMutationId: `wfc-s-${identity}`,
|
||||
};
|
||||
}
|
||||
|
||||
async function setupScenario({ databasePath, profile }) {
|
||||
await setupWorkflowAdmissionScenario({ databasePath, profile });
|
||||
const value = fixture(profile);
|
||||
const plan = executionPlan(value);
|
||||
const database = client(databasePath);
|
||||
const authority = new LocalSqliteOperationAuthority(database);
|
||||
try {
|
||||
await new LocalSqlitePluginPackageTaskReconciliationRepository(
|
||||
authority,
|
||||
value.registry,
|
||||
).reconcile(value.revision, {
|
||||
async findActiveResourceGeneration() {
|
||||
return value.revision.generation;
|
||||
},
|
||||
});
|
||||
await new LocalSqlitePluginPackageWorkflowAdmissionRepository(
|
||||
authority,
|
||||
).admit(plan);
|
||||
const collect = plan.steps.find(({ stepKey }) => stepKey === 'collect');
|
||||
if (!collect) throw new Error('Workflow collect Step is missing');
|
||||
const admitted =
|
||||
await new LocalSqlitePluginPackageWorkflowTaskAttemptAdmissionRepository(
|
||||
authority,
|
||||
).admit(plan.runId, collect.stepRunId);
|
||||
const runs = new LocalSqliteRunRepository(database);
|
||||
const execution = new LocalSqliteWorkflowTaskExecutionRepository(authority);
|
||||
const callbackTokenHash = 'c'.repeat(64);
|
||||
const startingAtMs = admitted.receipt.admittedAtMs + 1;
|
||||
const runningAtMs = startingAtMs + 1;
|
||||
const prepared = await execution.prepare({
|
||||
runId: plan.runId,
|
||||
attemptId: admitted.receipt.attemptId,
|
||||
stepRunId: collect.stepRunId,
|
||||
callbackTokenHash,
|
||||
deadlineAtMs: startingAtMs + 60_000,
|
||||
logArtifactId: 'local-0123456789abcdef0123456789abcd',
|
||||
atMs: startingAtMs,
|
||||
eventId: `wfc-start-${profile}`,
|
||||
});
|
||||
if (prepared.status !== 'applied') {
|
||||
throw new Error('Workflow Task did not enter starting');
|
||||
}
|
||||
const run = await runs.findRunById(plan.runId);
|
||||
const attempt = await runs.findAttemptById(admitted.receipt.attemptId);
|
||||
if (!run || !attempt) {
|
||||
throw new Error('Workflow Task starting authority is missing');
|
||||
}
|
||||
const running = await execution.recordRunning({
|
||||
run,
|
||||
attempt,
|
||||
callbackTokenHash,
|
||||
executorHandle: `qlp:v1:workflow-control-${profile}`,
|
||||
pid: 321,
|
||||
startedAtMs: runningAtMs,
|
||||
attemptEventId: `wfc-running-a-${profile}`,
|
||||
stepMutationId: `wfc-running-s-${profile}`,
|
||||
});
|
||||
if (running.status !== 'applied') {
|
||||
throw new Error('Workflow Task did not enter running');
|
||||
}
|
||||
const cancellation = database
|
||||
.prepare(
|
||||
`UPDATE "Runs"
|
||||
SET cancel_requested_at_ms = ?,
|
||||
cancel_reason = 'user'
|
||||
WHERE id = ? AND status = 'running'
|
||||
AND cancel_requested_at_ms IS NULL`,
|
||||
)
|
||||
.run(runningAtMs + 1, plan.runId);
|
||||
if (cancellation.changes !== 1) {
|
||||
throw new Error('Workflow cancellation intent was not recorded');
|
||||
}
|
||||
return Object.freeze({
|
||||
runId: plan.runId,
|
||||
attemptId: admitted.receipt.attemptId,
|
||||
stepRunId: collect.stepRunId,
|
||||
});
|
||||
} finally {
|
||||
await authority.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function runCrashScenario({
|
||||
databasePath,
|
||||
markerPath,
|
||||
pointName,
|
||||
profile,
|
||||
}) {
|
||||
const database = client(databasePath);
|
||||
const runs = new LocalSqliteRunRepository(database);
|
||||
const value = fixture(profile);
|
||||
const plan = executionPlan(value);
|
||||
const run = await runs.findRunById(plan.runId);
|
||||
const attempt = await runs.findLatestAttemptByRunId(plan.runId);
|
||||
if (!run || !attempt) {
|
||||
throw new Error('Workflow Task control authority is missing');
|
||||
}
|
||||
const authority = new LocalSqliteOperationAuthority(
|
||||
crashClient(database, pointName, markerPath),
|
||||
);
|
||||
await new LocalSqliteWorkflowTaskExecutionRepository(
|
||||
authority,
|
||||
).recordControlTerminal(command(run, attempt, pointName, profile));
|
||||
throw new Error(`crash point ${pointName} was not reached`);
|
||||
}
|
||||
|
||||
async function verifyScenario({ databasePath, pointName, profile }) {
|
||||
const point = CRASH_POINTS[pointName];
|
||||
if (!point) throw new Error(`unknown crash point ${pointName}`);
|
||||
const database = client(databasePath);
|
||||
const authority = new LocalSqliteOperationAuthority(database);
|
||||
try {
|
||||
const value = fixture(profile);
|
||||
const plan = executionPlan(value);
|
||||
const runs = new LocalSqliteRunRepository(database);
|
||||
const execution = new LocalSqliteWorkflowTaskExecutionRepository(authority);
|
||||
let run = await runs.findRunById(plan.runId);
|
||||
let attempt = await runs.findLatestAttemptByRunId(plan.runId);
|
||||
if (!run || !attempt) {
|
||||
throw new Error('Workflow Task recovery authority is missing');
|
||||
}
|
||||
if (point.durable !== (attempt.status === 'cancelled')) {
|
||||
throw new Error(`${profile}/${pointName} durability is inconsistent`);
|
||||
}
|
||||
const recovered = await execution.recordControlTerminal(
|
||||
command(run, attempt, pointName, profile),
|
||||
);
|
||||
if (recovered !== (point.durable ? 'already_terminal' : 'terminal')) {
|
||||
throw new Error(`${profile}/${pointName} replay is inconsistent`);
|
||||
}
|
||||
const cancellation =
|
||||
new LocalSqlitePluginPackageWorkflowCancellationConvergenceRepository(
|
||||
authority,
|
||||
);
|
||||
const converged = await cancellation.convergePage({ limit: 8 });
|
||||
if (
|
||||
converged.settledRuns !== 1 ||
|
||||
converged.settledAttempts !== 0 ||
|
||||
converged.blocked !== 0
|
||||
) {
|
||||
throw new Error(`${profile}/${pointName} parent did not converge`);
|
||||
}
|
||||
run = await runs.findRunById(plan.runId);
|
||||
attempt = await runs.findLatestAttemptByRunId(plan.runId);
|
||||
const facts = database
|
||||
.prepare(
|
||||
`SELECT
|
||||
(SELECT COUNT(*) FROM "RunAttempts"
|
||||
WHERE run_id = ?) AS attempts,
|
||||
(SELECT COUNT(*) FROM "RunEvents"
|
||||
WHERE run_id = ? AND type =
|
||||
'workflow.task_attempt.cancelled') AS attemptEvents,
|
||||
(SELECT COUNT(*) FROM "RunEvents"
|
||||
WHERE run_id = ? AND type = 'step.cancelled') AS stepEvents,
|
||||
(SELECT COUNT(*) FROM "RunEvents"
|
||||
WHERE run_id = ? AND type =
|
||||
'workflow.cancelled') AS workflowEvents,
|
||||
(SELECT COUNT(*) FROM "StepRuns"
|
||||
WHERE run_id = ? AND status = 'cancelled') AS cancelledSteps`,
|
||||
)
|
||||
.get(plan.runId, plan.runId, plan.runId, plan.runId, plan.runId);
|
||||
if (
|
||||
run?.status !== 'cancelled' ||
|
||||
run.version !== run.eventSequence ||
|
||||
attempt?.status !== 'cancelled' ||
|
||||
facts.attempts !== 1 ||
|
||||
facts.attemptEvents !== 1 ||
|
||||
facts.stepEvents !== 2 ||
|
||||
facts.workflowEvents !== 1 ||
|
||||
facts.cancelledSteps !== 2
|
||||
) {
|
||||
throw new Error(`${profile}/${pointName} terminal facts are incomplete`);
|
||||
}
|
||||
const replay = await execution.recordControlTerminal(
|
||||
command(run, attempt, pointName, profile),
|
||||
);
|
||||
if (replay !== 'already_terminal') {
|
||||
throw new Error(`${profile}/${pointName} terminal replay drifted`);
|
||||
}
|
||||
const empty = await cancellation.convergePage({ limit: 8 });
|
||||
if (
|
||||
empty.scanned !== 0 ||
|
||||
empty.settledRuns !== 0 ||
|
||||
empty.settledAttempts !== 0
|
||||
) {
|
||||
throw new Error(`${profile}/${pointName} cancellation replay drifted`);
|
||||
}
|
||||
await auditLocalSqliteReadiness(database);
|
||||
const integrity = database.prepare('PRAGMA integrity_check').get();
|
||||
const foreignKey = database
|
||||
.prepare('SELECT * FROM pragma_foreign_key_check LIMIT 1')
|
||||
.get();
|
||||
const journal = database.prepare('PRAGMA journal_mode').get();
|
||||
return Object.freeze({
|
||||
profile,
|
||||
pointName,
|
||||
crashAfterConclusiveStop: true,
|
||||
crashBeforeCommit: !point.durable,
|
||||
durableAfterCrash: point.durable,
|
||||
exactTerminalReplay: replay === 'already_terminal',
|
||||
parentConverged: run.status === 'cancelled',
|
||||
integrityCheck: Object.values(integrity)[0],
|
||||
foreignKeyCheck: foreignKey === undefined ? 'ok' : 'failed',
|
||||
journalMode: journal.journal_mode,
|
||||
});
|
||||
} finally {
|
||||
await authority.close();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
CRASH_POINTS,
|
||||
setupScenario,
|
||||
verifyScenario,
|
||||
};
|
||||
|
||||
if (require.main === module) {
|
||||
const [, , action, databasePath, markerPath, pointName, profile] =
|
||||
process.argv;
|
||||
if (action !== 'crash') {
|
||||
throw new Error('fixture action must be crash');
|
||||
}
|
||||
runCrashScenario({
|
||||
databasePath,
|
||||
markerPath,
|
||||
pointName,
|
||||
profile,
|
||||
}).catch((error) => {
|
||||
process.stderr.write(`${error.stack ?? error}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,957 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const fs = require('node:fs');
|
||||
|
||||
const {
|
||||
createPluginPackageResourceGenerationFromReferences,
|
||||
} = require('@qinglong/runtime-core/plugin-package-resource-generation');
|
||||
const {
|
||||
createProjectToolDefinitionSnapshot,
|
||||
projectToolDefinitionRegistry,
|
||||
} = require('@qinglong/runtime-core/project-tool-definition-snapshot');
|
||||
const {
|
||||
createStepRunMutation,
|
||||
transitionStepRunMutation,
|
||||
} = require('@qinglong/runtime-core/step-run');
|
||||
const {
|
||||
createToolExecutionCompletionCommand,
|
||||
createToolExecutionResultArtifact,
|
||||
toolExecutionResultKeyBinding,
|
||||
} = require('@qinglong/runtime-core/tool-execution-completion');
|
||||
const {
|
||||
createToolExecutionEvidenceBundle,
|
||||
toolExecutionAdmissionEvidence,
|
||||
TOOL_EXECUTION_START_AUDIT_OPERATION,
|
||||
} = require('@qinglong/runtime-core/tool-execution-evidence');
|
||||
const {
|
||||
createToolExecutionStartCommand,
|
||||
} = require('@qinglong/runtime-core/tool-execution-start-barrier');
|
||||
const {
|
||||
TrustedToolHandlerBindingRegistry,
|
||||
admitTrustedToolExecution,
|
||||
createTrustedToolHandlerBinding,
|
||||
createTrustedToolInvocationPlan,
|
||||
trustedToolContractIdentityDigest,
|
||||
} = require('@qinglong/runtime-core/trusted-tool-invocation');
|
||||
const {
|
||||
TRUSTED_TOOL_EXECUTION_RESULT_SCHEMA,
|
||||
} = require('@qinglong/runtime-core/trusted-tool-execution');
|
||||
const {
|
||||
prepareToolInvocation,
|
||||
} = require('@qinglong/runtime-core/tool-registry');
|
||||
const {
|
||||
createToolResultKeyCatalogBootstrapCommand,
|
||||
createToolResultKeyRetirementCommand,
|
||||
createToolResultKeyRotationCommand,
|
||||
requireActiveToolResultKey,
|
||||
toolResultKeyCatalogFence,
|
||||
toolResultKeyMaterialProof,
|
||||
} = require('@qinglong/runtime-core/tool-result-key-catalog');
|
||||
const {
|
||||
createToolExecutionResultRekeyCommand,
|
||||
createToolResultKeyRetirementReceiptCommand,
|
||||
} = require('@qinglong/runtime-core/tool-result-rekey');
|
||||
|
||||
const { openLocalSqliteClient } = require('../../dist/storage/config');
|
||||
const {
|
||||
migrateLocalSqliteDatabase,
|
||||
} = require('../../dist/migration/migration');
|
||||
const {
|
||||
LocalSqliteOperationAuthority,
|
||||
} = require('../../dist/authority/operationAuthority');
|
||||
const {
|
||||
LocalSqliteStepRunRepository,
|
||||
} = require('../../dist/run/stepRunRepository');
|
||||
const {
|
||||
LocalSqliteToolExecutionCompletionRepository,
|
||||
} = require('../../dist/tool-execution/toolExecutionCompletionRepository');
|
||||
const {
|
||||
LocalSqliteToolExecutionStartBarrierRepository,
|
||||
} = require('../../dist/tool-execution/toolExecutionStartBarrierRepository');
|
||||
const {
|
||||
LocalSqliteToolInvocationArtifactRepository,
|
||||
} = require('../../dist/tool-execution/toolInvocationArtifactRepository');
|
||||
const {
|
||||
LocalSqliteToolResultKeyCatalogRepository,
|
||||
} = require('../../dist/tool-execution/toolResultKeyCatalogRepository');
|
||||
const {
|
||||
LocalSqliteToolResultRekeyRepository,
|
||||
} = require('../../dist/tool-execution/toolResultRekeyRepository');
|
||||
|
||||
const PROJECT_ID = 'crash-tool-result-project';
|
||||
const RUN_ID = 'crash-tool-result-run';
|
||||
const STEP_RUN_ID = 'crash-tool-result-step';
|
||||
const START_ID = 'crash-tool-result-start';
|
||||
const RESULT_KEY_A_ID = 'crash-result-key-a';
|
||||
const RESULT_KEY_B_ID = 'crash-result-key-b';
|
||||
const RESULT_KEY_A = Buffer.alloc(32, 41);
|
||||
const RESULT_KEY_B = Buffer.alloc(32, 42);
|
||||
const INVOCATION_KEY = Buffer.alloc(32, 43);
|
||||
const SUBJECT = Object.freeze({
|
||||
type: 'user',
|
||||
id: 'usr-crash-tool-result',
|
||||
});
|
||||
const POLICY_FENCE = Object.freeze({
|
||||
projectVersion: 1,
|
||||
bindingVersion: 1,
|
||||
});
|
||||
const TOOL = Object.freeze({
|
||||
name: 'crash.result.read',
|
||||
version: '1.0.0',
|
||||
});
|
||||
const OUTPUT = Object.freeze({
|
||||
summary: 'SQLite crash matrix durable result',
|
||||
});
|
||||
const OUTPUT_DIGEST_DOMAIN = Buffer.from(
|
||||
'qinglong/trusted-tool-execution-output-digest@v1\0',
|
||||
'utf8',
|
||||
);
|
||||
const RESULT_DIGEST_DOMAIN = Buffer.from(
|
||||
'qinglong/trusted-tool-execution-result-digest@v1\0',
|
||||
'utf8',
|
||||
);
|
||||
|
||||
const CRASH_POINTS = Object.freeze({
|
||||
completion_before_begin: Object.freeze({
|
||||
operation: 'completion',
|
||||
timing: 'beforeExec',
|
||||
sql: 'BEGIN IMMEDIATE',
|
||||
durable: false,
|
||||
}),
|
||||
completion_after_binding: Object.freeze({
|
||||
operation: 'completion',
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "ToolExecutionResultKeyBindings"',
|
||||
durable: false,
|
||||
}),
|
||||
completion_after_commit: Object.freeze({
|
||||
operation: 'completion',
|
||||
timing: 'afterExec',
|
||||
sql: 'COMMIT',
|
||||
durable: true,
|
||||
}),
|
||||
rekey_after_overlay: Object.freeze({
|
||||
operation: 'rekey',
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "ToolExecutionResultRekeyOverlays"',
|
||||
durable: false,
|
||||
}),
|
||||
rekey_after_head: Object.freeze({
|
||||
operation: 'rekey',
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "ToolExecutionResultRekeyHeads"',
|
||||
durable: false,
|
||||
}),
|
||||
rekey_after_commit: Object.freeze({
|
||||
operation: 'rekey',
|
||||
timing: 'afterExec',
|
||||
sql: 'COMMIT',
|
||||
durable: true,
|
||||
}),
|
||||
receipt_after_insert: Object.freeze({
|
||||
operation: 'receipt',
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "ToolResultKeyRetirementReceipts"',
|
||||
durable: false,
|
||||
}),
|
||||
receipt_after_commit: Object.freeze({
|
||||
operation: 'receipt',
|
||||
timing: 'afterExec',
|
||||
sql: 'COMMIT',
|
||||
durable: true,
|
||||
}),
|
||||
retire_after_insert: Object.freeze({
|
||||
operation: 'retire',
|
||||
timing: 'afterRun',
|
||||
sql: 'INSERT INTO "ToolResultKeyCatalogGenerations"',
|
||||
durable: false,
|
||||
}),
|
||||
retire_after_commit: Object.freeze({
|
||||
operation: 'retire',
|
||||
timing: 'afterExec',
|
||||
sql: 'COMMIT',
|
||||
durable: true,
|
||||
}),
|
||||
});
|
||||
|
||||
function hash(domain, value) {
|
||||
return createHash('sha256')
|
||||
.update(domain)
|
||||
.update(JSON.stringify(value))
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
function snapshot() {
|
||||
const generation = createPluginPackageResourceGenerationFromReferences({
|
||||
installationId: 'install-crash-tool-result',
|
||||
projectId: PROJECT_ID,
|
||||
packageName: 'crash',
|
||||
lockDigest: 'a'.repeat(64),
|
||||
generation: 1,
|
||||
previousActiveLockDigest: null,
|
||||
contentDigest: 'b'.repeat(64),
|
||||
resources: [],
|
||||
});
|
||||
return createProjectToolDefinitionSnapshot({
|
||||
projectId: PROJECT_ID,
|
||||
contributions: [
|
||||
{
|
||||
generation,
|
||||
revisionDigest: 'c'.repeat(64),
|
||||
definitions: [
|
||||
{
|
||||
...TOOL,
|
||||
description: 'Read one SQLite crash matrix fixture',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
runId: { type: 'string', minLength: 1, maxLength: 64 },
|
||||
},
|
||||
required: ['runId'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
outputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
summary: { type: 'string', maxLength: 1024 },
|
||||
},
|
||||
required: ['summary'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
effect: 'read',
|
||||
risk: 'low',
|
||||
requiredPermissions: ['run.read'],
|
||||
timeoutSeconds: 30,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
function principal() {
|
||||
return Object.freeze({
|
||||
subject: SUBJECT,
|
||||
authenticationId: 'auth-crash-tool-result',
|
||||
authenticatedAtMs: 800,
|
||||
expiresAtMs: 10_000,
|
||||
assurance: 'local_console',
|
||||
});
|
||||
}
|
||||
|
||||
function authorizer() {
|
||||
return Object.freeze({
|
||||
async authorize() {
|
||||
return Object.freeze({
|
||||
effect: 'allow',
|
||||
reasons: Object.freeze(['role_grant']),
|
||||
fence: POLICY_FENCE,
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function bindingRegistry(definitionSnapshot) {
|
||||
const binding = createTrustedToolHandlerBinding(definitionSnapshot, {
|
||||
tool: TOOL,
|
||||
adapter: {
|
||||
id: 'builtin.crash-result-read',
|
||||
version: '1.0.0',
|
||||
},
|
||||
executionClass: 'builtin_in_process',
|
||||
profiles: ['edge', 'standalone'],
|
||||
authorities: ['database.read'],
|
||||
timeoutSeconds: 20,
|
||||
redactionContract: {
|
||||
id: 'redaction.crash-result-read',
|
||||
version: '1.0.0',
|
||||
},
|
||||
auditContract: {
|
||||
id: 'audit.tool-call',
|
||||
version: '1.0.0',
|
||||
},
|
||||
});
|
||||
return Object.freeze({
|
||||
binding,
|
||||
bindings: new TrustedToolHandlerBindingRegistry(definitionSnapshot, [
|
||||
binding,
|
||||
]),
|
||||
});
|
||||
}
|
||||
|
||||
function openClient(databasePath, profile) {
|
||||
return openLocalSqliteClient(
|
||||
{
|
||||
databasePath,
|
||||
profile,
|
||||
busyTimeoutMs: 5_000,
|
||||
},
|
||||
false,
|
||||
);
|
||||
}
|
||||
|
||||
async function prepareCompletionFixture(client, authority, profile) {
|
||||
client.exec(`
|
||||
INSERT INTO "QingLong3Projects" (
|
||||
id, name, slug, status, version, created_at_ms, updated_at_ms
|
||||
) VALUES (
|
||||
'${PROJECT_ID}', 'Crash Tool Result', 'crash-tool-result',
|
||||
'active', 1, 1, 1
|
||||
);
|
||||
INSERT INTO "Runs" (
|
||||
id, project_id, task_id, task_revision, trigger_type,
|
||||
execution_origin, execution_owner, status, version,
|
||||
event_sequence, priority, created_at_ms
|
||||
) VALUES (
|
||||
'${RUN_ID}', '${PROJECT_ID}', 'crash-tool-result-task', 'v1',
|
||||
'manual', 'manual', 'runtime', 'running', 0, 0, 0, 1
|
||||
);
|
||||
`);
|
||||
const definitionSnapshot = snapshot();
|
||||
const { binding, bindings } = bindingRegistry(definitionSnapshot);
|
||||
const stepRuns = new LocalSqliteStepRunRepository(authority);
|
||||
const creation = createStepRunMutation(
|
||||
{
|
||||
id: STEP_RUN_ID,
|
||||
runId: RUN_ID,
|
||||
stepKey: 'workflow.crash-result-read',
|
||||
kind: 'tool',
|
||||
definitionRef: `tool:${TOOL.name}@${TOOL.version}`,
|
||||
definitionDigest: definitionSnapshot.definitions[0].definitionDigest,
|
||||
required: true,
|
||||
initialStatus: 'ready',
|
||||
inputRef: 'artifact:crash-tool-result-input',
|
||||
mutationId: 'crash-tool-result-create',
|
||||
createdAtMs: 1_000,
|
||||
},
|
||||
{
|
||||
expectedRunVersion: 0,
|
||||
expectedRunEventSequence: 0,
|
||||
eventId: '51000000-0000-4000-8000-000000000001',
|
||||
dedupeKey: 'crash-tool-result:create',
|
||||
actor: SUBJECT,
|
||||
},
|
||||
);
|
||||
assert.equal((await stepRuns.apply(creation)).status, 'applied');
|
||||
|
||||
const invocation = await prepareToolInvocation(
|
||||
projectToolDefinitionRegistry(definitionSnapshot),
|
||||
{
|
||||
projectId: PROJECT_ID,
|
||||
principal: principal(),
|
||||
nowMs: 900,
|
||||
tool: TOOL,
|
||||
input: { runId: RUN_ID },
|
||||
},
|
||||
authorizer(),
|
||||
);
|
||||
const planBundle = createTrustedToolInvocationPlan(bindings, invocation, {
|
||||
actionRef: `tool-plan:${RUN_ID}`,
|
||||
inputArtifactId: 'crash-tool-result-input-artifact',
|
||||
previewArtifactId: 'crash-tool-result-preview-artifact',
|
||||
artifactKeyId: 'crash-tool-invocation-key',
|
||||
artifactKey: INVOCATION_KEY,
|
||||
artifactNonce: Buffer.alloc(12, 44),
|
||||
profile,
|
||||
preview: {
|
||||
title: 'SQLite Crash Tool Result',
|
||||
summary: 'Creates one encrypted crash-matrix completion',
|
||||
fields: [
|
||||
{
|
||||
kind: 'identifier',
|
||||
label: 'Run',
|
||||
value: RUN_ID,
|
||||
},
|
||||
],
|
||||
warnings: [],
|
||||
},
|
||||
sealedAtMs: 1_100,
|
||||
});
|
||||
assert.deepEqual(
|
||||
await new LocalSqliteToolInvocationArtifactRepository(authority).put(
|
||||
planBundle.inputArtifact,
|
||||
planBundle.previewArtifact,
|
||||
),
|
||||
{ status: 'inserted' },
|
||||
);
|
||||
const startedAtMs = 1_200;
|
||||
const evidence = createToolExecutionEvidenceBundle({
|
||||
traceId: '1'.repeat(32),
|
||||
spanId: '2'.repeat(16),
|
||||
projectId: PROJECT_ID,
|
||||
runId: RUN_ID,
|
||||
stepRunId: STEP_RUN_ID,
|
||||
invocationPlanDigest: planBundle.plan.planDigest,
|
||||
bindingDigest: binding.bindingDigest,
|
||||
adapterDigest: trustedToolContractIdentityDigest(binding.adapter),
|
||||
redactionContractDigest: trustedToolContractIdentityDigest(
|
||||
binding.redactionContract,
|
||||
),
|
||||
auditContractDigest: trustedToolContractIdentityDigest(
|
||||
binding.auditContract,
|
||||
),
|
||||
audit: {
|
||||
eventId: '41000000-0000-4000-8000-000000000001',
|
||||
requestId: 'crash-tool-result-request',
|
||||
operationId: TOOL_EXECUTION_START_AUDIT_OPERATION,
|
||||
projectId: PROJECT_ID,
|
||||
subject: SUBJECT,
|
||||
authenticationId: 'auth-crash-tool-result',
|
||||
outcome: 'allowed',
|
||||
reasons: ['tool_execution_start'],
|
||||
fence: POLICY_FENCE,
|
||||
occurredAtMs: startedAtMs,
|
||||
},
|
||||
createdAtMs: startedAtMs,
|
||||
});
|
||||
const admission = await admitTrustedToolExecution(
|
||||
bindings,
|
||||
planBundle.plan,
|
||||
{
|
||||
principal: principal(),
|
||||
profile,
|
||||
nowMs: startedAtMs,
|
||||
authorizer: authorizer(),
|
||||
evidence: {
|
||||
stepRun: {
|
||||
id: creation.stepRun.id,
|
||||
version: creation.stepRun.version,
|
||||
digest: creation.stepRun.stepRunDigest,
|
||||
},
|
||||
...toolExecutionAdmissionEvidence(evidence),
|
||||
},
|
||||
},
|
||||
);
|
||||
const runningMutation = transitionStepRunMutation(
|
||||
creation.stepRun,
|
||||
{
|
||||
expectedVersion: creation.stepRun.version,
|
||||
expectedDigest: creation.stepRun.stepRunDigest,
|
||||
mutationId: 'crash-tool-result-running',
|
||||
to: 'running',
|
||||
atMs: startedAtMs,
|
||||
},
|
||||
{
|
||||
expectedRunVersion: 1,
|
||||
expectedRunEventSequence: 1,
|
||||
eventId: '51000000-0000-4000-8000-000000000002',
|
||||
dedupeKey: 'crash-tool-result:running',
|
||||
actor: SUBJECT,
|
||||
},
|
||||
);
|
||||
const start = await new LocalSqliteToolExecutionStartBarrierRepository(
|
||||
authority,
|
||||
).prepare(
|
||||
createToolExecutionStartCommand({
|
||||
startId: START_ID,
|
||||
admission,
|
||||
evidence,
|
||||
stepRunMutation: runningMutation,
|
||||
}),
|
||||
);
|
||||
assert.equal(start.status, 'created');
|
||||
|
||||
const catalogRepository =
|
||||
new LocalSqliteToolResultKeyCatalogRepository(authority);
|
||||
const catalog = await catalogRepository.append(
|
||||
createToolResultKeyCatalogBootstrapCommand({
|
||||
keyId: RESULT_KEY_A_ID,
|
||||
materialProof: toolResultKeyMaterialProof(
|
||||
RESULT_KEY_A_ID,
|
||||
RESULT_KEY_A,
|
||||
),
|
||||
mutationId: 'crash-result-key-bootstrap-a',
|
||||
}),
|
||||
);
|
||||
const executionResultUnsigned = Object.freeze({
|
||||
schema: TRUSTED_TOOL_EXECUTION_RESULT_SCHEMA,
|
||||
startId: START_ID,
|
||||
barrierDigest: start.barrier.barrierDigest,
|
||||
adapterDigest: start.barrier.adapterDigest,
|
||||
output: OUTPUT,
|
||||
outputDigest: hash(OUTPUT_DIGEST_DOMAIN, OUTPUT),
|
||||
completedAtMs: 1_300,
|
||||
});
|
||||
const executionResult = Object.freeze({
|
||||
...executionResultUnsigned,
|
||||
resultDigest: hash(RESULT_DIGEST_DOMAIN, executionResultUnsigned),
|
||||
});
|
||||
const resultArtifact = createToolExecutionResultArtifact(
|
||||
{
|
||||
artifactId: 'crash-tool-result-output-artifact',
|
||||
projectId: PROJECT_ID,
|
||||
runId: RUN_ID,
|
||||
stepRunId: STEP_RUN_ID,
|
||||
tool: TOOL,
|
||||
executionResult,
|
||||
keyId: RESULT_KEY_A_ID,
|
||||
key: RESULT_KEY_A,
|
||||
},
|
||||
projectToolDefinitionRegistry(definitionSnapshot),
|
||||
() => Buffer.alloc(12, 45),
|
||||
);
|
||||
const running = await stepRuns.findById(STEP_RUN_ID);
|
||||
assert.ok(running);
|
||||
const succeededMutation = transitionStepRunMutation(
|
||||
running,
|
||||
{
|
||||
expectedVersion: running.version,
|
||||
expectedDigest: running.stepRunDigest,
|
||||
mutationId: 'crash-tool-result-succeeded',
|
||||
to: 'succeeded',
|
||||
atMs: executionResult.completedAtMs,
|
||||
outputRef: resultArtifact.artifactId,
|
||||
},
|
||||
{
|
||||
expectedRunVersion: 2,
|
||||
expectedRunEventSequence: 2,
|
||||
eventId: '51000000-0000-4000-8000-000000000003',
|
||||
dedupeKey: 'crash-tool-result:succeeded',
|
||||
actor: SUBJECT,
|
||||
},
|
||||
);
|
||||
return Object.freeze({
|
||||
catalogRepository,
|
||||
definitionSnapshot,
|
||||
completionCommand: createToolExecutionCompletionCommand({
|
||||
barrier: start.barrier,
|
||||
executionResult,
|
||||
resultArtifact,
|
||||
resultKeyCatalogFence: toolResultKeyCatalogFence(
|
||||
catalog.catalog,
|
||||
requireActiveToolResultKey(catalog.catalog),
|
||||
),
|
||||
stepRunMutation: succeededMutation,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
async function setupScenario({
|
||||
databasePath,
|
||||
statePath,
|
||||
profile,
|
||||
operation,
|
||||
}) {
|
||||
const client = openClient(databasePath, profile);
|
||||
await migrateLocalSqliteDatabase(client);
|
||||
const authority = new LocalSqliteOperationAuthority(client);
|
||||
try {
|
||||
const prepared = await prepareCompletionFixture(
|
||||
client,
|
||||
authority,
|
||||
profile,
|
||||
);
|
||||
const state = {
|
||||
profile,
|
||||
operation,
|
||||
completionCommand: prepared.completionCommand,
|
||||
};
|
||||
if (operation !== 'completion') {
|
||||
const completed =
|
||||
await new LocalSqliteToolExecutionCompletionRepository(
|
||||
authority,
|
||||
).commit(prepared.completionCommand);
|
||||
assert.equal(completed.status, 'created');
|
||||
const rotated = await prepared.catalogRepository.append(
|
||||
createToolResultKeyRotationCommand(
|
||||
await prepared.catalogRepository.findCurrent(),
|
||||
{
|
||||
keyId: RESULT_KEY_B_ID,
|
||||
materialProof: toolResultKeyMaterialProof(
|
||||
RESULT_KEY_B_ID,
|
||||
RESULT_KEY_B,
|
||||
),
|
||||
mutationId: 'crash-result-key-rotate-b',
|
||||
},
|
||||
),
|
||||
);
|
||||
const rekeyCommand = createToolExecutionResultRekeyCommand({
|
||||
artifact: prepared.completionCommand.resultArtifact,
|
||||
binding: toolExecutionResultKeyBinding(
|
||||
prepared.completionCommand,
|
||||
),
|
||||
previousOverlay: null,
|
||||
overlayId: 'crash-tool-result-rekey-overlay',
|
||||
mutationId: 'crash-tool-result-rekey',
|
||||
targetCatalogFence: toolResultKeyCatalogFence(
|
||||
rotated.catalog,
|
||||
requireActiveToolResultKey(rotated.catalog),
|
||||
),
|
||||
targetKey: RESULT_KEY_B,
|
||||
output: OUTPUT,
|
||||
rekeyedAtMs: 1_400,
|
||||
registry: projectToolDefinitionRegistry(
|
||||
prepared.definitionSnapshot,
|
||||
),
|
||||
nonceFactory: () => Buffer.alloc(12, 46),
|
||||
});
|
||||
state.rekeyCommand = rekeyCommand;
|
||||
state.receiptCommand =
|
||||
createToolResultKeyRetirementReceiptCommand({
|
||||
expectedCatalogGeneration: rotated.catalog.generation,
|
||||
expectedCatalogDigest: rotated.catalog.catalogDigest,
|
||||
keyId: RESULT_KEY_A_ID,
|
||||
mutationId: 'crash-tool-result-retirement-receipt',
|
||||
});
|
||||
if (operation !== 'rekey') {
|
||||
const rekeyed =
|
||||
await new LocalSqliteToolResultRekeyRepository(
|
||||
authority,
|
||||
).append(rekeyCommand);
|
||||
assert.equal(rekeyed.status, 'created');
|
||||
}
|
||||
if (operation === 'retire') {
|
||||
const receipt =
|
||||
await new LocalSqliteToolResultRekeyRepository(
|
||||
authority,
|
||||
).create(state.receiptCommand);
|
||||
assert.equal(receipt.status, 'created');
|
||||
state.retireCommand = createToolResultKeyRetirementCommand(
|
||||
rotated.catalog,
|
||||
{
|
||||
keyId: RESULT_KEY_A_ID,
|
||||
retirementReceiptDigest: receipt.receipt.receiptDigest,
|
||||
mutationId: 'crash-result-key-retire-a',
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
fs.writeFileSync(statePath, JSON.stringify(state), {
|
||||
encoding: 'utf8',
|
||||
flag: 'wx',
|
||||
mode: 0o600,
|
||||
});
|
||||
} finally {
|
||||
await authority.close();
|
||||
}
|
||||
}
|
||||
|
||||
function writeCrashMarker(markerPath, pointName) {
|
||||
const file = fs.openSync(markerPath, 'wx', 0o600);
|
||||
try {
|
||||
fs.writeSync(
|
||||
file,
|
||||
JSON.stringify({
|
||||
schema: 'qinglong/sqlite-tool-result-crash-marker@v1',
|
||||
point: pointName,
|
||||
pid: process.pid,
|
||||
}),
|
||||
);
|
||||
fs.fsyncSync(file);
|
||||
} finally {
|
||||
fs.closeSync(file);
|
||||
}
|
||||
}
|
||||
|
||||
function crashClient(client, pointName, markerPath) {
|
||||
const point = CRASH_POINTS[pointName];
|
||||
if (!point) throw new Error(`unknown crash point ${pointName}`);
|
||||
let triggered = false;
|
||||
const crash = () => {
|
||||
if (triggered) return;
|
||||
triggered = true;
|
||||
writeCrashMarker(markerPath, pointName);
|
||||
process.kill(process.pid, 'SIGKILL');
|
||||
throw new Error(`SIGKILL did not terminate ${pointName}`);
|
||||
};
|
||||
const matches = (timing, sql) =>
|
||||
!triggered &&
|
||||
point.timing === timing &&
|
||||
sql.trim().includes(point.sql);
|
||||
return new Proxy(client, {
|
||||
get(target, property) {
|
||||
if (property === 'exec') {
|
||||
return (sql) => {
|
||||
if (matches('beforeExec', sql)) crash();
|
||||
const result = target.exec(sql);
|
||||
if (matches('afterExec', sql)) crash();
|
||||
return result;
|
||||
};
|
||||
}
|
||||
if (property === 'prepare') {
|
||||
return (sql) => {
|
||||
const statement = target.prepare(sql);
|
||||
return new Proxy(statement, {
|
||||
get(statementTarget, statementProperty) {
|
||||
const value = Reflect.get(
|
||||
statementTarget,
|
||||
statementProperty,
|
||||
statementTarget,
|
||||
);
|
||||
if (statementProperty === 'run') {
|
||||
return (...values) => {
|
||||
const result = value.apply(statementTarget, values);
|
||||
if (matches('afterRun', sql)) crash();
|
||||
return result;
|
||||
};
|
||||
}
|
||||
return typeof value === 'function'
|
||||
? value.bind(statementTarget)
|
||||
: value;
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
const value = Reflect.get(target, property, target);
|
||||
return typeof value === 'function' ? value.bind(target) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function runCrashScenario({
|
||||
databasePath,
|
||||
statePath,
|
||||
markerPath,
|
||||
pointName,
|
||||
}) {
|
||||
const point = CRASH_POINTS[pointName];
|
||||
if (!point) throw new Error(`unknown crash point ${pointName}`);
|
||||
const state = JSON.parse(fs.readFileSync(statePath, 'utf8'));
|
||||
if (state.operation !== point.operation) {
|
||||
throw new Error(
|
||||
`crash point ${pointName} does not match ${state.operation}`,
|
||||
);
|
||||
}
|
||||
const client = openClient(databasePath, state.profile);
|
||||
const authority = new LocalSqliteOperationAuthority(
|
||||
crashClient(client, pointName, markerPath),
|
||||
);
|
||||
if (point.operation === 'completion') {
|
||||
await new LocalSqliteToolExecutionCompletionRepository(
|
||||
authority,
|
||||
).commit(state.completionCommand);
|
||||
} else if (point.operation === 'rekey') {
|
||||
await new LocalSqliteToolResultRekeyRepository(authority).append(
|
||||
state.rekeyCommand,
|
||||
);
|
||||
} else if (point.operation === 'receipt') {
|
||||
await new LocalSqliteToolResultRekeyRepository(authority).create(
|
||||
state.receiptCommand,
|
||||
);
|
||||
} else {
|
||||
await new LocalSqliteToolResultKeyCatalogRepository(authority).append(
|
||||
state.retireCommand,
|
||||
);
|
||||
}
|
||||
throw new Error(`crash point ${pointName} was not reached`);
|
||||
}
|
||||
|
||||
function completionFacts(client) {
|
||||
return {
|
||||
...client
|
||||
.prepare(
|
||||
`SELECT
|
||||
step.status AS "stepStatus",
|
||||
step.version AS "stepVersion",
|
||||
run.version AS "runVersion",
|
||||
run.event_sequence AS "runEventSequence",
|
||||
(SELECT count(*) FROM "ToolExecutionCompletions"
|
||||
WHERE start_id = ?) AS "completionCount",
|
||||
(SELECT count(*) FROM "ToolExecutionResultKeyBindings"
|
||||
WHERE start_id = ?) AS "bindingCount",
|
||||
(SELECT count(*) FROM "StepRunMutations"
|
||||
WHERE mutation_id = 'crash-tool-result-succeeded')
|
||||
AS "completionMutationCount",
|
||||
(SELECT count(*) FROM "RunEvents"
|
||||
WHERE id = '51000000-0000-4000-8000-000000000003')
|
||||
AS "completionEventCount"
|
||||
FROM "StepRuns" AS step
|
||||
JOIN "Runs" AS run ON run.id = step.run_id
|
||||
WHERE step.id = ? AND run.id = ?`,
|
||||
)
|
||||
.get(START_ID, START_ID, STEP_RUN_ID, RUN_ID),
|
||||
};
|
||||
}
|
||||
|
||||
function rekeyFacts(client) {
|
||||
return {
|
||||
...client
|
||||
.prepare(
|
||||
`SELECT
|
||||
(SELECT count(*) FROM "ToolExecutionResultRekeyOverlays"
|
||||
WHERE overlay_id = 'crash-tool-result-rekey-overlay')
|
||||
AS "overlayCount",
|
||||
(SELECT count(*) FROM "ToolExecutionResultRekeyHeads"
|
||||
WHERE artifact_id = 'crash-tool-result-output-artifact')
|
||||
AS "headCount"`,
|
||||
)
|
||||
.get(),
|
||||
};
|
||||
}
|
||||
|
||||
function receiptCount(client) {
|
||||
return client
|
||||
.prepare(
|
||||
`SELECT count(*) AS count
|
||||
FROM "ToolResultKeyRetirementReceipts"
|
||||
WHERE mutation_id = 'crash-tool-result-retirement-receipt'`,
|
||||
)
|
||||
.get().count;
|
||||
}
|
||||
|
||||
async function verifyScenario({
|
||||
databasePath,
|
||||
statePath,
|
||||
pointName,
|
||||
}) {
|
||||
const point = CRASH_POINTS[pointName];
|
||||
if (!point) throw new Error(`unknown crash point ${pointName}`);
|
||||
const state = JSON.parse(fs.readFileSync(statePath, 'utf8'));
|
||||
const client = openClient(databasePath, state.profile);
|
||||
const authority = new LocalSqliteOperationAuthority(client);
|
||||
try {
|
||||
assert.equal(
|
||||
client.prepare('PRAGMA integrity_check').get().integrity_check,
|
||||
'ok',
|
||||
);
|
||||
assert.equal(
|
||||
client.prepare('PRAGMA journal_mode').get().journal_mode,
|
||||
state.profile === 'edge' ? 'delete' : 'wal',
|
||||
);
|
||||
let replayStatus;
|
||||
if (point.operation === 'completion') {
|
||||
assert.deepEqual(
|
||||
completionFacts(client),
|
||||
point.durable
|
||||
? {
|
||||
stepStatus: 'succeeded',
|
||||
stepVersion: 3,
|
||||
runVersion: 3,
|
||||
runEventSequence: 3,
|
||||
completionCount: 1,
|
||||
bindingCount: 1,
|
||||
completionMutationCount: 1,
|
||||
completionEventCount: 1,
|
||||
}
|
||||
: {
|
||||
stepStatus: 'running',
|
||||
stepVersion: 2,
|
||||
runVersion: 2,
|
||||
runEventSequence: 2,
|
||||
completionCount: 0,
|
||||
bindingCount: 0,
|
||||
completionMutationCount: 0,
|
||||
completionEventCount: 0,
|
||||
},
|
||||
);
|
||||
const repository =
|
||||
new LocalSqliteToolExecutionCompletionRepository(authority);
|
||||
replayStatus = (
|
||||
await repository.commit(state.completionCommand)
|
||||
).status;
|
||||
assert.equal(
|
||||
(
|
||||
await repository.commit(state.completionCommand)
|
||||
).status,
|
||||
'existing',
|
||||
);
|
||||
assert.deepEqual(completionFacts(client), {
|
||||
stepStatus: 'succeeded',
|
||||
stepVersion: 3,
|
||||
runVersion: 3,
|
||||
runEventSequence: 3,
|
||||
completionCount: 1,
|
||||
bindingCount: 1,
|
||||
completionMutationCount: 1,
|
||||
completionEventCount: 1,
|
||||
});
|
||||
} else if (point.operation === 'rekey') {
|
||||
assert.deepEqual(
|
||||
rekeyFacts(client),
|
||||
point.durable
|
||||
? { overlayCount: 1, headCount: 1 }
|
||||
: { overlayCount: 0, headCount: 0 },
|
||||
);
|
||||
const repository =
|
||||
new LocalSqliteToolResultRekeyRepository(authority);
|
||||
replayStatus = (
|
||||
await repository.append(state.rekeyCommand)
|
||||
).status;
|
||||
assert.equal(
|
||||
(await repository.append(state.rekeyCommand)).status,
|
||||
'existing',
|
||||
);
|
||||
assert.deepEqual(rekeyFacts(client), {
|
||||
overlayCount: 1,
|
||||
headCount: 1,
|
||||
});
|
||||
} else if (point.operation === 'receipt') {
|
||||
assert.equal(receiptCount(client), point.durable ? 1 : 0);
|
||||
const repository =
|
||||
new LocalSqliteToolResultRekeyRepository(authority);
|
||||
replayStatus = (
|
||||
await repository.create(state.receiptCommand)
|
||||
).status;
|
||||
assert.equal(
|
||||
(await repository.create(state.receiptCommand)).status,
|
||||
'existing',
|
||||
);
|
||||
assert.equal(receiptCount(client), 1);
|
||||
} else {
|
||||
const catalogRepository =
|
||||
new LocalSqliteToolResultKeyCatalogRepository(authority);
|
||||
const before = await catalogRepository.findCurrent();
|
||||
assert.ok(before);
|
||||
assert.equal(before.generation, point.durable ? 3 : 2);
|
||||
assert.equal(
|
||||
before.keys.find((entry) => entry.keyId === RESULT_KEY_A_ID)
|
||||
.state,
|
||||
point.durable ? 'retired' : 'decrypt_only',
|
||||
);
|
||||
replayStatus = (
|
||||
await catalogRepository.append(state.retireCommand)
|
||||
).status;
|
||||
assert.equal(
|
||||
(
|
||||
await catalogRepository.append(state.retireCommand)
|
||||
).status,
|
||||
'existing',
|
||||
);
|
||||
const after = await catalogRepository.findCurrent();
|
||||
assert.ok(after);
|
||||
assert.equal(after.generation, 3);
|
||||
assert.equal(
|
||||
after.keys.find((entry) => entry.keyId === RESULT_KEY_A_ID)
|
||||
.state,
|
||||
'retired',
|
||||
);
|
||||
}
|
||||
assert.equal(replayStatus, point.durable ? 'existing' : 'created');
|
||||
return Object.freeze({
|
||||
point: pointName,
|
||||
operation: point.operation,
|
||||
crashBeforeCommit: !point.durable,
|
||||
durableAfterCrash: point.durable,
|
||||
replayStatus,
|
||||
integrityCheck: 'ok',
|
||||
journalMode: state.profile === 'edge' ? 'delete' : 'wal',
|
||||
});
|
||||
} finally {
|
||||
await authority.close();
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
const [, , action, databasePath, statePath, markerPath, pointName] =
|
||||
process.argv;
|
||||
if (action !== 'crash') {
|
||||
throw new Error('fixture action must be crash');
|
||||
}
|
||||
runCrashScenario({
|
||||
databasePath,
|
||||
statePath,
|
||||
markerPath,
|
||||
pointName,
|
||||
}).catch((error) => {
|
||||
process.stderr.write(
|
||||
`${error instanceof Error ? error.stack : String(error)}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
CRASH_POINTS,
|
||||
setupScenario,
|
||||
verifyScenario,
|
||||
};
|
||||
Reference in New Issue
Block a user