mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,686 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const { QueryTypes, Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const {
|
||||
projectPolicyMigration,
|
||||
} = require('../../back/migrations/0017-project-policy');
|
||||
const {
|
||||
approvalRequestMigration,
|
||||
} = require('../../back/migrations/0020-approval-requests');
|
||||
const {
|
||||
APPROVED_ACTION_DISPATCH_EXECUTION_TABLE,
|
||||
approvedActionDispatchExecutionMigration,
|
||||
} = require('../../back/migrations/0021-approved-action-dispatch-executions');
|
||||
const {
|
||||
APPROVED_ACTION_RECOVERY_CONTROL_TABLE,
|
||||
APPROVED_ACTION_RECOVERY_DUE_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_LEASE_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_PROJECT_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_RESOLUTION_MUTATION_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_RESOLUTION_PROJECT_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE,
|
||||
approvedActionRecoveryMigration,
|
||||
} = require('../../back/migrations/0022-approved-action-recovery');
|
||||
const {
|
||||
approvedActionRecoveryAuthorizationMigration,
|
||||
} = require('../../back/migrations/0024-approved-action-recovery-authorization');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeApprovedActionDispatchRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/approvedActionDispatchRepository');
|
||||
const {
|
||||
LegacySequelizeApprovedActionRecoveryRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/approvedActionRecoveryRepository');
|
||||
const {
|
||||
LegacySequelizeApprovalRequestRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/approvalRequestRepository');
|
||||
const {
|
||||
LegacySequelizeProjectPolicyRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/projectPolicyRepository');
|
||||
const {
|
||||
ApprovalRequestService,
|
||||
} = require('../../back/runtime/application/approvalRequestService');
|
||||
const {
|
||||
ProjectPolicyEngine,
|
||||
} = require('../../back/runtime/application/projectPolicyEngine');
|
||||
const {
|
||||
ApprovedActionDispatchRepositoryError,
|
||||
} = require('../../back/runtime/domain/approvedActionDispatchExecution');
|
||||
const {
|
||||
ApprovedActionRecoveryFenceRejectedError,
|
||||
} = require('../../back/runtime/domain/approvedActionRecovery');
|
||||
const {
|
||||
createApprovedActionRecoveryAuthorizationFact,
|
||||
} = require('../../back/runtime/domain/approvedActionRecoveryAuthorization');
|
||||
|
||||
const PROJECT_ID = 'default';
|
||||
const AGENT = Object.freeze({ type: 'agent', id: 'agent-1' });
|
||||
const OWNER = Object.freeze({ type: 'user', id: 'owner-1' });
|
||||
const SYSTEM = Object.freeze({ type: 'system', id: 'approval-dispatcher' });
|
||||
const BASE_TIME = 100_000;
|
||||
|
||||
function action(name, digestCharacter = 'a') {
|
||||
return {
|
||||
permission: 'tool.call:filesystem.write',
|
||||
actionType: 'tool_call',
|
||||
actionRef: `planned-${name}`,
|
||||
actionDigest: digestCharacter.repeat(64),
|
||||
previewDigest: 'f'.repeat(64),
|
||||
};
|
||||
}
|
||||
|
||||
async function migrate(database, migrations) {
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations,
|
||||
logger: { info() {} },
|
||||
});
|
||||
}
|
||||
|
||||
async function bind(policyRepository, subject, role, mutationId) {
|
||||
await policyRepository.append({
|
||||
expectedCurrentVersion: 0,
|
||||
binding: {
|
||||
projectId: PROJECT_ID,
|
||||
subject,
|
||||
version: 1,
|
||||
state: 'active',
|
||||
role,
|
||||
mutationId,
|
||||
changedBy: OWNER,
|
||||
createdAtMs: BASE_TIME - 100,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function setup(t, storage = ':memory:', includeRecovery = true) {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
await migrate(database, [
|
||||
projectPolicyMigration,
|
||||
approvalRequestMigration,
|
||||
approvedActionDispatchExecutionMigration,
|
||||
...(includeRecovery
|
||||
? [
|
||||
approvedActionRecoveryMigration,
|
||||
approvedActionRecoveryAuthorizationMigration,
|
||||
]
|
||||
: []),
|
||||
]);
|
||||
const policyRepository = new LegacySequelizeProjectPolicyRepository(database);
|
||||
await bind(policyRepository, OWNER, 'owner', 'bind-owner');
|
||||
await bind(policyRepository, AGENT, 'operator', 'bind-agent');
|
||||
const approvalRepository = new LegacySequelizeApprovalRequestRepository(
|
||||
database,
|
||||
);
|
||||
return {
|
||||
database,
|
||||
approvalService: new ApprovalRequestService(
|
||||
approvalRepository,
|
||||
new ProjectPolicyEngine(policyRepository),
|
||||
),
|
||||
actionRepository: new LegacySequelizeApprovedActionDispatchRepository(
|
||||
database,
|
||||
),
|
||||
recoveryRepository: includeRecovery
|
||||
? new LegacySequelizeApprovedActionRecoveryRepository(database)
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
async function prepareDispatch(service, name, offset = 0, digest = 'a') {
|
||||
const requestedAtMs = BASE_TIME + offset;
|
||||
const binding = action(name, digest);
|
||||
await service.create({
|
||||
id: `approval-${name}`,
|
||||
projectId: PROJECT_ID,
|
||||
action: binding,
|
||||
risk: 'high',
|
||||
requestedBy: AGENT,
|
||||
requestedAtMs,
|
||||
expiresAtMs: requestedAtMs + 60_000,
|
||||
});
|
||||
await service.decide({
|
||||
requestId: `approval-${name}`,
|
||||
expectedVersion: 1,
|
||||
decisionId: `decision-${name}`,
|
||||
decision: 'approved',
|
||||
reasonCode: 'reviewed_action',
|
||||
decidedBy: OWNER,
|
||||
decidedAtMs: requestedAtMs + 10,
|
||||
});
|
||||
return (
|
||||
await service.consume({
|
||||
requestId: `approval-${name}`,
|
||||
expectedVersion: 2,
|
||||
consumptionId: `consumption-${name}`,
|
||||
dispatchId: `dispatch-${name}`,
|
||||
action: binding,
|
||||
requestedBy: AGENT,
|
||||
consumedBy: SYSTEM,
|
||||
consumedAtMs: requestedAtMs + 20,
|
||||
})
|
||||
).dispatch;
|
||||
}
|
||||
|
||||
async function startAction(actionRepository, dispatch, options = {}) {
|
||||
const nowMs = options.nowMs ?? BASE_TIME + 1_000;
|
||||
const leaseDurationMs = options.leaseDurationMs ?? 100;
|
||||
const owner = options.owner ?? 'dispatcher-1';
|
||||
const leaseToken = options.leaseToken ?? 'execution-lease-1';
|
||||
const claimed = await actionRepository.claim({
|
||||
dispatchId: dispatch.id,
|
||||
owner,
|
||||
leaseToken,
|
||||
nowMs,
|
||||
leaseDurationMs,
|
||||
});
|
||||
assert.equal(claimed.status, 'claimed');
|
||||
const started = await actionRepository.start({
|
||||
dispatchId: dispatch.id,
|
||||
approvalRequestId: dispatch.approvalRequestId,
|
||||
actionDigest: dispatch.action.actionDigest,
|
||||
owner,
|
||||
leaseToken,
|
||||
expectedVersion: claimed.snapshot.execution.version,
|
||||
startedAtMs: nowMs + 10,
|
||||
});
|
||||
return {
|
||||
claimed,
|
||||
started,
|
||||
owner,
|
||||
leaseToken,
|
||||
leaseExpiresAtMs: nowMs + leaseDurationMs,
|
||||
};
|
||||
}
|
||||
|
||||
async function claimRecovery(recoveryRepository, dispatchId, overrides = {}) {
|
||||
return recoveryRepository.claim({
|
||||
dispatchId,
|
||||
owner: 'resolver-1',
|
||||
leaseToken: 'recovery-lease-1',
|
||||
nowMs: BASE_TIME + 1_100,
|
||||
leaseDurationMs: 1_000,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
test('0022 backfills executing controls and owns bounded recovery indexes', async (t) => {
|
||||
const context = await setup(t, ':memory:', false);
|
||||
const dispatch = await prepareDispatch(context.approvalService, 'backfill');
|
||||
await context.database.query(
|
||||
`UPDATE "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}"
|
||||
SET status = 'executing', version = 2, attempt_count = 1,
|
||||
eligible_at_ms = NULL, lease_owner = 'dispatcher-1',
|
||||
lease_token = 'execution-lease-1',
|
||||
lease_expires_at_ms = :leaseExpiresAtMs,
|
||||
started_at_ms = :startedAtMs, updated_at_ms = :startedAtMs
|
||||
WHERE dispatch_id = :dispatchId`,
|
||||
{
|
||||
replacements: {
|
||||
dispatchId: dispatch.id,
|
||||
startedAtMs: BASE_TIME + 1_010,
|
||||
leaseExpiresAtMs: BASE_TIME + 1_100,
|
||||
},
|
||||
},
|
||||
);
|
||||
await migrate(context.database, [approvedActionRecoveryMigration]);
|
||||
const rows = await context.database.query(
|
||||
`SELECT * FROM "${APPROVED_ACTION_RECOVERY_CONTROL_TABLE}"`,
|
||||
{ type: QueryTypes.SELECT },
|
||||
);
|
||||
assert.equal(rows.length, 1);
|
||||
assert.equal(rows[0].dispatch_id, dispatch.id);
|
||||
assert.equal(rows[0].execution_version, 2);
|
||||
assert.equal(rows[0].status, 'armed');
|
||||
assert.equal(rows[0].next_scan_at_ms, BASE_TIME + 1_100);
|
||||
const controlIndexes = new Set(
|
||||
(
|
||||
await context.database
|
||||
.getQueryInterface()
|
||||
.showIndex(APPROVED_ACTION_RECOVERY_CONTROL_TABLE)
|
||||
).map((index) => index.name),
|
||||
);
|
||||
for (const name of [
|
||||
APPROVED_ACTION_RECOVERY_DUE_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_PROJECT_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_LEASE_INDEX,
|
||||
]) {
|
||||
assert.ok(controlIndexes.has(name));
|
||||
}
|
||||
const resolutionIndexes = new Set(
|
||||
(
|
||||
await context.database
|
||||
.getQueryInterface()
|
||||
.showIndex(APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE)
|
||||
).map((index) => index.name),
|
||||
);
|
||||
assert.ok(
|
||||
resolutionIndexes.has(APPROVED_ACTION_RECOVERY_RESOLUTION_MUTATION_INDEX),
|
||||
);
|
||||
assert.ok(
|
||||
resolutionIndexes.has(APPROVED_ACTION_RECOVERY_RESOLUTION_PROJECT_INDEX),
|
||||
);
|
||||
});
|
||||
|
||||
test('start barrier atomically arms recovery and rolls back when control insert fails', async (t) => {
|
||||
const context = await setup(t);
|
||||
const dispatch = await prepareDispatch(
|
||||
context.approvalService,
|
||||
'start-atomic',
|
||||
);
|
||||
const claimed = await context.actionRepository.claim({
|
||||
dispatchId: dispatch.id,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'execution-lease-1',
|
||||
nowMs: BASE_TIME + 1_000,
|
||||
leaseDurationMs: 100,
|
||||
});
|
||||
await context.database.query(
|
||||
`CREATE TRIGGER fail_recovery_control_insert
|
||||
BEFORE INSERT ON "${APPROVED_ACTION_RECOVERY_CONTROL_TABLE}"
|
||||
BEGIN SELECT RAISE(ABORT, 'forced recovery insert failure'); END`,
|
||||
);
|
||||
await assert.rejects(
|
||||
context.actionRepository.start({
|
||||
dispatchId: dispatch.id,
|
||||
approvalRequestId: dispatch.approvalRequestId,
|
||||
actionDigest: dispatch.action.actionDigest,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'execution-lease-1',
|
||||
expectedVersion: claimed.snapshot.execution.version,
|
||||
startedAtMs: BASE_TIME + 1_010,
|
||||
}),
|
||||
ApprovedActionDispatchRepositoryError,
|
||||
);
|
||||
assert.equal(
|
||||
(await context.actionRepository.findById(dispatch.id)).execution.status,
|
||||
'leased',
|
||||
);
|
||||
assert.equal(await context.recoveryRepository.findById(dispatch.id), null);
|
||||
});
|
||||
|
||||
test('normal completion closes the recovery control without creating a recovery resolution', async (t) => {
|
||||
const context = await setup(t);
|
||||
const dispatch = await prepareDispatch(context.approvalService, 'complete');
|
||||
const execution = await startAction(context.actionRepository, dispatch);
|
||||
const armed = await context.recoveryRepository.findById(dispatch.id);
|
||||
assert.equal(armed.recovery.status, 'armed');
|
||||
assert.equal(
|
||||
armed.recovery.executionVersion,
|
||||
execution.started.execution.version,
|
||||
);
|
||||
assert.equal(armed.recovery.nextScanAtMs, execution.leaseExpiresAtMs);
|
||||
const completed = await context.actionRepository.complete({
|
||||
dispatchId: dispatch.id,
|
||||
owner: execution.owner,
|
||||
leaseToken: execution.leaseToken,
|
||||
expectedVersion: execution.started.execution.version,
|
||||
resultMutationId: 'normal-completion-1',
|
||||
outcome: 'succeeded',
|
||||
resultCode: 'ok',
|
||||
completedAtMs: BASE_TIME + 1_120,
|
||||
});
|
||||
assert.equal(completed.execution.status, 'succeeded');
|
||||
const closed = await context.recoveryRepository.findById(dispatch.id);
|
||||
assert.equal(closed.recovery.status, 'resolved');
|
||||
assert.equal(closed.recovery.resolutionMutationId, 'normal-completion-1');
|
||||
assert.equal(closed.resolution, null);
|
||||
assert.equal(
|
||||
(await claimRecovery(context.recoveryRepository, dispatch.id)).status,
|
||||
'resolved',
|
||||
);
|
||||
});
|
||||
|
||||
test('a late execution renew re-arms recovery and fences the stale resolver lease', async (t) => {
|
||||
const context = await setup(t);
|
||||
const dispatch = await prepareDispatch(context.approvalService, 'renew');
|
||||
const execution = await startAction(context.actionRepository, dispatch);
|
||||
const claimed = await claimRecovery(context.recoveryRepository, dispatch.id);
|
||||
assert.equal(claimed.status, 'claimed');
|
||||
const renewed = await context.actionRepository.renew({
|
||||
dispatchId: dispatch.id,
|
||||
owner: execution.owner,
|
||||
leaseToken: execution.leaseToken,
|
||||
expectedVersion: execution.started.execution.version,
|
||||
nowMs: BASE_TIME + 1_150,
|
||||
leaseDurationMs: 1_000,
|
||||
});
|
||||
assert.equal(
|
||||
renewed.execution.version,
|
||||
execution.started.execution.version + 1,
|
||||
);
|
||||
const rearmed = await context.recoveryRepository.findById(dispatch.id);
|
||||
assert.equal(rearmed.recovery.status, 'armed');
|
||||
assert.equal(rearmed.recovery.executionVersion, renewed.execution.version);
|
||||
assert.equal(rearmed.recovery.nextScanAtMs, BASE_TIME + 2_150);
|
||||
assert.equal(rearmed.recovery.leaseOwner, null);
|
||||
await assert.rejects(
|
||||
context.recoveryRepository.recordFinding({
|
||||
dispatchId: dispatch.id,
|
||||
expectedExecutionVersion: claimed.snapshot.action.execution.version,
|
||||
expectedRecoveryVersion: claimed.snapshot.recovery.version,
|
||||
owner: 'resolver-1',
|
||||
leaseToken: 'recovery-lease-1',
|
||||
findingMutationId: 'stale-finding-1',
|
||||
finding: 'missing',
|
||||
resultCode: 'receipt_missing',
|
||||
observedAtMs: BASE_TIME + 1_160,
|
||||
retryAtMs: BASE_TIME + 1_200,
|
||||
}),
|
||||
ApprovedActionRecoveryFenceRejectedError,
|
||||
);
|
||||
});
|
||||
|
||||
test('lists a bounded stable recovery page and excludes live executions', async (t) => {
|
||||
const context = await setup(t);
|
||||
const firstDispatch = await prepareDispatch(
|
||||
context.approvalService,
|
||||
'due-a',
|
||||
0,
|
||||
'a',
|
||||
);
|
||||
const secondDispatch = await prepareDispatch(
|
||||
context.approvalService,
|
||||
'due-b',
|
||||
10,
|
||||
'b',
|
||||
);
|
||||
const liveDispatch = await prepareDispatch(
|
||||
context.approvalService,
|
||||
'live',
|
||||
20,
|
||||
'c',
|
||||
);
|
||||
await startAction(context.actionRepository, firstDispatch);
|
||||
await startAction(context.actionRepository, secondDispatch);
|
||||
await startAction(context.actionRepository, liveDispatch, {
|
||||
leaseDurationMs: 1_000,
|
||||
});
|
||||
const first = await context.recoveryRepository.listDue({
|
||||
nowMs: BASE_TIME + 1_100,
|
||||
limit: 1,
|
||||
});
|
||||
assert.deepEqual(
|
||||
first.recoveries.map((entry) => entry.action.dispatch.id),
|
||||
['dispatch-due-a'],
|
||||
);
|
||||
assert.equal(first.truncated, true);
|
||||
const second = await context.recoveryRepository.listDue({
|
||||
nowMs: BASE_TIME + 1_100,
|
||||
limit: 1,
|
||||
cursor: first.nextCursor,
|
||||
});
|
||||
assert.deepEqual(
|
||||
second.recoveries.map((entry) => entry.action.dispatch.id),
|
||||
['dispatch-due-b'],
|
||||
);
|
||||
assert.equal(second.truncated, false);
|
||||
});
|
||||
|
||||
test('two SQLite resolvers claim once and only an expired recovery lease is taken over', async (t) => {
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-recovery-'));
|
||||
t.after(() => fs.rm(directory, { recursive: true, force: true }));
|
||||
const storage = path.join(directory, 'database.sqlite');
|
||||
const first = await setup(t, storage);
|
||||
const dispatch = await prepareDispatch(first.approvalService, 'claim-race');
|
||||
await startAction(first.actionRepository, dispatch);
|
||||
const secondDatabase = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => secondDatabase.close());
|
||||
const secondRepository = new LegacySequelizeApprovedActionRecoveryRepository(
|
||||
secondDatabase,
|
||||
);
|
||||
const results = await Promise.all([
|
||||
claimRecovery(first.recoveryRepository, dispatch.id, {
|
||||
leaseDurationMs: 100,
|
||||
}),
|
||||
claimRecovery(secondRepository, dispatch.id, {
|
||||
owner: 'resolver-2',
|
||||
leaseToken: 'recovery-lease-2',
|
||||
leaseDurationMs: 100,
|
||||
}),
|
||||
]);
|
||||
assert.equal(
|
||||
results.filter((result) => result.status === 'claimed').length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
results.filter((result) => result.status === 'leased').length,
|
||||
1,
|
||||
);
|
||||
const winner = results.find((result) => result.status === 'claimed');
|
||||
assert.equal(
|
||||
(
|
||||
await claimRecovery(first.recoveryRepository, dispatch.id, {
|
||||
owner: winner.snapshot.recovery.leaseOwner,
|
||||
leaseToken: winner.snapshot.recovery.leaseToken,
|
||||
leaseDurationMs: 100,
|
||||
})
|
||||
).status,
|
||||
'claimed',
|
||||
);
|
||||
const takeover = await claimRecovery(secondRepository, dispatch.id, {
|
||||
owner: 'resolver-3',
|
||||
leaseToken: 'recovery-lease-3',
|
||||
nowMs: BASE_TIME + 1_200,
|
||||
leaseDurationMs: 100,
|
||||
});
|
||||
assert.equal(takeover.status, 'claimed');
|
||||
assert.equal(takeover.snapshot.recovery.leaseOwner, 'resolver-3');
|
||||
});
|
||||
|
||||
test('recovery finding replay is exact and unsupported evidence becomes manual-only', async (t) => {
|
||||
const context = await setup(t);
|
||||
const dispatch = await prepareDispatch(context.approvalService, 'finding');
|
||||
await startAction(context.actionRepository, dispatch);
|
||||
const claimed = await claimRecovery(context.recoveryRepository, dispatch.id);
|
||||
assert.equal(claimed.status, 'claimed');
|
||||
const finding = {
|
||||
dispatchId: dispatch.id,
|
||||
expectedExecutionVersion: claimed.snapshot.action.execution.version,
|
||||
expectedRecoveryVersion: claimed.snapshot.recovery.version,
|
||||
owner: 'resolver-1',
|
||||
leaseToken: 'recovery-lease-1',
|
||||
findingMutationId: 'finding-missing-1',
|
||||
finding: 'missing',
|
||||
resultCode: 'receipt_missing',
|
||||
observedAtMs: BASE_TIME + 1_110,
|
||||
retryAtMs: BASE_TIME + 1_200,
|
||||
};
|
||||
const deferred = await context.recoveryRepository.recordFinding(finding);
|
||||
assert.equal(deferred.recovery.status, 'armed');
|
||||
assert.equal(deferred.recovery.findingCount, 1);
|
||||
assert.deepEqual(
|
||||
await context.recoveryRepository.recordFinding(finding),
|
||||
deferred,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await claimRecovery(context.recoveryRepository, dispatch.id, {
|
||||
nowMs: BASE_TIME + 1_199,
|
||||
})
|
||||
).status,
|
||||
'not_due',
|
||||
);
|
||||
const second = await claimRecovery(context.recoveryRepository, dispatch.id, {
|
||||
owner: 'resolver-2',
|
||||
leaseToken: 'recovery-lease-2',
|
||||
nowMs: BASE_TIME + 1_200,
|
||||
});
|
||||
assert.equal(second.status, 'claimed');
|
||||
const manual = await context.recoveryRepository.recordFinding({
|
||||
dispatchId: dispatch.id,
|
||||
expectedExecutionVersion: second.snapshot.action.execution.version,
|
||||
expectedRecoveryVersion: second.snapshot.recovery.version,
|
||||
owner: 'resolver-2',
|
||||
leaseToken: 'recovery-lease-2',
|
||||
findingMutationId: 'finding-unsupported-1',
|
||||
finding: 'unsupported',
|
||||
resultCode: 'automatic_recovery_unsupported',
|
||||
observedAtMs: BASE_TIME + 1_210,
|
||||
});
|
||||
assert.equal(manual.recovery.status, 'manual_required');
|
||||
assert.equal(manual.recovery.nextScanAtMs, null);
|
||||
assert.equal(
|
||||
(
|
||||
await context.recoveryRepository.listDue({
|
||||
nowMs: BASE_TIME + 10_000,
|
||||
limit: 64,
|
||||
})
|
||||
).recoveries.length,
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
test('verified evidence resolves atomically and exact mutation replay returns the same resolution', async (t) => {
|
||||
const context = await setup(t);
|
||||
const dispatch = await prepareDispatch(context.approvalService, 'verified');
|
||||
await startAction(context.actionRepository, dispatch);
|
||||
const claimed = await claimRecovery(context.recoveryRepository, dispatch.id);
|
||||
const command = {
|
||||
dispatchId: dispatch.id,
|
||||
expectedExecutionVersion: claimed.snapshot.action.execution.version,
|
||||
expectedRecoveryVersion: claimed.snapshot.recovery.version,
|
||||
owner: 'resolver-1',
|
||||
leaseToken: 'recovery-lease-1',
|
||||
mutationId: 'resolution-verified-1',
|
||||
source: 'automatic_evidence',
|
||||
decision: 'confirm_succeeded',
|
||||
evidenceDigest: 'e'.repeat(64),
|
||||
reasonCode: 'provider_receipt_verified',
|
||||
resolvedAtMs: BASE_TIME + 1_120,
|
||||
};
|
||||
const resolved = await context.recoveryRepository.resolve(command);
|
||||
assert.equal(resolved.status, 'resolved');
|
||||
assert.equal(resolved.snapshot.action.execution.status, 'succeeded');
|
||||
assert.equal(resolved.snapshot.recovery.status, 'resolved');
|
||||
assert.equal(resolved.snapshot.resolution.source, 'automatic_evidence');
|
||||
assert.equal(resolved.snapshot.resolution.evidenceDigest, 'e'.repeat(64));
|
||||
assert.deepEqual(await context.recoveryRepository.resolve(command), resolved);
|
||||
});
|
||||
|
||||
test('manual resolution cannot preempt a live execution and never resets it for retry', async (t) => {
|
||||
const context = await setup(t);
|
||||
const dispatch = await prepareDispatch(context.approvalService, 'manual');
|
||||
const execution = await startAction(context.actionRepository, dispatch, {
|
||||
leaseDurationMs: 500,
|
||||
});
|
||||
const armed = await context.recoveryRepository.findById(dispatch.id);
|
||||
const commandAt = (resolvedAtMs) => {
|
||||
const command = {
|
||||
dispatchId: dispatch.id,
|
||||
expectedExecutionVersion: armed.action.execution.version,
|
||||
expectedRecoveryVersion: armed.recovery.version,
|
||||
mutationId: 'resolution-manual-1',
|
||||
source: 'human',
|
||||
decision: 'abandon_unknown',
|
||||
reasonCode: 'operator_abandoned_unknown',
|
||||
resolvedBy: OWNER,
|
||||
resolvedAtMs,
|
||||
};
|
||||
return {
|
||||
...command,
|
||||
authorizationFact: createApprovedActionRecoveryAuthorizationFact({
|
||||
dispatchId: dispatch.id,
|
||||
projectId: PROJECT_ID,
|
||||
mutationId: command.mutationId,
|
||||
resolvedBy: OWNER,
|
||||
authenticationId: 'mfa-session-1',
|
||||
assurance: 'multi_factor',
|
||||
authenticatedAtMs: resolvedAtMs - 100,
|
||||
projectVersion: 1,
|
||||
bindingVersion: 1,
|
||||
authorizedAtMs: resolvedAtMs,
|
||||
}),
|
||||
};
|
||||
};
|
||||
await assert.rejects(
|
||||
context.recoveryRepository.resolve(
|
||||
commandAt(execution.leaseExpiresAtMs - 1),
|
||||
),
|
||||
ApprovedActionRecoveryFenceRejectedError,
|
||||
);
|
||||
const resolved = await context.recoveryRepository.resolve(
|
||||
commandAt(execution.leaseExpiresAtMs),
|
||||
);
|
||||
assert.equal(resolved.snapshot.action.execution.status, 'blocked');
|
||||
assert.equal(resolved.snapshot.resolution.decision, 'abandon_unknown');
|
||||
assert.equal(resolved.snapshot.resolution.resolvedBy.type, 'user');
|
||||
assert.equal(resolved.snapshot.action.execution.eligibleAtMs, null);
|
||||
});
|
||||
|
||||
test('late completion and recovery resolution have one durable terminal winner', async (t) => {
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-recovery-'));
|
||||
t.after(() => fs.rm(directory, { recursive: true, force: true }));
|
||||
const storage = path.join(directory, 'database.sqlite');
|
||||
const first = await setup(t, storage);
|
||||
const dispatch = await prepareDispatch(first.approvalService, 'race');
|
||||
const execution = await startAction(first.actionRepository, dispatch);
|
||||
const secondDatabase = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => secondDatabase.close());
|
||||
const secondRecovery = new LegacySequelizeApprovedActionRecoveryRepository(
|
||||
secondDatabase,
|
||||
);
|
||||
const claimed = await claimRecovery(secondRecovery, dispatch.id);
|
||||
const recoveryCommand = {
|
||||
dispatchId: dispatch.id,
|
||||
expectedExecutionVersion: claimed.snapshot.action.execution.version,
|
||||
expectedRecoveryVersion: claimed.snapshot.recovery.version,
|
||||
owner: 'resolver-1',
|
||||
leaseToken: 'recovery-lease-1',
|
||||
mutationId: 'resolution-race-1',
|
||||
source: 'automatic_evidence',
|
||||
decision: 'confirm_failed',
|
||||
evidenceDigest: 'd'.repeat(64),
|
||||
reasonCode: 'provider_failed',
|
||||
resolvedAtMs: BASE_TIME + 1_120,
|
||||
};
|
||||
await Promise.allSettled([
|
||||
first.actionRepository.complete({
|
||||
dispatchId: dispatch.id,
|
||||
owner: execution.owner,
|
||||
leaseToken: execution.leaseToken,
|
||||
expectedVersion: execution.started.execution.version,
|
||||
resultMutationId: 'normal-race-1',
|
||||
outcome: 'succeeded',
|
||||
resultCode: 'ok',
|
||||
completedAtMs: BASE_TIME + 1_120,
|
||||
}),
|
||||
secondRecovery.resolve(recoveryCommand),
|
||||
]);
|
||||
const final = await secondRecovery.findById(dispatch.id);
|
||||
assert.ok(['succeeded', 'failed'].includes(final.action.execution.status));
|
||||
assert.equal(final.recovery.status, 'resolved');
|
||||
assert.equal(
|
||||
final.recovery.resolutionMutationId,
|
||||
final.action.execution.resultMutationId,
|
||||
);
|
||||
if (final.resolution) {
|
||||
assert.equal(final.resolution.mutationId, 'resolution-race-1');
|
||||
assert.equal(final.action.execution.status, 'failed');
|
||||
} else {
|
||||
assert.equal(final.action.execution.resultMutationId, 'normal-race-1');
|
||||
assert.equal(final.action.execution.status, 'succeeded');
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user