mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 01:32:44 +08:00
687 lines
24 KiB
JavaScript
687 lines
24 KiB
JavaScript
require('ts-node/register/transpile-only');
|
|
|
|
const assert = require('node:assert/strict');
|
|
const fs = require('node:fs/promises');
|
|
const os = require('node:os');
|
|
const path = require('node:path');
|
|
const { test } = require('node:test');
|
|
const { QueryTypes, Sequelize } = require('sequelize');
|
|
const {
|
|
defineSchemaMigrationModel,
|
|
} = require('../../back/data/schemaMigration');
|
|
const {
|
|
projectPolicyMigration,
|
|
} = require('../../back/migrations/0017-project-policy');
|
|
const {
|
|
approvalRequestMigration,
|
|
} = require('../../back/migrations/0020-approval-requests');
|
|
const {
|
|
APPROVED_ACTION_DISPATCH_EXECUTION_TABLE,
|
|
approvedActionDispatchExecutionMigration,
|
|
} = require('../../back/migrations/0021-approved-action-dispatch-executions');
|
|
const {
|
|
APPROVED_ACTION_RECOVERY_CONTROL_TABLE,
|
|
APPROVED_ACTION_RECOVERY_DUE_INDEX,
|
|
APPROVED_ACTION_RECOVERY_LEASE_INDEX,
|
|
APPROVED_ACTION_RECOVERY_PROJECT_INDEX,
|
|
APPROVED_ACTION_RECOVERY_RESOLUTION_MUTATION_INDEX,
|
|
APPROVED_ACTION_RECOVERY_RESOLUTION_PROJECT_INDEX,
|
|
APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE,
|
|
approvedActionRecoveryMigration,
|
|
} = require('../../back/migrations/0022-approved-action-recovery');
|
|
const {
|
|
approvedActionRecoveryAuthorizationMigration,
|
|
} = require('../../back/migrations/0024-approved-action-recovery-authorization');
|
|
const { runMigrations } = require('../../back/migrations/runner');
|
|
const {
|
|
LegacySequelizeApprovedActionDispatchRepository,
|
|
} = require('../../back/runtime/adapters/legacy-sequelize/approvedActionDispatchRepository');
|
|
const {
|
|
LegacySequelizeApprovedActionRecoveryRepository,
|
|
} = require('../../back/runtime/adapters/legacy-sequelize/approvedActionRecoveryRepository');
|
|
const {
|
|
LegacySequelizeApprovalRequestRepository,
|
|
} = require('../../back/runtime/adapters/legacy-sequelize/approvalRequestRepository');
|
|
const {
|
|
LegacySequelizeProjectPolicyRepository,
|
|
} = require('../../back/runtime/adapters/legacy-sequelize/projectPolicyRepository');
|
|
const {
|
|
ApprovalRequestService,
|
|
} = require('../../back/runtime/application/approvalRequestService');
|
|
const {
|
|
ProjectPolicyEngine,
|
|
} = require('../../back/runtime/application/projectPolicyEngine');
|
|
const {
|
|
ApprovedActionDispatchRepositoryError,
|
|
} = require('../../back/runtime/domain/approvedActionDispatchExecution');
|
|
const {
|
|
ApprovedActionRecoveryFenceRejectedError,
|
|
} = require('../../back/runtime/domain/approvedActionRecovery');
|
|
const {
|
|
createApprovedActionRecoveryAuthorizationFact,
|
|
} = require('../../back/runtime/domain/approvedActionRecoveryAuthorization');
|
|
|
|
const PROJECT_ID = 'default';
|
|
const AGENT = Object.freeze({ type: 'agent', id: 'agent-1' });
|
|
const OWNER = Object.freeze({ type: 'user', id: 'owner-1' });
|
|
const SYSTEM = Object.freeze({ type: 'system', id: 'approval-dispatcher' });
|
|
const BASE_TIME = 100_000;
|
|
|
|
function action(name, digestCharacter = 'a') {
|
|
return {
|
|
permission: 'tool.call:filesystem.write',
|
|
actionType: 'tool_call',
|
|
actionRef: `planned-${name}`,
|
|
actionDigest: digestCharacter.repeat(64),
|
|
previewDigest: 'f'.repeat(64),
|
|
};
|
|
}
|
|
|
|
async function migrate(database, migrations) {
|
|
await runMigrations({
|
|
database,
|
|
migrationModel: defineSchemaMigrationModel(database),
|
|
migrations,
|
|
logger: { info() {} },
|
|
});
|
|
}
|
|
|
|
async function bind(policyRepository, subject, role, mutationId) {
|
|
await policyRepository.append({
|
|
expectedCurrentVersion: 0,
|
|
binding: {
|
|
projectId: PROJECT_ID,
|
|
subject,
|
|
version: 1,
|
|
state: 'active',
|
|
role,
|
|
mutationId,
|
|
changedBy: OWNER,
|
|
createdAtMs: BASE_TIME - 100,
|
|
},
|
|
});
|
|
}
|
|
|
|
async function setup(t, storage = ':memory:', includeRecovery = true) {
|
|
const database = new Sequelize({
|
|
dialect: 'sqlite',
|
|
storage,
|
|
logging: false,
|
|
});
|
|
t.after(() => database.close());
|
|
await migrate(database, [
|
|
projectPolicyMigration,
|
|
approvalRequestMigration,
|
|
approvedActionDispatchExecutionMigration,
|
|
...(includeRecovery
|
|
? [
|
|
approvedActionRecoveryMigration,
|
|
approvedActionRecoveryAuthorizationMigration,
|
|
]
|
|
: []),
|
|
]);
|
|
const policyRepository = new LegacySequelizeProjectPolicyRepository(database);
|
|
await bind(policyRepository, OWNER, 'owner', 'bind-owner');
|
|
await bind(policyRepository, AGENT, 'operator', 'bind-agent');
|
|
const approvalRepository = new LegacySequelizeApprovalRequestRepository(
|
|
database,
|
|
);
|
|
return {
|
|
database,
|
|
approvalService: new ApprovalRequestService(
|
|
approvalRepository,
|
|
new ProjectPolicyEngine(policyRepository),
|
|
),
|
|
actionRepository: new LegacySequelizeApprovedActionDispatchRepository(
|
|
database,
|
|
),
|
|
recoveryRepository: includeRecovery
|
|
? new LegacySequelizeApprovedActionRecoveryRepository(database)
|
|
: null,
|
|
};
|
|
}
|
|
|
|
async function prepareDispatch(service, name, offset = 0, digest = 'a') {
|
|
const requestedAtMs = BASE_TIME + offset;
|
|
const binding = action(name, digest);
|
|
await service.create({
|
|
id: `approval-${name}`,
|
|
projectId: PROJECT_ID,
|
|
action: binding,
|
|
risk: 'high',
|
|
requestedBy: AGENT,
|
|
requestedAtMs,
|
|
expiresAtMs: requestedAtMs + 60_000,
|
|
});
|
|
await service.decide({
|
|
requestId: `approval-${name}`,
|
|
expectedVersion: 1,
|
|
decisionId: `decision-${name}`,
|
|
decision: 'approved',
|
|
reasonCode: 'reviewed_action',
|
|
decidedBy: OWNER,
|
|
decidedAtMs: requestedAtMs + 10,
|
|
});
|
|
return (
|
|
await service.consume({
|
|
requestId: `approval-${name}`,
|
|
expectedVersion: 2,
|
|
consumptionId: `consumption-${name}`,
|
|
dispatchId: `dispatch-${name}`,
|
|
action: binding,
|
|
requestedBy: AGENT,
|
|
consumedBy: SYSTEM,
|
|
consumedAtMs: requestedAtMs + 20,
|
|
})
|
|
).dispatch;
|
|
}
|
|
|
|
async function startAction(actionRepository, dispatch, options = {}) {
|
|
const nowMs = options.nowMs ?? BASE_TIME + 1_000;
|
|
const leaseDurationMs = options.leaseDurationMs ?? 100;
|
|
const owner = options.owner ?? 'dispatcher-1';
|
|
const leaseToken = options.leaseToken ?? 'execution-lease-1';
|
|
const claimed = await actionRepository.claim({
|
|
dispatchId: dispatch.id,
|
|
owner,
|
|
leaseToken,
|
|
nowMs,
|
|
leaseDurationMs,
|
|
});
|
|
assert.equal(claimed.status, 'claimed');
|
|
const started = await actionRepository.start({
|
|
dispatchId: dispatch.id,
|
|
approvalRequestId: dispatch.approvalRequestId,
|
|
actionDigest: dispatch.action.actionDigest,
|
|
owner,
|
|
leaseToken,
|
|
expectedVersion: claimed.snapshot.execution.version,
|
|
startedAtMs: nowMs + 10,
|
|
});
|
|
return {
|
|
claimed,
|
|
started,
|
|
owner,
|
|
leaseToken,
|
|
leaseExpiresAtMs: nowMs + leaseDurationMs,
|
|
};
|
|
}
|
|
|
|
async function claimRecovery(recoveryRepository, dispatchId, overrides = {}) {
|
|
return recoveryRepository.claim({
|
|
dispatchId,
|
|
owner: 'resolver-1',
|
|
leaseToken: 'recovery-lease-1',
|
|
nowMs: BASE_TIME + 1_100,
|
|
leaseDurationMs: 1_000,
|
|
...overrides,
|
|
});
|
|
}
|
|
|
|
test('0022 backfills executing controls and owns bounded recovery indexes', async (t) => {
|
|
const context = await setup(t, ':memory:', false);
|
|
const dispatch = await prepareDispatch(context.approvalService, 'backfill');
|
|
await context.database.query(
|
|
`UPDATE "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}"
|
|
SET status = 'executing', version = 2, attempt_count = 1,
|
|
eligible_at_ms = NULL, lease_owner = 'dispatcher-1',
|
|
lease_token = 'execution-lease-1',
|
|
lease_expires_at_ms = :leaseExpiresAtMs,
|
|
started_at_ms = :startedAtMs, updated_at_ms = :startedAtMs
|
|
WHERE dispatch_id = :dispatchId`,
|
|
{
|
|
replacements: {
|
|
dispatchId: dispatch.id,
|
|
startedAtMs: BASE_TIME + 1_010,
|
|
leaseExpiresAtMs: BASE_TIME + 1_100,
|
|
},
|
|
},
|
|
);
|
|
await migrate(context.database, [approvedActionRecoveryMigration]);
|
|
const rows = await context.database.query(
|
|
`SELECT * FROM "${APPROVED_ACTION_RECOVERY_CONTROL_TABLE}"`,
|
|
{ type: QueryTypes.SELECT },
|
|
);
|
|
assert.equal(rows.length, 1);
|
|
assert.equal(rows[0].dispatch_id, dispatch.id);
|
|
assert.equal(rows[0].execution_version, 2);
|
|
assert.equal(rows[0].status, 'armed');
|
|
assert.equal(rows[0].next_scan_at_ms, BASE_TIME + 1_100);
|
|
const controlIndexes = new Set(
|
|
(
|
|
await context.database
|
|
.getQueryInterface()
|
|
.showIndex(APPROVED_ACTION_RECOVERY_CONTROL_TABLE)
|
|
).map((index) => index.name),
|
|
);
|
|
for (const name of [
|
|
APPROVED_ACTION_RECOVERY_DUE_INDEX,
|
|
APPROVED_ACTION_RECOVERY_PROJECT_INDEX,
|
|
APPROVED_ACTION_RECOVERY_LEASE_INDEX,
|
|
]) {
|
|
assert.ok(controlIndexes.has(name));
|
|
}
|
|
const resolutionIndexes = new Set(
|
|
(
|
|
await context.database
|
|
.getQueryInterface()
|
|
.showIndex(APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE)
|
|
).map((index) => index.name),
|
|
);
|
|
assert.ok(
|
|
resolutionIndexes.has(APPROVED_ACTION_RECOVERY_RESOLUTION_MUTATION_INDEX),
|
|
);
|
|
assert.ok(
|
|
resolutionIndexes.has(APPROVED_ACTION_RECOVERY_RESOLUTION_PROJECT_INDEX),
|
|
);
|
|
});
|
|
|
|
test('start barrier atomically arms recovery and rolls back when control insert fails', async (t) => {
|
|
const context = await setup(t);
|
|
const dispatch = await prepareDispatch(
|
|
context.approvalService,
|
|
'start-atomic',
|
|
);
|
|
const claimed = await context.actionRepository.claim({
|
|
dispatchId: dispatch.id,
|
|
owner: 'dispatcher-1',
|
|
leaseToken: 'execution-lease-1',
|
|
nowMs: BASE_TIME + 1_000,
|
|
leaseDurationMs: 100,
|
|
});
|
|
await context.database.query(
|
|
`CREATE TRIGGER fail_recovery_control_insert
|
|
BEFORE INSERT ON "${APPROVED_ACTION_RECOVERY_CONTROL_TABLE}"
|
|
BEGIN SELECT RAISE(ABORT, 'forced recovery insert failure'); END`,
|
|
);
|
|
await assert.rejects(
|
|
context.actionRepository.start({
|
|
dispatchId: dispatch.id,
|
|
approvalRequestId: dispatch.approvalRequestId,
|
|
actionDigest: dispatch.action.actionDigest,
|
|
owner: 'dispatcher-1',
|
|
leaseToken: 'execution-lease-1',
|
|
expectedVersion: claimed.snapshot.execution.version,
|
|
startedAtMs: BASE_TIME + 1_010,
|
|
}),
|
|
ApprovedActionDispatchRepositoryError,
|
|
);
|
|
assert.equal(
|
|
(await context.actionRepository.findById(dispatch.id)).execution.status,
|
|
'leased',
|
|
);
|
|
assert.equal(await context.recoveryRepository.findById(dispatch.id), null);
|
|
});
|
|
|
|
test('normal completion closes the recovery control without creating a recovery resolution', async (t) => {
|
|
const context = await setup(t);
|
|
const dispatch = await prepareDispatch(context.approvalService, 'complete');
|
|
const execution = await startAction(context.actionRepository, dispatch);
|
|
const armed = await context.recoveryRepository.findById(dispatch.id);
|
|
assert.equal(armed.recovery.status, 'armed');
|
|
assert.equal(
|
|
armed.recovery.executionVersion,
|
|
execution.started.execution.version,
|
|
);
|
|
assert.equal(armed.recovery.nextScanAtMs, execution.leaseExpiresAtMs);
|
|
const completed = await context.actionRepository.complete({
|
|
dispatchId: dispatch.id,
|
|
owner: execution.owner,
|
|
leaseToken: execution.leaseToken,
|
|
expectedVersion: execution.started.execution.version,
|
|
resultMutationId: 'normal-completion-1',
|
|
outcome: 'succeeded',
|
|
resultCode: 'ok',
|
|
completedAtMs: BASE_TIME + 1_120,
|
|
});
|
|
assert.equal(completed.execution.status, 'succeeded');
|
|
const closed = await context.recoveryRepository.findById(dispatch.id);
|
|
assert.equal(closed.recovery.status, 'resolved');
|
|
assert.equal(closed.recovery.resolutionMutationId, 'normal-completion-1');
|
|
assert.equal(closed.resolution, null);
|
|
assert.equal(
|
|
(await claimRecovery(context.recoveryRepository, dispatch.id)).status,
|
|
'resolved',
|
|
);
|
|
});
|
|
|
|
test('a late execution renew re-arms recovery and fences the stale resolver lease', async (t) => {
|
|
const context = await setup(t);
|
|
const dispatch = await prepareDispatch(context.approvalService, 'renew');
|
|
const execution = await startAction(context.actionRepository, dispatch);
|
|
const claimed = await claimRecovery(context.recoveryRepository, dispatch.id);
|
|
assert.equal(claimed.status, 'claimed');
|
|
const renewed = await context.actionRepository.renew({
|
|
dispatchId: dispatch.id,
|
|
owner: execution.owner,
|
|
leaseToken: execution.leaseToken,
|
|
expectedVersion: execution.started.execution.version,
|
|
nowMs: BASE_TIME + 1_150,
|
|
leaseDurationMs: 1_000,
|
|
});
|
|
assert.equal(
|
|
renewed.execution.version,
|
|
execution.started.execution.version + 1,
|
|
);
|
|
const rearmed = await context.recoveryRepository.findById(dispatch.id);
|
|
assert.equal(rearmed.recovery.status, 'armed');
|
|
assert.equal(rearmed.recovery.executionVersion, renewed.execution.version);
|
|
assert.equal(rearmed.recovery.nextScanAtMs, BASE_TIME + 2_150);
|
|
assert.equal(rearmed.recovery.leaseOwner, null);
|
|
await assert.rejects(
|
|
context.recoveryRepository.recordFinding({
|
|
dispatchId: dispatch.id,
|
|
expectedExecutionVersion: claimed.snapshot.action.execution.version,
|
|
expectedRecoveryVersion: claimed.snapshot.recovery.version,
|
|
owner: 'resolver-1',
|
|
leaseToken: 'recovery-lease-1',
|
|
findingMutationId: 'stale-finding-1',
|
|
finding: 'missing',
|
|
resultCode: 'receipt_missing',
|
|
observedAtMs: BASE_TIME + 1_160,
|
|
retryAtMs: BASE_TIME + 1_200,
|
|
}),
|
|
ApprovedActionRecoveryFenceRejectedError,
|
|
);
|
|
});
|
|
|
|
test('lists a bounded stable recovery page and excludes live executions', async (t) => {
|
|
const context = await setup(t);
|
|
const firstDispatch = await prepareDispatch(
|
|
context.approvalService,
|
|
'due-a',
|
|
0,
|
|
'a',
|
|
);
|
|
const secondDispatch = await prepareDispatch(
|
|
context.approvalService,
|
|
'due-b',
|
|
10,
|
|
'b',
|
|
);
|
|
const liveDispatch = await prepareDispatch(
|
|
context.approvalService,
|
|
'live',
|
|
20,
|
|
'c',
|
|
);
|
|
await startAction(context.actionRepository, firstDispatch);
|
|
await startAction(context.actionRepository, secondDispatch);
|
|
await startAction(context.actionRepository, liveDispatch, {
|
|
leaseDurationMs: 1_000,
|
|
});
|
|
const first = await context.recoveryRepository.listDue({
|
|
nowMs: BASE_TIME + 1_100,
|
|
limit: 1,
|
|
});
|
|
assert.deepEqual(
|
|
first.recoveries.map((entry) => entry.action.dispatch.id),
|
|
['dispatch-due-a'],
|
|
);
|
|
assert.equal(first.truncated, true);
|
|
const second = await context.recoveryRepository.listDue({
|
|
nowMs: BASE_TIME + 1_100,
|
|
limit: 1,
|
|
cursor: first.nextCursor,
|
|
});
|
|
assert.deepEqual(
|
|
second.recoveries.map((entry) => entry.action.dispatch.id),
|
|
['dispatch-due-b'],
|
|
);
|
|
assert.equal(second.truncated, false);
|
|
});
|
|
|
|
test('two SQLite resolvers claim once and only an expired recovery lease is taken over', async (t) => {
|
|
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-recovery-'));
|
|
t.after(() => fs.rm(directory, { recursive: true, force: true }));
|
|
const storage = path.join(directory, 'database.sqlite');
|
|
const first = await setup(t, storage);
|
|
const dispatch = await prepareDispatch(first.approvalService, 'claim-race');
|
|
await startAction(first.actionRepository, dispatch);
|
|
const secondDatabase = new Sequelize({
|
|
dialect: 'sqlite',
|
|
storage,
|
|
logging: false,
|
|
});
|
|
t.after(() => secondDatabase.close());
|
|
const secondRepository = new LegacySequelizeApprovedActionRecoveryRepository(
|
|
secondDatabase,
|
|
);
|
|
const results = await Promise.all([
|
|
claimRecovery(first.recoveryRepository, dispatch.id, {
|
|
leaseDurationMs: 100,
|
|
}),
|
|
claimRecovery(secondRepository, dispatch.id, {
|
|
owner: 'resolver-2',
|
|
leaseToken: 'recovery-lease-2',
|
|
leaseDurationMs: 100,
|
|
}),
|
|
]);
|
|
assert.equal(
|
|
results.filter((result) => result.status === 'claimed').length,
|
|
1,
|
|
);
|
|
assert.equal(
|
|
results.filter((result) => result.status === 'leased').length,
|
|
1,
|
|
);
|
|
const winner = results.find((result) => result.status === 'claimed');
|
|
assert.equal(
|
|
(
|
|
await claimRecovery(first.recoveryRepository, dispatch.id, {
|
|
owner: winner.snapshot.recovery.leaseOwner,
|
|
leaseToken: winner.snapshot.recovery.leaseToken,
|
|
leaseDurationMs: 100,
|
|
})
|
|
).status,
|
|
'claimed',
|
|
);
|
|
const takeover = await claimRecovery(secondRepository, dispatch.id, {
|
|
owner: 'resolver-3',
|
|
leaseToken: 'recovery-lease-3',
|
|
nowMs: BASE_TIME + 1_200,
|
|
leaseDurationMs: 100,
|
|
});
|
|
assert.equal(takeover.status, 'claimed');
|
|
assert.equal(takeover.snapshot.recovery.leaseOwner, 'resolver-3');
|
|
});
|
|
|
|
test('recovery finding replay is exact and unsupported evidence becomes manual-only', async (t) => {
|
|
const context = await setup(t);
|
|
const dispatch = await prepareDispatch(context.approvalService, 'finding');
|
|
await startAction(context.actionRepository, dispatch);
|
|
const claimed = await claimRecovery(context.recoveryRepository, dispatch.id);
|
|
assert.equal(claimed.status, 'claimed');
|
|
const finding = {
|
|
dispatchId: dispatch.id,
|
|
expectedExecutionVersion: claimed.snapshot.action.execution.version,
|
|
expectedRecoveryVersion: claimed.snapshot.recovery.version,
|
|
owner: 'resolver-1',
|
|
leaseToken: 'recovery-lease-1',
|
|
findingMutationId: 'finding-missing-1',
|
|
finding: 'missing',
|
|
resultCode: 'receipt_missing',
|
|
observedAtMs: BASE_TIME + 1_110,
|
|
retryAtMs: BASE_TIME + 1_200,
|
|
};
|
|
const deferred = await context.recoveryRepository.recordFinding(finding);
|
|
assert.equal(deferred.recovery.status, 'armed');
|
|
assert.equal(deferred.recovery.findingCount, 1);
|
|
assert.deepEqual(
|
|
await context.recoveryRepository.recordFinding(finding),
|
|
deferred,
|
|
);
|
|
assert.equal(
|
|
(
|
|
await claimRecovery(context.recoveryRepository, dispatch.id, {
|
|
nowMs: BASE_TIME + 1_199,
|
|
})
|
|
).status,
|
|
'not_due',
|
|
);
|
|
const second = await claimRecovery(context.recoveryRepository, dispatch.id, {
|
|
owner: 'resolver-2',
|
|
leaseToken: 'recovery-lease-2',
|
|
nowMs: BASE_TIME + 1_200,
|
|
});
|
|
assert.equal(second.status, 'claimed');
|
|
const manual = await context.recoveryRepository.recordFinding({
|
|
dispatchId: dispatch.id,
|
|
expectedExecutionVersion: second.snapshot.action.execution.version,
|
|
expectedRecoveryVersion: second.snapshot.recovery.version,
|
|
owner: 'resolver-2',
|
|
leaseToken: 'recovery-lease-2',
|
|
findingMutationId: 'finding-unsupported-1',
|
|
finding: 'unsupported',
|
|
resultCode: 'automatic_recovery_unsupported',
|
|
observedAtMs: BASE_TIME + 1_210,
|
|
});
|
|
assert.equal(manual.recovery.status, 'manual_required');
|
|
assert.equal(manual.recovery.nextScanAtMs, null);
|
|
assert.equal(
|
|
(
|
|
await context.recoveryRepository.listDue({
|
|
nowMs: BASE_TIME + 10_000,
|
|
limit: 64,
|
|
})
|
|
).recoveries.length,
|
|
0,
|
|
);
|
|
});
|
|
|
|
test('verified evidence resolves atomically and exact mutation replay returns the same resolution', async (t) => {
|
|
const context = await setup(t);
|
|
const dispatch = await prepareDispatch(context.approvalService, 'verified');
|
|
await startAction(context.actionRepository, dispatch);
|
|
const claimed = await claimRecovery(context.recoveryRepository, dispatch.id);
|
|
const command = {
|
|
dispatchId: dispatch.id,
|
|
expectedExecutionVersion: claimed.snapshot.action.execution.version,
|
|
expectedRecoveryVersion: claimed.snapshot.recovery.version,
|
|
owner: 'resolver-1',
|
|
leaseToken: 'recovery-lease-1',
|
|
mutationId: 'resolution-verified-1',
|
|
source: 'automatic_evidence',
|
|
decision: 'confirm_succeeded',
|
|
evidenceDigest: 'e'.repeat(64),
|
|
reasonCode: 'provider_receipt_verified',
|
|
resolvedAtMs: BASE_TIME + 1_120,
|
|
};
|
|
const resolved = await context.recoveryRepository.resolve(command);
|
|
assert.equal(resolved.status, 'resolved');
|
|
assert.equal(resolved.snapshot.action.execution.status, 'succeeded');
|
|
assert.equal(resolved.snapshot.recovery.status, 'resolved');
|
|
assert.equal(resolved.snapshot.resolution.source, 'automatic_evidence');
|
|
assert.equal(resolved.snapshot.resolution.evidenceDigest, 'e'.repeat(64));
|
|
assert.deepEqual(await context.recoveryRepository.resolve(command), resolved);
|
|
});
|
|
|
|
test('manual resolution cannot preempt a live execution and never resets it for retry', async (t) => {
|
|
const context = await setup(t);
|
|
const dispatch = await prepareDispatch(context.approvalService, 'manual');
|
|
const execution = await startAction(context.actionRepository, dispatch, {
|
|
leaseDurationMs: 500,
|
|
});
|
|
const armed = await context.recoveryRepository.findById(dispatch.id);
|
|
const commandAt = (resolvedAtMs) => {
|
|
const command = {
|
|
dispatchId: dispatch.id,
|
|
expectedExecutionVersion: armed.action.execution.version,
|
|
expectedRecoveryVersion: armed.recovery.version,
|
|
mutationId: 'resolution-manual-1',
|
|
source: 'human',
|
|
decision: 'abandon_unknown',
|
|
reasonCode: 'operator_abandoned_unknown',
|
|
resolvedBy: OWNER,
|
|
resolvedAtMs,
|
|
};
|
|
return {
|
|
...command,
|
|
authorizationFact: createApprovedActionRecoveryAuthorizationFact({
|
|
dispatchId: dispatch.id,
|
|
projectId: PROJECT_ID,
|
|
mutationId: command.mutationId,
|
|
resolvedBy: OWNER,
|
|
authenticationId: 'mfa-session-1',
|
|
assurance: 'multi_factor',
|
|
authenticatedAtMs: resolvedAtMs - 100,
|
|
projectVersion: 1,
|
|
bindingVersion: 1,
|
|
authorizedAtMs: resolvedAtMs,
|
|
}),
|
|
};
|
|
};
|
|
await assert.rejects(
|
|
context.recoveryRepository.resolve(
|
|
commandAt(execution.leaseExpiresAtMs - 1),
|
|
),
|
|
ApprovedActionRecoveryFenceRejectedError,
|
|
);
|
|
const resolved = await context.recoveryRepository.resolve(
|
|
commandAt(execution.leaseExpiresAtMs),
|
|
);
|
|
assert.equal(resolved.snapshot.action.execution.status, 'blocked');
|
|
assert.equal(resolved.snapshot.resolution.decision, 'abandon_unknown');
|
|
assert.equal(resolved.snapshot.resolution.resolvedBy.type, 'user');
|
|
assert.equal(resolved.snapshot.action.execution.eligibleAtMs, null);
|
|
});
|
|
|
|
test('late completion and recovery resolution have one durable terminal winner', async (t) => {
|
|
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-recovery-'));
|
|
t.after(() => fs.rm(directory, { recursive: true, force: true }));
|
|
const storage = path.join(directory, 'database.sqlite');
|
|
const first = await setup(t, storage);
|
|
const dispatch = await prepareDispatch(first.approvalService, 'race');
|
|
const execution = await startAction(first.actionRepository, dispatch);
|
|
const secondDatabase = new Sequelize({
|
|
dialect: 'sqlite',
|
|
storage,
|
|
logging: false,
|
|
});
|
|
t.after(() => secondDatabase.close());
|
|
const secondRecovery = new LegacySequelizeApprovedActionRecoveryRepository(
|
|
secondDatabase,
|
|
);
|
|
const claimed = await claimRecovery(secondRecovery, dispatch.id);
|
|
const recoveryCommand = {
|
|
dispatchId: dispatch.id,
|
|
expectedExecutionVersion: claimed.snapshot.action.execution.version,
|
|
expectedRecoveryVersion: claimed.snapshot.recovery.version,
|
|
owner: 'resolver-1',
|
|
leaseToken: 'recovery-lease-1',
|
|
mutationId: 'resolution-race-1',
|
|
source: 'automatic_evidence',
|
|
decision: 'confirm_failed',
|
|
evidenceDigest: 'd'.repeat(64),
|
|
reasonCode: 'provider_failed',
|
|
resolvedAtMs: BASE_TIME + 1_120,
|
|
};
|
|
await Promise.allSettled([
|
|
first.actionRepository.complete({
|
|
dispatchId: dispatch.id,
|
|
owner: execution.owner,
|
|
leaseToken: execution.leaseToken,
|
|
expectedVersion: execution.started.execution.version,
|
|
resultMutationId: 'normal-race-1',
|
|
outcome: 'succeeded',
|
|
resultCode: 'ok',
|
|
completedAtMs: BASE_TIME + 1_120,
|
|
}),
|
|
secondRecovery.resolve(recoveryCommand),
|
|
]);
|
|
const final = await secondRecovery.findById(dispatch.id);
|
|
assert.ok(['succeeded', 'failed'].includes(final.action.execution.status));
|
|
assert.equal(final.recovery.status, 'resolved');
|
|
assert.equal(
|
|
final.recovery.resolutionMutationId,
|
|
final.action.execution.resultMutationId,
|
|
);
|
|
if (final.resolution) {
|
|
assert.equal(final.resolution.mutationId, 'resolution-race-1');
|
|
assert.equal(final.action.execution.status, 'failed');
|
|
} else {
|
|
assert.equal(final.action.execution.resultMutationId, 'normal-race-1');
|
|
assert.equal(final.action.execution.status, 'succeeded');
|
|
}
|
|
});
|