mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,489 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const { Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const {
|
||||
projectPolicyMigration,
|
||||
} = require('../../back/migrations/0017-project-policy');
|
||||
const {
|
||||
APPROVAL_REQUEST_CONSUMPTION_ID_INDEX,
|
||||
APPROVAL_REQUEST_DECISION_ID_INDEX,
|
||||
APPROVAL_REQUEST_PENDING_INDEX,
|
||||
APPROVAL_REQUEST_REQUESTER_INDEX,
|
||||
APPROVAL_REQUEST_TABLE,
|
||||
APPROVED_ACTION_DISPATCH_PENDING_INDEX,
|
||||
APPROVED_ACTION_DISPATCH_REQUEST_INDEX,
|
||||
APPROVED_ACTION_DISPATCH_TABLE,
|
||||
approvalRequestMigration,
|
||||
} = require('../../back/migrations/0020-approval-requests');
|
||||
const {
|
||||
approvedActionDispatchExecutionMigration,
|
||||
} = require('../../back/migrations/0021-approved-action-dispatch-executions');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeApprovalRequestRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/approvalRequestRepository');
|
||||
const {
|
||||
LegacySequelizeProjectPolicyRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/projectPolicyRepository');
|
||||
const {
|
||||
ApprovalRequestService,
|
||||
} = require('../../back/runtime/application/approvalRequestService');
|
||||
const {
|
||||
ProjectPolicyEngine,
|
||||
} = require('../../back/runtime/application/projectPolicyEngine');
|
||||
const {
|
||||
ApprovalHumanDecisionRequiredError,
|
||||
ApprovalMutationConflictError,
|
||||
ApprovalPolicyDeniedError,
|
||||
ApprovalPolicyFenceConflictError,
|
||||
ApprovalRequestExpiredError,
|
||||
ApprovalRequestStateConflictError,
|
||||
ApprovalRequestVersionConflictError,
|
||||
} = require('../../back/runtime/domain/approvalRequest');
|
||||
|
||||
const PROJECT_ID = 'default';
|
||||
const AGENT = Object.freeze({ type: 'agent', id: 'agent-1' });
|
||||
const OWNER = Object.freeze({ type: 'user', id: 'owner-1' });
|
||||
const VIEWER = Object.freeze({ type: 'user', id: 'viewer-1' });
|
||||
const SYSTEM = Object.freeze({ type: 'system', id: 'approval-dispatcher' });
|
||||
const NOW = 100_000;
|
||||
const EXPIRES_AT = 200_000;
|
||||
|
||||
function action(overrides = {}) {
|
||||
return {
|
||||
permission: 'tool.call:filesystem.write',
|
||||
actionType: 'tool_call',
|
||||
actionRef: 'planned-action-1',
|
||||
actionDigest: 'a'.repeat(64),
|
||||
previewDigest: 'b'.repeat(64),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createInput(overrides = {}) {
|
||||
return {
|
||||
id: 'approval-1',
|
||||
projectId: PROJECT_ID,
|
||||
action: action(),
|
||||
risk: 'high',
|
||||
requestedBy: AGENT,
|
||||
requestedAtMs: NOW,
|
||||
expiresAtMs: EXPIRES_AT,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function decisionInput(overrides = {}) {
|
||||
return {
|
||||
requestId: 'approval-1',
|
||||
expectedVersion: 1,
|
||||
decisionId: 'decision-1',
|
||||
decision: 'approved',
|
||||
reasonCode: 'reviewed_action',
|
||||
decidedBy: OWNER,
|
||||
decidedAtMs: NOW + 10,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function consumptionInput(overrides = {}) {
|
||||
return {
|
||||
requestId: 'approval-1',
|
||||
expectedVersion: 2,
|
||||
consumptionId: 'consumption-1',
|
||||
dispatchId: 'dispatch-1',
|
||||
action: action(),
|
||||
requestedBy: AGENT,
|
||||
consumedBy: SYSTEM,
|
||||
consumedAtMs: NOW + 20,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function migrate(database) {
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [
|
||||
projectPolicyMigration,
|
||||
approvalRequestMigration,
|
||||
approvedActionDispatchExecutionMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
}
|
||||
|
||||
async function bind(policyRepository, subject, role, mutationId) {
|
||||
await policyRepository.append({
|
||||
expectedCurrentVersion: 0,
|
||||
binding: {
|
||||
projectId: PROJECT_ID,
|
||||
subject,
|
||||
version: 1,
|
||||
state: 'active',
|
||||
role,
|
||||
mutationId,
|
||||
changedBy: OWNER,
|
||||
createdAtMs: NOW - 100,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function setup(t, storage = ':memory:') {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
await migrate(database);
|
||||
const policyRepository = new LegacySequelizeProjectPolicyRepository(database);
|
||||
await bind(policyRepository, OWNER, 'owner', 'bind-owner');
|
||||
await bind(policyRepository, AGENT, 'operator', 'bind-agent');
|
||||
await bind(policyRepository, VIEWER, 'viewer', 'bind-viewer');
|
||||
const repository = new LegacySequelizeApprovalRequestRepository(database);
|
||||
const policy = new ProjectPolicyEngine(policyRepository);
|
||||
return {
|
||||
database,
|
||||
policyRepository,
|
||||
repository,
|
||||
policy,
|
||||
service: new ApprovalRequestService(repository, policy),
|
||||
};
|
||||
}
|
||||
|
||||
test('migration owns bounded approval and durable dispatch indexes', async (t) => {
|
||||
const { database, service } = await setup(t);
|
||||
await service.create(createInput());
|
||||
const rows = await database
|
||||
.getQueryInterface()
|
||||
.select(null, APPROVAL_REQUEST_TABLE);
|
||||
assert.equal(rows.length, 1);
|
||||
assert.equal(rows[0].action_digest, 'a'.repeat(64));
|
||||
assert.equal(rows[0].preview_digest, 'b'.repeat(64));
|
||||
assert.equal(JSON.stringify(rows).includes('filesystem.write'), true);
|
||||
assert.equal('preview' in rows[0], false);
|
||||
assert.equal('arguments' in rows[0], false);
|
||||
assert.equal('secret' in rows[0], false);
|
||||
|
||||
const requestIndexes = new Set(
|
||||
(await database.getQueryInterface().showIndex(APPROVAL_REQUEST_TABLE)).map(
|
||||
(index) => index.name,
|
||||
),
|
||||
);
|
||||
for (const name of [
|
||||
APPROVAL_REQUEST_DECISION_ID_INDEX,
|
||||
APPROVAL_REQUEST_CONSUMPTION_ID_INDEX,
|
||||
APPROVAL_REQUEST_PENDING_INDEX,
|
||||
APPROVAL_REQUEST_REQUESTER_INDEX,
|
||||
]) {
|
||||
assert.ok(requestIndexes.has(name));
|
||||
}
|
||||
const dispatchIndexes = new Set(
|
||||
(
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.showIndex(APPROVED_ACTION_DISPATCH_TABLE)
|
||||
).map((index) => index.name),
|
||||
);
|
||||
assert.ok(dispatchIndexes.has(APPROVED_ACTION_DISPATCH_REQUEST_INDEX));
|
||||
assert.ok(dispatchIndexes.has(APPROVED_ACTION_DISPATCH_PENDING_INDEX));
|
||||
});
|
||||
|
||||
test('creates, approves and atomically emits a one-time durable dispatch', async (t) => {
|
||||
const { database, service } = await setup(t);
|
||||
const pending = await service.create(createInput());
|
||||
assert.equal(pending.state, 'pending');
|
||||
assert.equal(pending.version, 1);
|
||||
|
||||
const approved = await service.decide(decisionInput());
|
||||
assert.equal(approved.state, 'approved');
|
||||
assert.equal(approved.version, 2);
|
||||
assert.deepEqual(approved.decidedBy, OWNER);
|
||||
|
||||
const consumed = await service.consume(consumptionInput());
|
||||
assert.equal(consumed.request.state, 'consumed');
|
||||
assert.equal(consumed.request.version, 3);
|
||||
assert.equal(consumed.dispatch.state, 'pending');
|
||||
assert.equal(consumed.dispatch.approvalRequestId, pending.id);
|
||||
assert.equal(
|
||||
consumed.dispatch.action.actionDigest,
|
||||
pending.action.actionDigest,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.select(null, APPROVED_ACTION_DISPATCH_TABLE)
|
||||
).length,
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
test('only require-approval actions can create requests', async (t) => {
|
||||
const { service } = await setup(t);
|
||||
await assert.rejects(
|
||||
service.create(createInput({ requestedBy: OWNER })),
|
||||
ApprovalPolicyDeniedError,
|
||||
);
|
||||
await assert.rejects(
|
||||
service.create(
|
||||
createInput({ requestedBy: { type: 'agent', id: 'unbound-agent' } }),
|
||||
),
|
||||
ApprovalPolicyDeniedError,
|
||||
);
|
||||
});
|
||||
|
||||
test('decisions require a bound human user with approval.decide', async (t) => {
|
||||
const { service } = await setup(t);
|
||||
await service.create(createInput());
|
||||
await assert.rejects(
|
||||
service.decide(decisionInput({ decidedBy: AGENT })),
|
||||
ApprovalHumanDecisionRequiredError,
|
||||
);
|
||||
await assert.rejects(
|
||||
service.decide(decisionInput({ decidedBy: VIEWER })),
|
||||
ApprovalPolicyDeniedError,
|
||||
);
|
||||
const rejected = await service.decide(
|
||||
decisionInput({ decision: 'rejected', reasonCode: 'unsafe_action' }),
|
||||
);
|
||||
assert.equal(rejected.state, 'rejected');
|
||||
await assert.rejects(
|
||||
service.consume(consumptionInput()),
|
||||
ApprovalRequestStateConflictError,
|
||||
);
|
||||
});
|
||||
|
||||
test('expiry is exact and does not require background timers', async (t) => {
|
||||
const { service } = await setup(t);
|
||||
await service.create(createInput());
|
||||
assert.equal(
|
||||
(await service.get('approval-1', EXPIRES_AT - 1)).effectiveStatus,
|
||||
'pending',
|
||||
);
|
||||
assert.equal(
|
||||
(await service.get('approval-1', EXPIRES_AT)).effectiveStatus,
|
||||
'expired',
|
||||
);
|
||||
await assert.rejects(
|
||||
service.decide(decisionInput({ decidedAtMs: EXPIRES_AT })),
|
||||
ApprovalRequestExpiredError,
|
||||
);
|
||||
|
||||
await service.create(createInput({ id: 'approval-2' }));
|
||||
await service.decide(
|
||||
decisionInput({
|
||||
requestId: 'approval-2',
|
||||
decisionId: 'decision-2',
|
||||
decidedAtMs: EXPIRES_AT - 2,
|
||||
}),
|
||||
);
|
||||
await assert.rejects(
|
||||
service.consume(
|
||||
consumptionInput({
|
||||
requestId: 'approval-2',
|
||||
consumptionId: 'consumption-2',
|
||||
dispatchId: 'dispatch-2',
|
||||
consumedAtMs: EXPIRES_AT,
|
||||
}),
|
||||
),
|
||||
ApprovalRequestExpiredError,
|
||||
);
|
||||
});
|
||||
|
||||
test('exact mutation replays are idempotent and drift conflicts', async (t) => {
|
||||
const { service } = await setup(t);
|
||||
const pending = await service.create(createInput());
|
||||
assert.deepEqual(await service.create(createInput()), pending);
|
||||
await assert.rejects(
|
||||
service.create(createInput({ risk: 'critical' })),
|
||||
ApprovalMutationConflictError,
|
||||
);
|
||||
|
||||
const approved = await service.decide(decisionInput());
|
||||
assert.deepEqual(await service.decide(decisionInput()), approved);
|
||||
await assert.rejects(
|
||||
service.decide(decisionInput({ reasonCode: 'different_review' })),
|
||||
ApprovalMutationConflictError,
|
||||
);
|
||||
|
||||
const consumed = await service.consume(consumptionInput());
|
||||
assert.deepEqual(await service.consume(consumptionInput()), consumed);
|
||||
assert.deepEqual(await service.create(createInput()), consumed.request);
|
||||
await assert.rejects(
|
||||
service.consume(
|
||||
consumptionInput({ action: action({ actionDigest: 'c'.repeat(64) }) }),
|
||||
),
|
||||
ApprovalMutationConflictError,
|
||||
);
|
||||
});
|
||||
|
||||
test('policy versions fence every mutation against revocation races', async (t) => {
|
||||
const { policyRepository, policy, repository } = await setup(t);
|
||||
const authorization = await policy.decideWithFence({
|
||||
projectId: PROJECT_ID,
|
||||
subject: AGENT,
|
||||
permission: action().permission,
|
||||
});
|
||||
assert.equal(authorization.decision.effect, 'require_approval');
|
||||
await policyRepository.append({
|
||||
expectedCurrentVersion: 1,
|
||||
binding: {
|
||||
projectId: PROJECT_ID,
|
||||
subject: AGENT,
|
||||
version: 2,
|
||||
state: 'revoked',
|
||||
mutationId: 'revoke-agent',
|
||||
changedBy: OWNER,
|
||||
createdAtMs: NOW,
|
||||
},
|
||||
});
|
||||
const request = {
|
||||
...createInput(),
|
||||
version: 1,
|
||||
state: 'pending',
|
||||
decisionId: null,
|
||||
decision: null,
|
||||
decisionReasonCode: null,
|
||||
decidedBy: null,
|
||||
decidedAtMs: null,
|
||||
consumptionId: null,
|
||||
dispatchId: null,
|
||||
consumedBy: null,
|
||||
consumedAtMs: null,
|
||||
};
|
||||
await assert.rejects(
|
||||
repository.create({
|
||||
request,
|
||||
authorizationFence: authorization.fence,
|
||||
}),
|
||||
ApprovalPolicyFenceConflictError,
|
||||
);
|
||||
});
|
||||
|
||||
test('two SQLite connections allow only one decision and one consumption', async (t) => {
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-approval-'));
|
||||
t.after(() => fs.rm(directory, { recursive: true, force: true }));
|
||||
const storage = path.join(directory, 'database.sqlite');
|
||||
const first = await setup(t, storage);
|
||||
const secondDatabase = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => secondDatabase.close());
|
||||
const secondPolicyRepository = new LegacySequelizeProjectPolicyRepository(
|
||||
secondDatabase,
|
||||
);
|
||||
const secondRepository = new LegacySequelizeApprovalRequestRepository(
|
||||
secondDatabase,
|
||||
);
|
||||
const secondService = new ApprovalRequestService(
|
||||
secondRepository,
|
||||
new ProjectPolicyEngine(secondPolicyRepository),
|
||||
);
|
||||
|
||||
await first.service.create(createInput());
|
||||
const decisions = await Promise.allSettled([
|
||||
first.service.decide(decisionInput()),
|
||||
secondService.decide(
|
||||
decisionInput({ decisionId: 'decision-racer', reasonCode: 'racer' }),
|
||||
),
|
||||
]);
|
||||
assert.equal(
|
||||
decisions.filter((entry) => entry.status === 'fulfilled').length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
decisions.filter((entry) => entry.status === 'rejected').length,
|
||||
1,
|
||||
);
|
||||
assert.ok(
|
||||
decisions
|
||||
.filter((entry) => entry.status === 'rejected')
|
||||
.every(
|
||||
(entry) =>
|
||||
entry.reason instanceof ApprovalRequestVersionConflictError ||
|
||||
entry.reason instanceof ApprovalRequestStateConflictError,
|
||||
),
|
||||
);
|
||||
|
||||
const approved = await first.repository.findById('approval-1');
|
||||
const winnerDecisionId = approved.decisionId;
|
||||
const winnerDecision =
|
||||
winnerDecisionId === 'decision-1'
|
||||
? decisionInput()
|
||||
: decisionInput({ decisionId: 'decision-racer', reasonCode: 'racer' });
|
||||
assert.deepEqual(await first.service.decide(winnerDecision), approved);
|
||||
|
||||
const consumptions = await Promise.allSettled([
|
||||
first.service.consume(consumptionInput()),
|
||||
secondService.consume(
|
||||
consumptionInput({
|
||||
consumptionId: 'consumption-racer',
|
||||
dispatchId: 'dispatch-racer',
|
||||
}),
|
||||
),
|
||||
]);
|
||||
assert.equal(
|
||||
consumptions.filter((entry) => entry.status === 'fulfilled').length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
consumptions.filter((entry) => entry.status === 'rejected').length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await first.database
|
||||
.getQueryInterface()
|
||||
.select(null, APPROVED_ACTION_DISPATCH_TABLE)
|
||||
).length,
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
test('dispatch collisions roll back consumption and leave approval reusable', async (t) => {
|
||||
const { database, service, repository } = await setup(t);
|
||||
await service.create(createInput());
|
||||
await service.decide(decisionInput());
|
||||
await service.create(
|
||||
createInput({
|
||||
id: 'approval-2',
|
||||
action: action({ actionRef: 'planned-action-2' }),
|
||||
}),
|
||||
);
|
||||
await service.decide(
|
||||
decisionInput({ requestId: 'approval-2', decisionId: 'decision-2' }),
|
||||
);
|
||||
await service.consume(consumptionInput());
|
||||
await assert.rejects(
|
||||
service.consume(
|
||||
consumptionInput({
|
||||
requestId: 'approval-2',
|
||||
consumptionId: 'consumption-2',
|
||||
dispatchId: 'dispatch-1',
|
||||
action: action({ actionRef: 'planned-action-2' }),
|
||||
}),
|
||||
),
|
||||
ApprovalMutationConflictError,
|
||||
);
|
||||
assert.equal((await repository.findById('approval-2')).state, 'approved');
|
||||
assert.equal(
|
||||
(
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.select(null, APPROVED_ACTION_DISPATCH_TABLE)
|
||||
).length,
|
||||
1,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,645 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const { QueryTypes, Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const {
|
||||
projectPolicyMigration,
|
||||
} = require('../../back/migrations/0017-project-policy');
|
||||
const {
|
||||
APPROVAL_REQUEST_TABLE,
|
||||
APPROVED_ACTION_DISPATCH_TABLE,
|
||||
approvalRequestMigration,
|
||||
} = require('../../back/migrations/0020-approval-requests');
|
||||
const {
|
||||
APPROVED_ACTION_DISPATCH_EXECUTION_DUE_INDEX,
|
||||
APPROVED_ACTION_DISPATCH_EXECUTION_LEASE_INDEX,
|
||||
APPROVED_ACTION_DISPATCH_EXECUTION_PROJECT_INDEX,
|
||||
APPROVED_ACTION_DISPATCH_EXECUTION_TABLE,
|
||||
approvedActionDispatchExecutionMigration,
|
||||
} = require('../../back/migrations/0021-approved-action-dispatch-executions');
|
||||
const {
|
||||
approvedActionRecoveryMigration,
|
||||
} = require('../../back/migrations/0022-approved-action-recovery');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeApprovedActionDispatchRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/approvedActionDispatchRepository');
|
||||
const {
|
||||
LegacySequelizeApprovalRequestRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/approvalRequestRepository');
|
||||
const {
|
||||
LegacySequelizeProjectPolicyRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/projectPolicyRepository');
|
||||
const {
|
||||
ApprovalRequestService,
|
||||
} = require('../../back/runtime/application/approvalRequestService');
|
||||
const {
|
||||
ProjectPolicyEngine,
|
||||
} = require('../../back/runtime/application/projectPolicyEngine');
|
||||
const {
|
||||
ApprovedActionDispatchBindingConflictError,
|
||||
ApprovedActionDispatchFenceRejectedError,
|
||||
ApprovedActionDispatchRepositoryError,
|
||||
} = require('../../back/runtime/domain/approvedActionDispatchExecution');
|
||||
const {
|
||||
ApprovalUnavailableError,
|
||||
} = require('../../back/runtime/domain/approvalRequest');
|
||||
|
||||
const PROJECT_ID = 'default';
|
||||
const AGENT = Object.freeze({ type: 'agent', id: 'agent-1' });
|
||||
const OWNER = Object.freeze({ type: 'user', id: 'owner-1' });
|
||||
const SYSTEM = Object.freeze({ type: 'system', id: 'approval-dispatcher' });
|
||||
const BASE_TIME = 100_000;
|
||||
|
||||
function action(name, digestCharacter = 'a') {
|
||||
return {
|
||||
permission: 'tool.call:filesystem.write',
|
||||
actionType: 'tool_call',
|
||||
actionRef: `planned-${name}`,
|
||||
actionDigest: digestCharacter.repeat(64),
|
||||
previewDigest: 'f'.repeat(64),
|
||||
};
|
||||
}
|
||||
|
||||
async function migrate(database, migrations) {
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations,
|
||||
logger: { info() {} },
|
||||
});
|
||||
}
|
||||
|
||||
async function bind(policyRepository, subject, role, mutationId) {
|
||||
await policyRepository.append({
|
||||
expectedCurrentVersion: 0,
|
||||
binding: {
|
||||
projectId: PROJECT_ID,
|
||||
subject,
|
||||
version: 1,
|
||||
state: 'active',
|
||||
role,
|
||||
mutationId,
|
||||
changedBy: OWNER,
|
||||
createdAtMs: BASE_TIME - 100,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function setup(t, storage = ':memory:') {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
await migrate(database, [
|
||||
projectPolicyMigration,
|
||||
approvalRequestMigration,
|
||||
approvedActionDispatchExecutionMigration,
|
||||
approvedActionRecoveryMigration,
|
||||
]);
|
||||
const policyRepository = new LegacySequelizeProjectPolicyRepository(database);
|
||||
await bind(policyRepository, OWNER, 'owner', 'bind-owner');
|
||||
await bind(policyRepository, AGENT, 'operator', 'bind-agent');
|
||||
const approvalRepository = new LegacySequelizeApprovalRequestRepository(
|
||||
database,
|
||||
);
|
||||
return {
|
||||
database,
|
||||
approvalRepository,
|
||||
approvalService: new ApprovalRequestService(
|
||||
approvalRepository,
|
||||
new ProjectPolicyEngine(policyRepository),
|
||||
),
|
||||
executionRepository: new LegacySequelizeApprovedActionDispatchRepository(
|
||||
database,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
async function prepareDispatch(service, name, offset = 0, digest = 'a') {
|
||||
const requestedAtMs = BASE_TIME + offset;
|
||||
const binding = action(name, digest);
|
||||
await service.create({
|
||||
id: `approval-${name}`,
|
||||
projectId: PROJECT_ID,
|
||||
action: binding,
|
||||
risk: 'high',
|
||||
requestedBy: AGENT,
|
||||
requestedAtMs,
|
||||
expiresAtMs: requestedAtMs + 60_000,
|
||||
});
|
||||
await service.decide({
|
||||
requestId: `approval-${name}`,
|
||||
expectedVersion: 1,
|
||||
decisionId: `decision-${name}`,
|
||||
decision: 'approved',
|
||||
reasonCode: 'reviewed_action',
|
||||
decidedBy: OWNER,
|
||||
decidedAtMs: requestedAtMs + 10,
|
||||
});
|
||||
const consumed = await service.consume({
|
||||
requestId: `approval-${name}`,
|
||||
expectedVersion: 2,
|
||||
consumptionId: `consumption-${name}`,
|
||||
dispatchId: `dispatch-${name}`,
|
||||
action: binding,
|
||||
requestedBy: AGENT,
|
||||
consumedBy: SYSTEM,
|
||||
consumedAtMs: requestedAtMs + 20,
|
||||
});
|
||||
return consumed.dispatch;
|
||||
}
|
||||
|
||||
function claimInput(dispatchId, overrides = {}) {
|
||||
return {
|
||||
dispatchId,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'lease-1',
|
||||
nowMs: BASE_TIME + 1_000,
|
||||
leaseDurationMs: 1_000,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('0021 backfills immutable dispatches and owns bounded execution indexes', async (t) => {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
await migrate(database, [projectPolicyMigration, approvalRequestMigration]);
|
||||
const queryInterface = database.getQueryInterface();
|
||||
await queryInterface.bulkInsert(APPROVAL_REQUEST_TABLE, [
|
||||
{
|
||||
id: 'approval-backfill',
|
||||
project_id: PROJECT_ID,
|
||||
version: 3,
|
||||
state: 'consumed',
|
||||
permission: 'tool.call:filesystem.write',
|
||||
action_type: 'tool_call',
|
||||
action_ref: 'planned-backfill',
|
||||
action_digest: 'a'.repeat(64),
|
||||
preview_digest: 'f'.repeat(64),
|
||||
risk: 'high',
|
||||
requested_by_type: 'agent',
|
||||
requested_by_id: AGENT.id,
|
||||
requested_at_ms: BASE_TIME,
|
||||
expires_at_ms: BASE_TIME + 60_000,
|
||||
decision_id: 'decision-backfill',
|
||||
decision: 'approved',
|
||||
decision_reason_code: 'reviewed_action',
|
||||
decided_by_type: 'user',
|
||||
decided_by_id: OWNER.id,
|
||||
decided_at_ms: BASE_TIME + 10,
|
||||
consumption_id: 'consumption-backfill',
|
||||
dispatch_id: 'dispatch-backfill',
|
||||
consumed_by_type: 'system',
|
||||
consumed_by_id: SYSTEM.id,
|
||||
consumed_at_ms: BASE_TIME + 20,
|
||||
},
|
||||
]);
|
||||
await queryInterface.bulkInsert(APPROVED_ACTION_DISPATCH_TABLE, [
|
||||
{
|
||||
id: 'dispatch-backfill',
|
||||
approval_request_id: 'approval-backfill',
|
||||
approval_request_version: 3,
|
||||
project_id: PROJECT_ID,
|
||||
state: 'pending',
|
||||
permission: 'tool.call:filesystem.write',
|
||||
action_type: 'tool_call',
|
||||
action_ref: 'planned-backfill',
|
||||
action_digest: 'a'.repeat(64),
|
||||
preview_digest: 'f'.repeat(64),
|
||||
requested_by_type: 'agent',
|
||||
requested_by_id: AGENT.id,
|
||||
consumed_by_type: 'system',
|
||||
consumed_by_id: SYSTEM.id,
|
||||
created_at_ms: BASE_TIME + 20,
|
||||
},
|
||||
]);
|
||||
await migrate(database, [approvedActionDispatchExecutionMigration]);
|
||||
|
||||
const rows = await queryInterface.select(
|
||||
null,
|
||||
APPROVED_ACTION_DISPATCH_EXECUTION_TABLE,
|
||||
);
|
||||
assert.equal(rows.length, 1);
|
||||
assert.equal(rows[0].dispatch_id, 'dispatch-backfill');
|
||||
assert.equal(rows[0].status, 'pending');
|
||||
assert.equal(rows[0].version, 0);
|
||||
assert.equal(rows[0].eligible_at_ms, BASE_TIME + 20);
|
||||
const indexes = new Set(
|
||||
(
|
||||
await queryInterface.showIndex(APPROVED_ACTION_DISPATCH_EXECUTION_TABLE)
|
||||
).map((index) => index.name),
|
||||
);
|
||||
for (const name of [
|
||||
APPROVED_ACTION_DISPATCH_EXECUTION_DUE_INDEX,
|
||||
APPROVED_ACTION_DISPATCH_EXECUTION_PROJECT_INDEX,
|
||||
APPROVED_ACTION_DISPATCH_EXECUTION_LEASE_INDEX,
|
||||
]) {
|
||||
assert.ok(indexes.has(name));
|
||||
}
|
||||
});
|
||||
|
||||
test('approval consumption atomically creates the immutable dispatch and execution baseline', async (t) => {
|
||||
const { database, approvalService, executionRepository } = await setup(t);
|
||||
const dispatch = await prepareDispatch(approvalService, 'atomic');
|
||||
const snapshot = await executionRepository.findById(dispatch.id);
|
||||
assert.equal(snapshot.execution.status, 'pending');
|
||||
assert.equal(snapshot.execution.version, 0);
|
||||
assert.equal(snapshot.execution.attemptCount, 0);
|
||||
assert.equal(snapshot.execution.eligibleAtMs, dispatch.createdAtMs);
|
||||
|
||||
await approvalService.create({
|
||||
id: 'approval-rollback',
|
||||
projectId: PROJECT_ID,
|
||||
action: action('rollback', 'b'),
|
||||
risk: 'high',
|
||||
requestedBy: AGENT,
|
||||
requestedAtMs: BASE_TIME + 100,
|
||||
expiresAtMs: BASE_TIME + 60_100,
|
||||
});
|
||||
await approvalService.decide({
|
||||
requestId: 'approval-rollback',
|
||||
expectedVersion: 1,
|
||||
decisionId: 'decision-rollback',
|
||||
decision: 'approved',
|
||||
reasonCode: 'reviewed_action',
|
||||
decidedBy: OWNER,
|
||||
decidedAtMs: BASE_TIME + 110,
|
||||
});
|
||||
await database.query(
|
||||
`CREATE TRIGGER fail_approved_execution_insert
|
||||
BEFORE INSERT ON "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}"
|
||||
BEGIN SELECT RAISE(ABORT, 'forced execution insert failure'); END`,
|
||||
);
|
||||
await assert.rejects(
|
||||
approvalService.consume({
|
||||
requestId: 'approval-rollback',
|
||||
expectedVersion: 2,
|
||||
consumptionId: 'consumption-rollback',
|
||||
dispatchId: 'dispatch-rollback',
|
||||
action: action('rollback', 'b'),
|
||||
requestedBy: AGENT,
|
||||
consumedBy: SYSTEM,
|
||||
consumedAtMs: BASE_TIME + 120,
|
||||
}),
|
||||
ApprovalUnavailableError,
|
||||
);
|
||||
assert.equal(await executionRepository.findById('dispatch-rollback'), null);
|
||||
assert.equal(
|
||||
(
|
||||
await database.query(
|
||||
`SELECT state FROM "${APPROVAL_REQUEST_TABLE}"
|
||||
WHERE id = 'approval-rollback'`,
|
||||
{ type: QueryTypes.SELECT },
|
||||
)
|
||||
)[0].state,
|
||||
'approved',
|
||||
);
|
||||
});
|
||||
|
||||
test('lists a bounded stable due page without loading executing or future work', async (t) => {
|
||||
const { approvalService, executionRepository } = await setup(t);
|
||||
await prepareDispatch(approvalService, 'page-a', 0, 'a');
|
||||
await prepareDispatch(approvalService, 'page-b', 10, 'b');
|
||||
await prepareDispatch(approvalService, 'page-c', 20, 'c');
|
||||
const first = await executionRepository.listDue({
|
||||
nowMs: BASE_TIME + 1_000,
|
||||
limit: 2,
|
||||
});
|
||||
assert.deepEqual(
|
||||
first.dispatches.map((entry) => entry.dispatch.id),
|
||||
['dispatch-page-a', 'dispatch-page-b'],
|
||||
);
|
||||
assert.equal(first.truncated, true);
|
||||
const second = await executionRepository.listDue({
|
||||
nowMs: BASE_TIME + 1_000,
|
||||
limit: 2,
|
||||
cursor: first.nextCursor,
|
||||
});
|
||||
assert.deepEqual(
|
||||
second.dispatches.map((entry) => entry.dispatch.id),
|
||||
['dispatch-page-c'],
|
||||
);
|
||||
assert.equal(second.truncated, false);
|
||||
});
|
||||
|
||||
test('claims idempotently and lets only an expired pre-start lease be taken over', async (t) => {
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-action-'));
|
||||
t.after(() => fs.rm(directory, { recursive: true, force: true }));
|
||||
const storage = path.join(directory, 'database.sqlite');
|
||||
const first = await setup(t, storage);
|
||||
await prepareDispatch(first.approvalService, 'race');
|
||||
const secondDatabase = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => secondDatabase.close());
|
||||
const secondRepository = new LegacySequelizeApprovedActionDispatchRepository(
|
||||
secondDatabase,
|
||||
);
|
||||
|
||||
const results = await Promise.all([
|
||||
first.executionRepository.claim(claimInput('dispatch-race')),
|
||||
secondRepository.claim(
|
||||
claimInput('dispatch-race', {
|
||||
owner: 'dispatcher-2',
|
||||
leaseToken: 'lease-2',
|
||||
}),
|
||||
),
|
||||
]);
|
||||
assert.equal(
|
||||
results.filter((result) => result.status === 'claimed').length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
results.filter((result) => result.status === 'leased').length,
|
||||
1,
|
||||
);
|
||||
const winner = results.find((result) => result.status === 'claimed');
|
||||
assert.equal(
|
||||
(
|
||||
await first.executionRepository.claim(
|
||||
claimInput('dispatch-race', {
|
||||
owner: winner.snapshot.execution.leaseOwner,
|
||||
leaseToken: winner.snapshot.execution.leaseToken,
|
||||
}),
|
||||
)
|
||||
).status,
|
||||
'claimed',
|
||||
);
|
||||
|
||||
const takeover = await secondRepository.claim(
|
||||
claimInput('dispatch-race', {
|
||||
owner: 'dispatcher-3',
|
||||
leaseToken: 'lease-3',
|
||||
nowMs: BASE_TIME + 2_000,
|
||||
}),
|
||||
);
|
||||
assert.equal(takeover.status, 'claimed');
|
||||
assert.equal(takeover.snapshot.execution.attemptCount, 2);
|
||||
assert.equal(takeover.snapshot.execution.leaseOwner, 'dispatcher-3');
|
||||
});
|
||||
|
||||
test('start barrier binds the exact approval and action digest before side effects', async (t) => {
|
||||
const { approvalService, executionRepository } = await setup(t);
|
||||
const dispatch = await prepareDispatch(approvalService, 'start');
|
||||
const claimed = await executionRepository.claim(
|
||||
claimInput(dispatch.id, { nowMs: BASE_TIME + 1_000 }),
|
||||
);
|
||||
await assert.rejects(
|
||||
executionRepository.start({
|
||||
dispatchId: dispatch.id,
|
||||
approvalRequestId: dispatch.approvalRequestId,
|
||||
actionDigest: 'e'.repeat(64),
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'lease-1',
|
||||
expectedVersion: claimed.snapshot.execution.version,
|
||||
startedAtMs: BASE_TIME + 1_100,
|
||||
}),
|
||||
ApprovedActionDispatchBindingConflictError,
|
||||
);
|
||||
const started = await executionRepository.start({
|
||||
dispatchId: dispatch.id,
|
||||
approvalRequestId: dispatch.approvalRequestId,
|
||||
actionDigest: dispatch.action.actionDigest,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'lease-1',
|
||||
expectedVersion: claimed.snapshot.execution.version,
|
||||
startedAtMs: BASE_TIME + 1_100,
|
||||
});
|
||||
assert.equal(started.execution.status, 'executing');
|
||||
assert.equal(started.execution.eligibleAtMs, null);
|
||||
assert.deepEqual(
|
||||
await executionRepository.start({
|
||||
dispatchId: dispatch.id,
|
||||
approvalRequestId: dispatch.approvalRequestId,
|
||||
actionDigest: dispatch.action.actionDigest,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'lease-1',
|
||||
expectedVersion: claimed.snapshot.execution.version,
|
||||
startedAtMs: BASE_TIME + 1_100,
|
||||
}),
|
||||
started,
|
||||
);
|
||||
});
|
||||
|
||||
test('executing lease expiry requires recovery and can never auto-take over', async (t) => {
|
||||
const { approvalService, executionRepository } = await setup(t);
|
||||
const dispatch = await prepareDispatch(approvalService, 'recovery');
|
||||
const claimed = await executionRepository.claim(
|
||||
claimInput(dispatch.id, {
|
||||
nowMs: BASE_TIME + 1_000,
|
||||
leaseDurationMs: 100,
|
||||
}),
|
||||
);
|
||||
const started = await executionRepository.start({
|
||||
dispatchId: dispatch.id,
|
||||
approvalRequestId: dispatch.approvalRequestId,
|
||||
actionDigest: dispatch.action.actionDigest,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'lease-1',
|
||||
expectedVersion: claimed.snapshot.execution.version,
|
||||
startedAtMs: BASE_TIME + 1_010,
|
||||
});
|
||||
const takeover = await executionRepository.claim(
|
||||
claimInput(dispatch.id, {
|
||||
owner: 'dispatcher-2',
|
||||
leaseToken: 'lease-2',
|
||||
nowMs: BASE_TIME + 1_100,
|
||||
}),
|
||||
);
|
||||
assert.equal(takeover.status, 'recovery_required');
|
||||
assert.equal(takeover.snapshot.execution.leaseOwner, 'dispatcher-1');
|
||||
assert.equal(
|
||||
(
|
||||
await executionRepository.listDue({
|
||||
nowMs: BASE_TIME + 10_000,
|
||||
limit: 64,
|
||||
})
|
||||
).dispatches.length,
|
||||
0,
|
||||
);
|
||||
const renewed = await executionRepository.renew({
|
||||
dispatchId: dispatch.id,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'lease-1',
|
||||
expectedVersion: started.execution.version,
|
||||
nowMs: BASE_TIME + 1_200,
|
||||
leaseDurationMs: 1_000,
|
||||
});
|
||||
assert.equal(renewed.execution.status, 'executing');
|
||||
});
|
||||
|
||||
test('pre-effect failures retry safely but exhaust into a terminal block', async (t) => {
|
||||
const { database, approvalService, executionRepository } = await setup(t);
|
||||
const dispatch = await prepareDispatch(approvalService, 'retry');
|
||||
await database.query(
|
||||
`UPDATE "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}"
|
||||
SET max_attempts = 2
|
||||
WHERE dispatch_id = :dispatchId`,
|
||||
{ replacements: { dispatchId: dispatch.id } },
|
||||
);
|
||||
const first = await executionRepository.claim(claimInput(dispatch.id));
|
||||
const retry = await executionRepository.releaseBeforeStart({
|
||||
dispatchId: dispatch.id,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'lease-1',
|
||||
expectedVersion: first.snapshot.execution.version,
|
||||
resultMutationId: 'preflight-result-1',
|
||||
resultCode: 'executor_unavailable',
|
||||
atMs: BASE_TIME + 1_010,
|
||||
retryAtMs: BASE_TIME + 1_020,
|
||||
});
|
||||
assert.equal(retry.execution.status, 'retry_wait');
|
||||
assert.deepEqual(
|
||||
await executionRepository.releaseBeforeStart({
|
||||
dispatchId: dispatch.id,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'lease-1',
|
||||
expectedVersion: first.snapshot.execution.version,
|
||||
resultMutationId: 'preflight-result-1',
|
||||
resultCode: 'executor_unavailable',
|
||||
atMs: BASE_TIME + 1_010,
|
||||
retryAtMs: BASE_TIME + 1_020,
|
||||
}),
|
||||
retry,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await executionRepository.claim(
|
||||
claimInput(dispatch.id, { nowMs: BASE_TIME + 1_019 }),
|
||||
)
|
||||
).status,
|
||||
'not_due',
|
||||
);
|
||||
const second = await executionRepository.claim(
|
||||
claimInput(dispatch.id, {
|
||||
leaseToken: 'lease-2',
|
||||
nowMs: BASE_TIME + 1_020,
|
||||
}),
|
||||
);
|
||||
const blocked = await executionRepository.releaseBeforeStart({
|
||||
dispatchId: dispatch.id,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'lease-2',
|
||||
expectedVersion: second.snapshot.execution.version,
|
||||
resultMutationId: 'preflight-result-2',
|
||||
resultCode: 'executor_unavailable',
|
||||
atMs: BASE_TIME + 1_030,
|
||||
retryAtMs: BASE_TIME + 1_040,
|
||||
});
|
||||
assert.equal(blocked.execution.status, 'blocked');
|
||||
assert.equal(blocked.execution.completedAtMs, BASE_TIME + 1_030);
|
||||
});
|
||||
|
||||
test('completion is fenced, idempotent, and indeterminate evidence blocks replay', async (t) => {
|
||||
const { approvalService, executionRepository } = await setup(t);
|
||||
const dispatch = await prepareDispatch(approvalService, 'complete');
|
||||
const claimed = await executionRepository.claim(claimInput(dispatch.id));
|
||||
const started = await executionRepository.start({
|
||||
dispatchId: dispatch.id,
|
||||
approvalRequestId: dispatch.approvalRequestId,
|
||||
actionDigest: dispatch.action.actionDigest,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'lease-1',
|
||||
expectedVersion: claimed.snapshot.execution.version,
|
||||
startedAtMs: BASE_TIME + 1_010,
|
||||
});
|
||||
await assert.rejects(
|
||||
executionRepository.complete({
|
||||
dispatchId: dispatch.id,
|
||||
owner: 'dispatcher-2',
|
||||
leaseToken: 'lease-2',
|
||||
expectedVersion: started.execution.version,
|
||||
resultMutationId: 'completion-wrong',
|
||||
outcome: 'succeeded',
|
||||
resultCode: 'ok',
|
||||
completedAtMs: BASE_TIME + 1_100,
|
||||
}),
|
||||
ApprovedActionDispatchFenceRejectedError,
|
||||
);
|
||||
const command = {
|
||||
dispatchId: dispatch.id,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'lease-1',
|
||||
expectedVersion: started.execution.version,
|
||||
resultMutationId: 'completion-1',
|
||||
outcome: 'indeterminate',
|
||||
resultCode: 'transport_lost_after_start',
|
||||
completedAtMs: BASE_TIME + 2_100,
|
||||
};
|
||||
const completed = await executionRepository.complete(command);
|
||||
assert.equal(completed.execution.status, 'blocked');
|
||||
assert.deepEqual(await executionRepository.complete(command), completed);
|
||||
assert.equal(
|
||||
(await executionRepository.claim(claimInput(dispatch.id))).status,
|
||||
'blocked',
|
||||
);
|
||||
});
|
||||
|
||||
test('a successful result can arrive after lease expiry without re-executing', async (t) => {
|
||||
const { approvalService, executionRepository } = await setup(t);
|
||||
const dispatch = await prepareDispatch(approvalService, 'success');
|
||||
const claimed = await executionRepository.claim(
|
||||
claimInput(dispatch.id, {
|
||||
nowMs: BASE_TIME + 1_000,
|
||||
leaseDurationMs: 100,
|
||||
}),
|
||||
);
|
||||
const started = await executionRepository.start({
|
||||
dispatchId: dispatch.id,
|
||||
approvalRequestId: dispatch.approvalRequestId,
|
||||
actionDigest: dispatch.action.actionDigest,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'lease-1',
|
||||
expectedVersion: claimed.snapshot.execution.version,
|
||||
startedAtMs: BASE_TIME + 1_010,
|
||||
});
|
||||
const command = {
|
||||
dispatchId: dispatch.id,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'lease-1',
|
||||
expectedVersion: started.execution.version,
|
||||
resultMutationId: 'completion-success',
|
||||
outcome: 'succeeded',
|
||||
resultCode: 'ok',
|
||||
completedAtMs: BASE_TIME + 2_000,
|
||||
};
|
||||
const completed = await executionRepository.complete(command);
|
||||
assert.equal(completed.execution.status, 'succeeded');
|
||||
assert.deepEqual(await executionRepository.complete(command), completed);
|
||||
assert.equal(
|
||||
(await executionRepository.claim(claimInput(dispatch.id))).status,
|
||||
'succeeded',
|
||||
);
|
||||
});
|
||||
|
||||
test('missing execution state is corruption rather than an absent dispatch', async (t) => {
|
||||
const { database, approvalService, executionRepository } = await setup(t);
|
||||
const dispatch = await prepareDispatch(approvalService, 'corrupt');
|
||||
await database.query('PRAGMA foreign_keys = OFF');
|
||||
await database.query(
|
||||
`DELETE FROM "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}"
|
||||
WHERE dispatch_id = :dispatchId`,
|
||||
{ replacements: { dispatchId: dispatch.id } },
|
||||
);
|
||||
await database.query('PRAGMA foreign_keys = ON');
|
||||
await assert.rejects(
|
||||
executionRepository.findById(dispatch.id),
|
||||
ApprovedActionDispatchRepositoryError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,349 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
ApprovedActionDispatcher,
|
||||
} = require('../../back/runtime/application/approvedActionDispatcher');
|
||||
|
||||
function pendingSnapshot(name = 'one') {
|
||||
return {
|
||||
dispatch: {
|
||||
id: `dispatch-${name}`,
|
||||
approvalRequestId: `approval-${name}`,
|
||||
approvalRequestVersion: 3,
|
||||
projectId: 'default',
|
||||
state: 'pending',
|
||||
action: {
|
||||
permission: 'tool.call:filesystem.write',
|
||||
actionType: 'tool_call',
|
||||
actionRef: `planned-${name}`,
|
||||
actionDigest: 'a'.repeat(64),
|
||||
previewDigest: 'f'.repeat(64),
|
||||
},
|
||||
requestedBy: { type: 'agent', id: 'agent-1' },
|
||||
consumedBy: { type: 'system', id: 'approval-dispatcher' },
|
||||
createdAtMs: 100,
|
||||
},
|
||||
execution: {
|
||||
dispatchId: `dispatch-${name}`,
|
||||
projectId: 'default',
|
||||
status: 'pending',
|
||||
version: 0,
|
||||
attemptCount: 0,
|
||||
maxAttempts: 5,
|
||||
eligibleAtMs: 100,
|
||||
nextAttemptAtMs: null,
|
||||
leaseOwner: null,
|
||||
leaseToken: null,
|
||||
leaseExpiresAtMs: null,
|
||||
startedAtMs: null,
|
||||
resultMutationId: null,
|
||||
lastResultCode: null,
|
||||
completedAtMs: null,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 100,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function fakeRepository(options = {}) {
|
||||
const calls = [];
|
||||
const initial = pendingSnapshot();
|
||||
return {
|
||||
calls,
|
||||
async listDue(query) {
|
||||
calls.push(['list', query]);
|
||||
return {
|
||||
dispatches: [initial],
|
||||
truncated: false,
|
||||
};
|
||||
},
|
||||
async claim(command) {
|
||||
calls.push(['claim', command]);
|
||||
return {
|
||||
status: 'claimed',
|
||||
snapshot: {
|
||||
dispatch: initial.dispatch,
|
||||
execution: {
|
||||
...initial.execution,
|
||||
status: 'leased',
|
||||
version: 1,
|
||||
attemptCount: 1,
|
||||
eligibleAtMs: command.nowMs + command.leaseDurationMs,
|
||||
leaseOwner: command.owner,
|
||||
leaseToken: command.leaseToken,
|
||||
leaseExpiresAtMs: command.nowMs + command.leaseDurationMs,
|
||||
updatedAtMs: command.nowMs,
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
async start(command) {
|
||||
calls.push(['start', command]);
|
||||
if (options.startError) throw new Error('start unavailable');
|
||||
const claim = calls.find((entry) => entry[0] === 'claim')[1];
|
||||
return {
|
||||
dispatch: initial.dispatch,
|
||||
execution: {
|
||||
...initial.execution,
|
||||
status: 'executing',
|
||||
version: 2,
|
||||
attemptCount: 1,
|
||||
eligibleAtMs: null,
|
||||
leaseOwner: claim.owner,
|
||||
leaseToken: claim.leaseToken,
|
||||
leaseExpiresAtMs: claim.nowMs + claim.leaseDurationMs,
|
||||
startedAtMs: command.startedAtMs,
|
||||
updatedAtMs: command.startedAtMs,
|
||||
},
|
||||
};
|
||||
},
|
||||
async releaseBeforeStart(command) {
|
||||
calls.push(['release', command]);
|
||||
return {
|
||||
dispatch: initial.dispatch,
|
||||
execution: {
|
||||
...initial.execution,
|
||||
status: command.retryAtMs === undefined ? 'blocked' : 'retry_wait',
|
||||
version: 2,
|
||||
attemptCount: 1,
|
||||
eligibleAtMs: command.retryAtMs ?? null,
|
||||
nextAttemptAtMs: command.retryAtMs ?? null,
|
||||
resultMutationId: command.resultMutationId,
|
||||
lastResultCode: command.resultCode,
|
||||
completedAtMs: command.retryAtMs === undefined ? command.atMs : null,
|
||||
updatedAtMs: command.atMs,
|
||||
},
|
||||
};
|
||||
},
|
||||
async complete(command) {
|
||||
calls.push(['complete', command]);
|
||||
if (options.completeError) throw new Error('completion unavailable');
|
||||
return {
|
||||
dispatch: initial.dispatch,
|
||||
execution: {
|
||||
...initial.execution,
|
||||
status:
|
||||
command.outcome === 'indeterminate' ? 'blocked' : command.outcome,
|
||||
version: 3,
|
||||
attemptCount: 1,
|
||||
eligibleAtMs: null,
|
||||
startedAtMs: 103,
|
||||
resultMutationId: command.resultMutationId,
|
||||
lastResultCode: command.resultCode,
|
||||
completedAtMs: command.completedAtMs,
|
||||
updatedAtMs: command.completedAtMs,
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function deterministicOptions() {
|
||||
let now = 100;
|
||||
let id = 0;
|
||||
return {
|
||||
owner: 'dispatcher-1',
|
||||
leaseDurationMs: 1_000,
|
||||
retryBaseMs: 10,
|
||||
retryMaxMs: 100,
|
||||
clock: () => ++now,
|
||||
createId: () => `mutation-${++id}`,
|
||||
};
|
||||
}
|
||||
|
||||
function handler(overrides = {}) {
|
||||
return {
|
||||
actionType: 'tool_call',
|
||||
async inspect(dispatch) {
|
||||
return { status: 'ready', actionDigest: dispatch.action.actionDigest };
|
||||
},
|
||||
async execute() {
|
||||
return { outcome: 'succeeded', resultCode: 'ok' };
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('persists the start barrier before invoking a successful handler', async () => {
|
||||
const repository = fakeRepository();
|
||||
const observed = [];
|
||||
const dispatcher = new ApprovedActionDispatcher(
|
||||
repository,
|
||||
[
|
||||
handler({
|
||||
async inspect(dispatch) {
|
||||
observed.push(['inspect', dispatch.id]);
|
||||
return {
|
||||
status: 'ready',
|
||||
actionDigest: dispatch.action.actionDigest,
|
||||
};
|
||||
},
|
||||
async execute(context) {
|
||||
observed.push(['execute', context.dispatch.id]);
|
||||
assert.equal(context.idempotencyKey, 'dispatch-one');
|
||||
assert.equal(context.execution.status, 'executing');
|
||||
assert.equal(context.fence.owner, 'dispatcher-1');
|
||||
assert.equal(context.fence.version, 2);
|
||||
return { outcome: 'succeeded', resultCode: 'ok' };
|
||||
},
|
||||
}),
|
||||
],
|
||||
deterministicOptions(),
|
||||
);
|
||||
const summary = await dispatcher.dispatchBatch({ limit: 1 });
|
||||
assert.deepEqual(
|
||||
repository.calls.map((entry) => entry[0]),
|
||||
['list', 'claim', 'start', 'complete'],
|
||||
);
|
||||
assert.deepEqual(observed, [
|
||||
['inspect', 'dispatch-one'],
|
||||
['execute', 'dispatch-one'],
|
||||
]);
|
||||
assert.equal(summary.claimed, 1);
|
||||
assert.equal(summary.started, 1);
|
||||
assert.equal(summary.succeeded, 1);
|
||||
assert.equal(summary.blocked, 0);
|
||||
});
|
||||
|
||||
test('missing handlers and inspection failures retry only before start', async () => {
|
||||
for (const handlers of [
|
||||
[],
|
||||
[
|
||||
handler({
|
||||
async inspect() {
|
||||
throw new Error('temporary inspection failure');
|
||||
},
|
||||
}),
|
||||
],
|
||||
]) {
|
||||
const repository = fakeRepository();
|
||||
const dispatcher = new ApprovedActionDispatcher(
|
||||
repository,
|
||||
handlers,
|
||||
deterministicOptions(),
|
||||
);
|
||||
const summary = await dispatcher.dispatchBatch();
|
||||
assert.deepEqual(
|
||||
repository.calls.map((entry) => entry[0]),
|
||||
['list', 'claim', 'release'],
|
||||
);
|
||||
assert.equal(summary.retrying, 1);
|
||||
assert.equal(summary.started, 0);
|
||||
}
|
||||
});
|
||||
|
||||
test('digest drift and explicit inspection blocks never invoke execute', async () => {
|
||||
for (const inspection of [
|
||||
{ status: 'ready', actionDigest: 'b'.repeat(64) },
|
||||
{ status: 'blocked', resultCode: 'plan_revoked' },
|
||||
]) {
|
||||
let executed = false;
|
||||
const repository = fakeRepository();
|
||||
const dispatcher = new ApprovedActionDispatcher(
|
||||
repository,
|
||||
[
|
||||
handler({
|
||||
async inspect() {
|
||||
return inspection;
|
||||
},
|
||||
async execute() {
|
||||
executed = true;
|
||||
return { outcome: 'succeeded', resultCode: 'ok' };
|
||||
},
|
||||
}),
|
||||
],
|
||||
deterministicOptions(),
|
||||
);
|
||||
const summary = await dispatcher.dispatchBatch();
|
||||
assert.equal(executed, false);
|
||||
assert.equal(summary.blocked, 1);
|
||||
assert.deepEqual(
|
||||
repository.calls.map((entry) => entry[0]),
|
||||
['list', 'claim', 'release'],
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('handler exceptions after start become indeterminate terminal evidence', async () => {
|
||||
const repository = fakeRepository();
|
||||
const dispatcher = new ApprovedActionDispatcher(
|
||||
repository,
|
||||
[
|
||||
handler({
|
||||
async execute() {
|
||||
throw new Error('transport disappeared after invocation');
|
||||
},
|
||||
}),
|
||||
],
|
||||
deterministicOptions(),
|
||||
);
|
||||
const summary = await dispatcher.dispatchBatch();
|
||||
const completion = repository.calls.find(
|
||||
(entry) => entry[0] === 'complete',
|
||||
)[1];
|
||||
assert.equal(completion.outcome, 'indeterminate');
|
||||
assert.equal(completion.resultCode, 'handler_failed_after_start');
|
||||
assert.equal(summary.blocked, 1);
|
||||
assert.equal(summary.retrying, 0);
|
||||
});
|
||||
|
||||
test('a completion persistence failure reports recovery instead of retrying execute', async () => {
|
||||
let executions = 0;
|
||||
const repository = fakeRepository({ completeError: true });
|
||||
const dispatcher = new ApprovedActionDispatcher(
|
||||
repository,
|
||||
[
|
||||
handler({
|
||||
async execute() {
|
||||
executions += 1;
|
||||
return { outcome: 'succeeded', resultCode: 'ok' };
|
||||
},
|
||||
}),
|
||||
],
|
||||
deterministicOptions(),
|
||||
);
|
||||
const summary = await dispatcher.dispatchBatch();
|
||||
assert.equal(executions, 1);
|
||||
assert.equal(summary.unavailable, 1);
|
||||
assert.equal(summary.recoveryRequired, 1);
|
||||
assert.equal(
|
||||
repository.calls.some((entry) => entry[0] === 'release'),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects duplicate handlers and extensible handler results', async () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
new ApprovedActionDispatcher(
|
||||
fakeRepository(),
|
||||
[handler(), handler()],
|
||||
deterministicOptions(),
|
||||
),
|
||||
/Duplicate approved action handler/,
|
||||
);
|
||||
const repository = fakeRepository();
|
||||
const dispatcher = new ApprovedActionDispatcher(
|
||||
repository,
|
||||
[
|
||||
handler({
|
||||
async inspect(dispatch) {
|
||||
return {
|
||||
status: 'ready',
|
||||
actionDigest: dispatch.action.actionDigest,
|
||||
injected: true,
|
||||
};
|
||||
},
|
||||
}),
|
||||
],
|
||||
deterministicOptions(),
|
||||
);
|
||||
const summary = await dispatcher.dispatchBatch();
|
||||
assert.equal(summary.retrying, 1);
|
||||
assert.equal(
|
||||
repository.calls.some((entry) => entry[0] === 'start'),
|
||||
false,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,394 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const { QueryTypes, Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const {
|
||||
projectPolicyMigration,
|
||||
} = require('../../back/migrations/0017-project-policy');
|
||||
const {
|
||||
approvalRequestMigration,
|
||||
} = require('../../back/migrations/0020-approval-requests');
|
||||
const {
|
||||
approvedActionDispatchExecutionMigration,
|
||||
} = require('../../back/migrations/0021-approved-action-dispatch-executions');
|
||||
const {
|
||||
approvedActionRecoveryMigration,
|
||||
} = require('../../back/migrations/0022-approved-action-recovery');
|
||||
const {
|
||||
APPROVED_ACTION_RECOVERY_AUTHORIZATION_AUTH_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_AUTHORIZATION_MUTATION_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_AUTHORIZATION_PROJECT_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE,
|
||||
approvedActionRecoveryAuthorizationMigration,
|
||||
} = require('../../back/migrations/0024-approved-action-recovery-authorization');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeApprovedActionDispatchRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/approvedActionDispatchRepository');
|
||||
const {
|
||||
LegacySequelizeApprovedActionRecoveryRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/approvedActionRecoveryRepository');
|
||||
const {
|
||||
LegacySequelizeApprovalRequestRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/approvalRequestRepository');
|
||||
const {
|
||||
LegacySequelizeProjectPolicyRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/projectPolicyRepository');
|
||||
const {
|
||||
ApprovedActionManualRecoveryService,
|
||||
} = require('../../back/runtime/application/approvedActionManualRecoveryService');
|
||||
const {
|
||||
ApprovalRequestService,
|
||||
} = require('../../back/runtime/application/approvalRequestService');
|
||||
const {
|
||||
ProjectPolicyEngine,
|
||||
} = require('../../back/runtime/application/projectPolicyEngine');
|
||||
const {
|
||||
ApprovedActionRecoveryRepositoryError,
|
||||
ApprovedActionRecoveryFenceRejectedError,
|
||||
} = require('../../back/runtime/domain/approvedActionRecovery');
|
||||
const {
|
||||
ApprovedActionRecoveryAuthorizationDeniedError,
|
||||
ApprovedActionRecoveryHumanRequiredError,
|
||||
ApprovedActionRecoveryStrongAuthenticationRequiredError,
|
||||
} = require('../../back/runtime/domain/approvedActionRecoveryAuthorization');
|
||||
|
||||
const PROJECT_ID = 'default';
|
||||
const AGENT = Object.freeze({ type: 'agent', id: 'agent-1' });
|
||||
const OWNER = Object.freeze({ type: 'user', id: 'owner-1' });
|
||||
const SYSTEM = Object.freeze({ type: 'system', id: 'approval-dispatcher' });
|
||||
const BASE_TIME = 300_000;
|
||||
const RESOLUTION_TIME = BASE_TIME + 2_000;
|
||||
|
||||
async function migrate(database) {
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [
|
||||
projectPolicyMigration,
|
||||
approvalRequestMigration,
|
||||
approvedActionDispatchExecutionMigration,
|
||||
approvedActionRecoveryMigration,
|
||||
approvedActionRecoveryAuthorizationMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
}
|
||||
|
||||
async function bind(repository, subject, role, mutationId) {
|
||||
await repository.append({
|
||||
expectedCurrentVersion: 0,
|
||||
binding: {
|
||||
projectId: PROJECT_ID,
|
||||
subject,
|
||||
version: 1,
|
||||
state: 'active',
|
||||
role,
|
||||
mutationId,
|
||||
changedBy: OWNER,
|
||||
createdAtMs: BASE_TIME - 100,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function setup(t) {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
await migrate(database);
|
||||
const policyRepository = new LegacySequelizeProjectPolicyRepository(database);
|
||||
await bind(policyRepository, OWNER, 'owner', 'bind-owner');
|
||||
await bind(policyRepository, AGENT, 'operator', 'bind-agent');
|
||||
const policy = new ProjectPolicyEngine(policyRepository);
|
||||
const approvals = new ApprovalRequestService(
|
||||
new LegacySequelizeApprovalRequestRepository(database),
|
||||
policy,
|
||||
);
|
||||
const executions = new LegacySequelizeApprovedActionDispatchRepository(
|
||||
database,
|
||||
);
|
||||
const recovery = new LegacySequelizeApprovedActionRecoveryRepository(
|
||||
database,
|
||||
);
|
||||
const action = {
|
||||
permission: 'tool.call:filesystem.write',
|
||||
actionType: 'tool_call',
|
||||
actionRef: 'manual-recovery-plan',
|
||||
actionDigest: 'a'.repeat(64),
|
||||
previewDigest: 'f'.repeat(64),
|
||||
};
|
||||
await approvals.create({
|
||||
id: 'approval-manual-recovery',
|
||||
projectId: PROJECT_ID,
|
||||
action,
|
||||
risk: 'critical',
|
||||
requestedBy: AGENT,
|
||||
requestedAtMs: BASE_TIME,
|
||||
expiresAtMs: BASE_TIME + 60_000,
|
||||
});
|
||||
await approvals.decide({
|
||||
requestId: 'approval-manual-recovery',
|
||||
expectedVersion: 1,
|
||||
decisionId: 'decision-manual-recovery',
|
||||
decision: 'approved',
|
||||
reasonCode: 'reviewed_action',
|
||||
decidedBy: OWNER,
|
||||
decidedAtMs: BASE_TIME + 10,
|
||||
});
|
||||
const dispatch = (
|
||||
await approvals.consume({
|
||||
requestId: 'approval-manual-recovery',
|
||||
expectedVersion: 2,
|
||||
consumptionId: 'consumption-manual-recovery',
|
||||
dispatchId: 'dispatch-manual-recovery',
|
||||
action,
|
||||
requestedBy: AGENT,
|
||||
consumedBy: SYSTEM,
|
||||
consumedAtMs: BASE_TIME + 20,
|
||||
})
|
||||
).dispatch;
|
||||
const claimed = await executions.claim({
|
||||
dispatchId: dispatch.id,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'execution-lease-1',
|
||||
nowMs: BASE_TIME + 100,
|
||||
leaseDurationMs: 100,
|
||||
});
|
||||
const started = await executions.start({
|
||||
dispatchId: dispatch.id,
|
||||
approvalRequestId: dispatch.approvalRequestId,
|
||||
actionDigest: dispatch.action.actionDigest,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'execution-lease-1',
|
||||
expectedVersion: claimed.snapshot.execution.version,
|
||||
startedAtMs: BASE_TIME + 110,
|
||||
});
|
||||
return {
|
||||
database,
|
||||
policyRepository,
|
||||
policy,
|
||||
recovery,
|
||||
dispatch,
|
||||
started,
|
||||
};
|
||||
}
|
||||
|
||||
function principal(overrides = {}) {
|
||||
return {
|
||||
subject: OWNER,
|
||||
authenticationId: 'mfa-session-1',
|
||||
authenticatedAtMs: RESOLUTION_TIME - 1_000,
|
||||
expiresAtMs: RESOLUTION_TIME + 60_000,
|
||||
assurance: 'multi_factor',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function input(state, overrides = {}) {
|
||||
return {
|
||||
dispatchId: state.dispatch.id,
|
||||
expectedExecutionVersion: state.started.execution.version,
|
||||
expectedRecoveryVersion: 0,
|
||||
mutationId: 'manual-resolution-1',
|
||||
decision: 'abandon_unknown',
|
||||
reasonCode: 'operator_abandoned_unknown',
|
||||
principal: principal(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('0024 stores a strong-auth and Policy-fenced fact with manual resolution', async (t) => {
|
||||
const state = await setup(t);
|
||||
const service = new ApprovedActionManualRecoveryService(
|
||||
state.recovery,
|
||||
state.policy,
|
||||
() => RESOLUTION_TIME,
|
||||
);
|
||||
const result = await service.resolve(input(state));
|
||||
assert.equal(result.status, 'resolved');
|
||||
assert.equal(result.snapshot.action.execution.status, 'blocked');
|
||||
assert.equal(result.snapshot.resolution.resolvedBy.id, OWNER.id);
|
||||
assert.deepEqual(await service.resolve(input(state)), result);
|
||||
|
||||
const [fact] = await state.database.query(
|
||||
`SELECT * FROM "${APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE}"`,
|
||||
{ type: QueryTypes.SELECT },
|
||||
);
|
||||
assert.equal(fact.dispatch_id, state.dispatch.id);
|
||||
assert.equal(fact.mutation_id, 'manual-resolution-1');
|
||||
assert.equal(fact.resolved_by_id, OWNER.id);
|
||||
assert.equal(fact.authentication_id, 'mfa-session-1');
|
||||
assert.equal(fact.assurance, 'multi_factor');
|
||||
assert.equal(fact.project_version, 1);
|
||||
assert.equal(fact.binding_version, 1);
|
||||
assert.match(fact.fact_digest, /^[0-9a-f]{64}$/);
|
||||
|
||||
const indexes = new Set(
|
||||
(
|
||||
await state.database
|
||||
.getQueryInterface()
|
||||
.showIndex(APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE)
|
||||
).map((index) => index.name),
|
||||
);
|
||||
for (const name of [
|
||||
APPROVED_ACTION_RECOVERY_AUTHORIZATION_MUTATION_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_AUTHORIZATION_PROJECT_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_AUTHORIZATION_AUTH_INDEX,
|
||||
]) {
|
||||
assert.ok(indexes.has(name));
|
||||
}
|
||||
});
|
||||
|
||||
test('requires a stable User and recent strong authentication before storage reads', async () => {
|
||||
let reads = 0;
|
||||
const repository = {
|
||||
async findById() {
|
||||
reads += 1;
|
||||
return null;
|
||||
},
|
||||
};
|
||||
const policy = {
|
||||
async decideWithFence() {
|
||||
throw new Error('unreachable');
|
||||
},
|
||||
};
|
||||
const service = new ApprovedActionManualRecoveryService(
|
||||
repository,
|
||||
policy,
|
||||
() => RESOLUTION_TIME,
|
||||
);
|
||||
const base = {
|
||||
dispatchId: 'dispatch-auth-check',
|
||||
expectedExecutionVersion: 2,
|
||||
expectedRecoveryVersion: 0,
|
||||
mutationId: 'manual-auth-check',
|
||||
decision: 'confirm_failed',
|
||||
reasonCode: 'human_confirmed_failure',
|
||||
};
|
||||
await assert.rejects(
|
||||
service.resolve({
|
||||
...base,
|
||||
principal: principal({
|
||||
subject: { type: 'agent', id: 'agent-1' },
|
||||
}),
|
||||
}),
|
||||
ApprovedActionRecoveryHumanRequiredError,
|
||||
);
|
||||
await assert.rejects(
|
||||
service.resolve({
|
||||
...base,
|
||||
principal: principal({ assurance: 'single_factor' }),
|
||||
}),
|
||||
ApprovedActionRecoveryStrongAuthenticationRequiredError,
|
||||
);
|
||||
await assert.rejects(
|
||||
service.resolve({
|
||||
...base,
|
||||
principal: principal({ authenticatedAtMs: RESOLUTION_TIME - 300_001 }),
|
||||
}),
|
||||
ApprovedActionRecoveryStrongAuthenticationRequiredError,
|
||||
);
|
||||
assert.equal(reads, 0);
|
||||
});
|
||||
|
||||
test('operator Policy is denied and an authorization revocation race is fenced', async (t) => {
|
||||
const denied = await setup(t);
|
||||
await denied.policyRepository.append({
|
||||
expectedCurrentVersion: 0,
|
||||
binding: {
|
||||
projectId: PROJECT_ID,
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
version: 1,
|
||||
state: 'active',
|
||||
role: 'operator',
|
||||
mutationId: 'bind-operator',
|
||||
changedBy: OWNER,
|
||||
createdAtMs: BASE_TIME,
|
||||
},
|
||||
});
|
||||
const deniedService = new ApprovedActionManualRecoveryService(
|
||||
denied.recovery,
|
||||
denied.policy,
|
||||
() => RESOLUTION_TIME,
|
||||
);
|
||||
await assert.rejects(
|
||||
deniedService.resolve(
|
||||
input(denied, {
|
||||
principal: principal({
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
}),
|
||||
}),
|
||||
),
|
||||
ApprovedActionRecoveryAuthorizationDeniedError,
|
||||
);
|
||||
|
||||
const raced = await setup(t);
|
||||
const racingPolicy = {
|
||||
async decideWithFence(request) {
|
||||
const decision = await raced.policy.decideWithFence(request);
|
||||
await raced.policyRepository.append({
|
||||
expectedCurrentVersion: 1,
|
||||
binding: {
|
||||
projectId: PROJECT_ID,
|
||||
subject: OWNER,
|
||||
version: 2,
|
||||
state: 'revoked',
|
||||
mutationId: 'revoke-owner-during-recovery',
|
||||
changedBy: OWNER,
|
||||
createdAtMs: RESOLUTION_TIME,
|
||||
},
|
||||
});
|
||||
return decision;
|
||||
},
|
||||
};
|
||||
const racedService = new ApprovedActionManualRecoveryService(
|
||||
raced.recovery,
|
||||
racingPolicy,
|
||||
() => RESOLUTION_TIME,
|
||||
);
|
||||
await assert.rejects(
|
||||
racedService.resolve(input(raced)),
|
||||
ApprovedActionRecoveryFenceRejectedError,
|
||||
);
|
||||
const after = await raced.recovery.findById(raced.dispatch.id);
|
||||
assert.equal(after.action.execution.status, 'executing');
|
||||
assert.equal(after.recovery.status, 'armed');
|
||||
});
|
||||
|
||||
test('authorization fact failure rolls the manual terminal transition back', async (t) => {
|
||||
const state = await setup(t);
|
||||
await state.database.query(
|
||||
`CREATE TRIGGER reject_recovery_authorization
|
||||
BEFORE INSERT ON "${APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE}"
|
||||
BEGIN SELECT RAISE(ABORT, 'authorization rejected'); END`,
|
||||
);
|
||||
const service = new ApprovedActionManualRecoveryService(
|
||||
state.recovery,
|
||||
state.policy,
|
||||
() => RESOLUTION_TIME,
|
||||
);
|
||||
await assert.rejects(
|
||||
service.resolve(input(state)),
|
||||
ApprovedActionRecoveryRepositoryError,
|
||||
);
|
||||
const after = await state.recovery.findById(state.dispatch.id);
|
||||
assert.equal(after.action.execution.status, 'executing');
|
||||
assert.equal(after.recovery.status, 'armed');
|
||||
assert.equal(after.resolution, null);
|
||||
assert.equal(
|
||||
(
|
||||
await state.database.query(
|
||||
`SELECT dispatch_id FROM "${APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE}"`,
|
||||
{ type: QueryTypes.SELECT },
|
||||
)
|
||||
).length,
|
||||
0,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,686 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const { QueryTypes, Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const {
|
||||
projectPolicyMigration,
|
||||
} = require('../../back/migrations/0017-project-policy');
|
||||
const {
|
||||
approvalRequestMigration,
|
||||
} = require('../../back/migrations/0020-approval-requests');
|
||||
const {
|
||||
APPROVED_ACTION_DISPATCH_EXECUTION_TABLE,
|
||||
approvedActionDispatchExecutionMigration,
|
||||
} = require('../../back/migrations/0021-approved-action-dispatch-executions');
|
||||
const {
|
||||
APPROVED_ACTION_RECOVERY_CONTROL_TABLE,
|
||||
APPROVED_ACTION_RECOVERY_DUE_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_LEASE_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_PROJECT_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_RESOLUTION_MUTATION_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_RESOLUTION_PROJECT_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE,
|
||||
approvedActionRecoveryMigration,
|
||||
} = require('../../back/migrations/0022-approved-action-recovery');
|
||||
const {
|
||||
approvedActionRecoveryAuthorizationMigration,
|
||||
} = require('../../back/migrations/0024-approved-action-recovery-authorization');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeApprovedActionDispatchRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/approvedActionDispatchRepository');
|
||||
const {
|
||||
LegacySequelizeApprovedActionRecoveryRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/approvedActionRecoveryRepository');
|
||||
const {
|
||||
LegacySequelizeApprovalRequestRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/approvalRequestRepository');
|
||||
const {
|
||||
LegacySequelizeProjectPolicyRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/projectPolicyRepository');
|
||||
const {
|
||||
ApprovalRequestService,
|
||||
} = require('../../back/runtime/application/approvalRequestService');
|
||||
const {
|
||||
ProjectPolicyEngine,
|
||||
} = require('../../back/runtime/application/projectPolicyEngine');
|
||||
const {
|
||||
ApprovedActionDispatchRepositoryError,
|
||||
} = require('../../back/runtime/domain/approvedActionDispatchExecution');
|
||||
const {
|
||||
ApprovedActionRecoveryFenceRejectedError,
|
||||
} = require('../../back/runtime/domain/approvedActionRecovery');
|
||||
const {
|
||||
createApprovedActionRecoveryAuthorizationFact,
|
||||
} = require('../../back/runtime/domain/approvedActionRecoveryAuthorization');
|
||||
|
||||
const PROJECT_ID = 'default';
|
||||
const AGENT = Object.freeze({ type: 'agent', id: 'agent-1' });
|
||||
const OWNER = Object.freeze({ type: 'user', id: 'owner-1' });
|
||||
const SYSTEM = Object.freeze({ type: 'system', id: 'approval-dispatcher' });
|
||||
const BASE_TIME = 100_000;
|
||||
|
||||
function action(name, digestCharacter = 'a') {
|
||||
return {
|
||||
permission: 'tool.call:filesystem.write',
|
||||
actionType: 'tool_call',
|
||||
actionRef: `planned-${name}`,
|
||||
actionDigest: digestCharacter.repeat(64),
|
||||
previewDigest: 'f'.repeat(64),
|
||||
};
|
||||
}
|
||||
|
||||
async function migrate(database, migrations) {
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations,
|
||||
logger: { info() {} },
|
||||
});
|
||||
}
|
||||
|
||||
async function bind(policyRepository, subject, role, mutationId) {
|
||||
await policyRepository.append({
|
||||
expectedCurrentVersion: 0,
|
||||
binding: {
|
||||
projectId: PROJECT_ID,
|
||||
subject,
|
||||
version: 1,
|
||||
state: 'active',
|
||||
role,
|
||||
mutationId,
|
||||
changedBy: OWNER,
|
||||
createdAtMs: BASE_TIME - 100,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function setup(t, storage = ':memory:', includeRecovery = true) {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
await migrate(database, [
|
||||
projectPolicyMigration,
|
||||
approvalRequestMigration,
|
||||
approvedActionDispatchExecutionMigration,
|
||||
...(includeRecovery
|
||||
? [
|
||||
approvedActionRecoveryMigration,
|
||||
approvedActionRecoveryAuthorizationMigration,
|
||||
]
|
||||
: []),
|
||||
]);
|
||||
const policyRepository = new LegacySequelizeProjectPolicyRepository(database);
|
||||
await bind(policyRepository, OWNER, 'owner', 'bind-owner');
|
||||
await bind(policyRepository, AGENT, 'operator', 'bind-agent');
|
||||
const approvalRepository = new LegacySequelizeApprovalRequestRepository(
|
||||
database,
|
||||
);
|
||||
return {
|
||||
database,
|
||||
approvalService: new ApprovalRequestService(
|
||||
approvalRepository,
|
||||
new ProjectPolicyEngine(policyRepository),
|
||||
),
|
||||
actionRepository: new LegacySequelizeApprovedActionDispatchRepository(
|
||||
database,
|
||||
),
|
||||
recoveryRepository: includeRecovery
|
||||
? new LegacySequelizeApprovedActionRecoveryRepository(database)
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
async function prepareDispatch(service, name, offset = 0, digest = 'a') {
|
||||
const requestedAtMs = BASE_TIME + offset;
|
||||
const binding = action(name, digest);
|
||||
await service.create({
|
||||
id: `approval-${name}`,
|
||||
projectId: PROJECT_ID,
|
||||
action: binding,
|
||||
risk: 'high',
|
||||
requestedBy: AGENT,
|
||||
requestedAtMs,
|
||||
expiresAtMs: requestedAtMs + 60_000,
|
||||
});
|
||||
await service.decide({
|
||||
requestId: `approval-${name}`,
|
||||
expectedVersion: 1,
|
||||
decisionId: `decision-${name}`,
|
||||
decision: 'approved',
|
||||
reasonCode: 'reviewed_action',
|
||||
decidedBy: OWNER,
|
||||
decidedAtMs: requestedAtMs + 10,
|
||||
});
|
||||
return (
|
||||
await service.consume({
|
||||
requestId: `approval-${name}`,
|
||||
expectedVersion: 2,
|
||||
consumptionId: `consumption-${name}`,
|
||||
dispatchId: `dispatch-${name}`,
|
||||
action: binding,
|
||||
requestedBy: AGENT,
|
||||
consumedBy: SYSTEM,
|
||||
consumedAtMs: requestedAtMs + 20,
|
||||
})
|
||||
).dispatch;
|
||||
}
|
||||
|
||||
async function startAction(actionRepository, dispatch, options = {}) {
|
||||
const nowMs = options.nowMs ?? BASE_TIME + 1_000;
|
||||
const leaseDurationMs = options.leaseDurationMs ?? 100;
|
||||
const owner = options.owner ?? 'dispatcher-1';
|
||||
const leaseToken = options.leaseToken ?? 'execution-lease-1';
|
||||
const claimed = await actionRepository.claim({
|
||||
dispatchId: dispatch.id,
|
||||
owner,
|
||||
leaseToken,
|
||||
nowMs,
|
||||
leaseDurationMs,
|
||||
});
|
||||
assert.equal(claimed.status, 'claimed');
|
||||
const started = await actionRepository.start({
|
||||
dispatchId: dispatch.id,
|
||||
approvalRequestId: dispatch.approvalRequestId,
|
||||
actionDigest: dispatch.action.actionDigest,
|
||||
owner,
|
||||
leaseToken,
|
||||
expectedVersion: claimed.snapshot.execution.version,
|
||||
startedAtMs: nowMs + 10,
|
||||
});
|
||||
return {
|
||||
claimed,
|
||||
started,
|
||||
owner,
|
||||
leaseToken,
|
||||
leaseExpiresAtMs: nowMs + leaseDurationMs,
|
||||
};
|
||||
}
|
||||
|
||||
async function claimRecovery(recoveryRepository, dispatchId, overrides = {}) {
|
||||
return recoveryRepository.claim({
|
||||
dispatchId,
|
||||
owner: 'resolver-1',
|
||||
leaseToken: 'recovery-lease-1',
|
||||
nowMs: BASE_TIME + 1_100,
|
||||
leaseDurationMs: 1_000,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
test('0022 backfills executing controls and owns bounded recovery indexes', async (t) => {
|
||||
const context = await setup(t, ':memory:', false);
|
||||
const dispatch = await prepareDispatch(context.approvalService, 'backfill');
|
||||
await context.database.query(
|
||||
`UPDATE "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}"
|
||||
SET status = 'executing', version = 2, attempt_count = 1,
|
||||
eligible_at_ms = NULL, lease_owner = 'dispatcher-1',
|
||||
lease_token = 'execution-lease-1',
|
||||
lease_expires_at_ms = :leaseExpiresAtMs,
|
||||
started_at_ms = :startedAtMs, updated_at_ms = :startedAtMs
|
||||
WHERE dispatch_id = :dispatchId`,
|
||||
{
|
||||
replacements: {
|
||||
dispatchId: dispatch.id,
|
||||
startedAtMs: BASE_TIME + 1_010,
|
||||
leaseExpiresAtMs: BASE_TIME + 1_100,
|
||||
},
|
||||
},
|
||||
);
|
||||
await migrate(context.database, [approvedActionRecoveryMigration]);
|
||||
const rows = await context.database.query(
|
||||
`SELECT * FROM "${APPROVED_ACTION_RECOVERY_CONTROL_TABLE}"`,
|
||||
{ type: QueryTypes.SELECT },
|
||||
);
|
||||
assert.equal(rows.length, 1);
|
||||
assert.equal(rows[0].dispatch_id, dispatch.id);
|
||||
assert.equal(rows[0].execution_version, 2);
|
||||
assert.equal(rows[0].status, 'armed');
|
||||
assert.equal(rows[0].next_scan_at_ms, BASE_TIME + 1_100);
|
||||
const controlIndexes = new Set(
|
||||
(
|
||||
await context.database
|
||||
.getQueryInterface()
|
||||
.showIndex(APPROVED_ACTION_RECOVERY_CONTROL_TABLE)
|
||||
).map((index) => index.name),
|
||||
);
|
||||
for (const name of [
|
||||
APPROVED_ACTION_RECOVERY_DUE_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_PROJECT_INDEX,
|
||||
APPROVED_ACTION_RECOVERY_LEASE_INDEX,
|
||||
]) {
|
||||
assert.ok(controlIndexes.has(name));
|
||||
}
|
||||
const resolutionIndexes = new Set(
|
||||
(
|
||||
await context.database
|
||||
.getQueryInterface()
|
||||
.showIndex(APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE)
|
||||
).map((index) => index.name),
|
||||
);
|
||||
assert.ok(
|
||||
resolutionIndexes.has(APPROVED_ACTION_RECOVERY_RESOLUTION_MUTATION_INDEX),
|
||||
);
|
||||
assert.ok(
|
||||
resolutionIndexes.has(APPROVED_ACTION_RECOVERY_RESOLUTION_PROJECT_INDEX),
|
||||
);
|
||||
});
|
||||
|
||||
test('start barrier atomically arms recovery and rolls back when control insert fails', async (t) => {
|
||||
const context = await setup(t);
|
||||
const dispatch = await prepareDispatch(
|
||||
context.approvalService,
|
||||
'start-atomic',
|
||||
);
|
||||
const claimed = await context.actionRepository.claim({
|
||||
dispatchId: dispatch.id,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'execution-lease-1',
|
||||
nowMs: BASE_TIME + 1_000,
|
||||
leaseDurationMs: 100,
|
||||
});
|
||||
await context.database.query(
|
||||
`CREATE TRIGGER fail_recovery_control_insert
|
||||
BEFORE INSERT ON "${APPROVED_ACTION_RECOVERY_CONTROL_TABLE}"
|
||||
BEGIN SELECT RAISE(ABORT, 'forced recovery insert failure'); END`,
|
||||
);
|
||||
await assert.rejects(
|
||||
context.actionRepository.start({
|
||||
dispatchId: dispatch.id,
|
||||
approvalRequestId: dispatch.approvalRequestId,
|
||||
actionDigest: dispatch.action.actionDigest,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: 'execution-lease-1',
|
||||
expectedVersion: claimed.snapshot.execution.version,
|
||||
startedAtMs: BASE_TIME + 1_010,
|
||||
}),
|
||||
ApprovedActionDispatchRepositoryError,
|
||||
);
|
||||
assert.equal(
|
||||
(await context.actionRepository.findById(dispatch.id)).execution.status,
|
||||
'leased',
|
||||
);
|
||||
assert.equal(await context.recoveryRepository.findById(dispatch.id), null);
|
||||
});
|
||||
|
||||
test('normal completion closes the recovery control without creating a recovery resolution', async (t) => {
|
||||
const context = await setup(t);
|
||||
const dispatch = await prepareDispatch(context.approvalService, 'complete');
|
||||
const execution = await startAction(context.actionRepository, dispatch);
|
||||
const armed = await context.recoveryRepository.findById(dispatch.id);
|
||||
assert.equal(armed.recovery.status, 'armed');
|
||||
assert.equal(
|
||||
armed.recovery.executionVersion,
|
||||
execution.started.execution.version,
|
||||
);
|
||||
assert.equal(armed.recovery.nextScanAtMs, execution.leaseExpiresAtMs);
|
||||
const completed = await context.actionRepository.complete({
|
||||
dispatchId: dispatch.id,
|
||||
owner: execution.owner,
|
||||
leaseToken: execution.leaseToken,
|
||||
expectedVersion: execution.started.execution.version,
|
||||
resultMutationId: 'normal-completion-1',
|
||||
outcome: 'succeeded',
|
||||
resultCode: 'ok',
|
||||
completedAtMs: BASE_TIME + 1_120,
|
||||
});
|
||||
assert.equal(completed.execution.status, 'succeeded');
|
||||
const closed = await context.recoveryRepository.findById(dispatch.id);
|
||||
assert.equal(closed.recovery.status, 'resolved');
|
||||
assert.equal(closed.recovery.resolutionMutationId, 'normal-completion-1');
|
||||
assert.equal(closed.resolution, null);
|
||||
assert.equal(
|
||||
(await claimRecovery(context.recoveryRepository, dispatch.id)).status,
|
||||
'resolved',
|
||||
);
|
||||
});
|
||||
|
||||
test('a late execution renew re-arms recovery and fences the stale resolver lease', async (t) => {
|
||||
const context = await setup(t);
|
||||
const dispatch = await prepareDispatch(context.approvalService, 'renew');
|
||||
const execution = await startAction(context.actionRepository, dispatch);
|
||||
const claimed = await claimRecovery(context.recoveryRepository, dispatch.id);
|
||||
assert.equal(claimed.status, 'claimed');
|
||||
const renewed = await context.actionRepository.renew({
|
||||
dispatchId: dispatch.id,
|
||||
owner: execution.owner,
|
||||
leaseToken: execution.leaseToken,
|
||||
expectedVersion: execution.started.execution.version,
|
||||
nowMs: BASE_TIME + 1_150,
|
||||
leaseDurationMs: 1_000,
|
||||
});
|
||||
assert.equal(
|
||||
renewed.execution.version,
|
||||
execution.started.execution.version + 1,
|
||||
);
|
||||
const rearmed = await context.recoveryRepository.findById(dispatch.id);
|
||||
assert.equal(rearmed.recovery.status, 'armed');
|
||||
assert.equal(rearmed.recovery.executionVersion, renewed.execution.version);
|
||||
assert.equal(rearmed.recovery.nextScanAtMs, BASE_TIME + 2_150);
|
||||
assert.equal(rearmed.recovery.leaseOwner, null);
|
||||
await assert.rejects(
|
||||
context.recoveryRepository.recordFinding({
|
||||
dispatchId: dispatch.id,
|
||||
expectedExecutionVersion: claimed.snapshot.action.execution.version,
|
||||
expectedRecoveryVersion: claimed.snapshot.recovery.version,
|
||||
owner: 'resolver-1',
|
||||
leaseToken: 'recovery-lease-1',
|
||||
findingMutationId: 'stale-finding-1',
|
||||
finding: 'missing',
|
||||
resultCode: 'receipt_missing',
|
||||
observedAtMs: BASE_TIME + 1_160,
|
||||
retryAtMs: BASE_TIME + 1_200,
|
||||
}),
|
||||
ApprovedActionRecoveryFenceRejectedError,
|
||||
);
|
||||
});
|
||||
|
||||
test('lists a bounded stable recovery page and excludes live executions', async (t) => {
|
||||
const context = await setup(t);
|
||||
const firstDispatch = await prepareDispatch(
|
||||
context.approvalService,
|
||||
'due-a',
|
||||
0,
|
||||
'a',
|
||||
);
|
||||
const secondDispatch = await prepareDispatch(
|
||||
context.approvalService,
|
||||
'due-b',
|
||||
10,
|
||||
'b',
|
||||
);
|
||||
const liveDispatch = await prepareDispatch(
|
||||
context.approvalService,
|
||||
'live',
|
||||
20,
|
||||
'c',
|
||||
);
|
||||
await startAction(context.actionRepository, firstDispatch);
|
||||
await startAction(context.actionRepository, secondDispatch);
|
||||
await startAction(context.actionRepository, liveDispatch, {
|
||||
leaseDurationMs: 1_000,
|
||||
});
|
||||
const first = await context.recoveryRepository.listDue({
|
||||
nowMs: BASE_TIME + 1_100,
|
||||
limit: 1,
|
||||
});
|
||||
assert.deepEqual(
|
||||
first.recoveries.map((entry) => entry.action.dispatch.id),
|
||||
['dispatch-due-a'],
|
||||
);
|
||||
assert.equal(first.truncated, true);
|
||||
const second = await context.recoveryRepository.listDue({
|
||||
nowMs: BASE_TIME + 1_100,
|
||||
limit: 1,
|
||||
cursor: first.nextCursor,
|
||||
});
|
||||
assert.deepEqual(
|
||||
second.recoveries.map((entry) => entry.action.dispatch.id),
|
||||
['dispatch-due-b'],
|
||||
);
|
||||
assert.equal(second.truncated, false);
|
||||
});
|
||||
|
||||
test('two SQLite resolvers claim once and only an expired recovery lease is taken over', async (t) => {
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-recovery-'));
|
||||
t.after(() => fs.rm(directory, { recursive: true, force: true }));
|
||||
const storage = path.join(directory, 'database.sqlite');
|
||||
const first = await setup(t, storage);
|
||||
const dispatch = await prepareDispatch(first.approvalService, 'claim-race');
|
||||
await startAction(first.actionRepository, dispatch);
|
||||
const secondDatabase = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => secondDatabase.close());
|
||||
const secondRepository = new LegacySequelizeApprovedActionRecoveryRepository(
|
||||
secondDatabase,
|
||||
);
|
||||
const results = await Promise.all([
|
||||
claimRecovery(first.recoveryRepository, dispatch.id, {
|
||||
leaseDurationMs: 100,
|
||||
}),
|
||||
claimRecovery(secondRepository, dispatch.id, {
|
||||
owner: 'resolver-2',
|
||||
leaseToken: 'recovery-lease-2',
|
||||
leaseDurationMs: 100,
|
||||
}),
|
||||
]);
|
||||
assert.equal(
|
||||
results.filter((result) => result.status === 'claimed').length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
results.filter((result) => result.status === 'leased').length,
|
||||
1,
|
||||
);
|
||||
const winner = results.find((result) => result.status === 'claimed');
|
||||
assert.equal(
|
||||
(
|
||||
await claimRecovery(first.recoveryRepository, dispatch.id, {
|
||||
owner: winner.snapshot.recovery.leaseOwner,
|
||||
leaseToken: winner.snapshot.recovery.leaseToken,
|
||||
leaseDurationMs: 100,
|
||||
})
|
||||
).status,
|
||||
'claimed',
|
||||
);
|
||||
const takeover = await claimRecovery(secondRepository, dispatch.id, {
|
||||
owner: 'resolver-3',
|
||||
leaseToken: 'recovery-lease-3',
|
||||
nowMs: BASE_TIME + 1_200,
|
||||
leaseDurationMs: 100,
|
||||
});
|
||||
assert.equal(takeover.status, 'claimed');
|
||||
assert.equal(takeover.snapshot.recovery.leaseOwner, 'resolver-3');
|
||||
});
|
||||
|
||||
test('recovery finding replay is exact and unsupported evidence becomes manual-only', async (t) => {
|
||||
const context = await setup(t);
|
||||
const dispatch = await prepareDispatch(context.approvalService, 'finding');
|
||||
await startAction(context.actionRepository, dispatch);
|
||||
const claimed = await claimRecovery(context.recoveryRepository, dispatch.id);
|
||||
assert.equal(claimed.status, 'claimed');
|
||||
const finding = {
|
||||
dispatchId: dispatch.id,
|
||||
expectedExecutionVersion: claimed.snapshot.action.execution.version,
|
||||
expectedRecoveryVersion: claimed.snapshot.recovery.version,
|
||||
owner: 'resolver-1',
|
||||
leaseToken: 'recovery-lease-1',
|
||||
findingMutationId: 'finding-missing-1',
|
||||
finding: 'missing',
|
||||
resultCode: 'receipt_missing',
|
||||
observedAtMs: BASE_TIME + 1_110,
|
||||
retryAtMs: BASE_TIME + 1_200,
|
||||
};
|
||||
const deferred = await context.recoveryRepository.recordFinding(finding);
|
||||
assert.equal(deferred.recovery.status, 'armed');
|
||||
assert.equal(deferred.recovery.findingCount, 1);
|
||||
assert.deepEqual(
|
||||
await context.recoveryRepository.recordFinding(finding),
|
||||
deferred,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await claimRecovery(context.recoveryRepository, dispatch.id, {
|
||||
nowMs: BASE_TIME + 1_199,
|
||||
})
|
||||
).status,
|
||||
'not_due',
|
||||
);
|
||||
const second = await claimRecovery(context.recoveryRepository, dispatch.id, {
|
||||
owner: 'resolver-2',
|
||||
leaseToken: 'recovery-lease-2',
|
||||
nowMs: BASE_TIME + 1_200,
|
||||
});
|
||||
assert.equal(second.status, 'claimed');
|
||||
const manual = await context.recoveryRepository.recordFinding({
|
||||
dispatchId: dispatch.id,
|
||||
expectedExecutionVersion: second.snapshot.action.execution.version,
|
||||
expectedRecoveryVersion: second.snapshot.recovery.version,
|
||||
owner: 'resolver-2',
|
||||
leaseToken: 'recovery-lease-2',
|
||||
findingMutationId: 'finding-unsupported-1',
|
||||
finding: 'unsupported',
|
||||
resultCode: 'automatic_recovery_unsupported',
|
||||
observedAtMs: BASE_TIME + 1_210,
|
||||
});
|
||||
assert.equal(manual.recovery.status, 'manual_required');
|
||||
assert.equal(manual.recovery.nextScanAtMs, null);
|
||||
assert.equal(
|
||||
(
|
||||
await context.recoveryRepository.listDue({
|
||||
nowMs: BASE_TIME + 10_000,
|
||||
limit: 64,
|
||||
})
|
||||
).recoveries.length,
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
test('verified evidence resolves atomically and exact mutation replay returns the same resolution', async (t) => {
|
||||
const context = await setup(t);
|
||||
const dispatch = await prepareDispatch(context.approvalService, 'verified');
|
||||
await startAction(context.actionRepository, dispatch);
|
||||
const claimed = await claimRecovery(context.recoveryRepository, dispatch.id);
|
||||
const command = {
|
||||
dispatchId: dispatch.id,
|
||||
expectedExecutionVersion: claimed.snapshot.action.execution.version,
|
||||
expectedRecoveryVersion: claimed.snapshot.recovery.version,
|
||||
owner: 'resolver-1',
|
||||
leaseToken: 'recovery-lease-1',
|
||||
mutationId: 'resolution-verified-1',
|
||||
source: 'automatic_evidence',
|
||||
decision: 'confirm_succeeded',
|
||||
evidenceDigest: 'e'.repeat(64),
|
||||
reasonCode: 'provider_receipt_verified',
|
||||
resolvedAtMs: BASE_TIME + 1_120,
|
||||
};
|
||||
const resolved = await context.recoveryRepository.resolve(command);
|
||||
assert.equal(resolved.status, 'resolved');
|
||||
assert.equal(resolved.snapshot.action.execution.status, 'succeeded');
|
||||
assert.equal(resolved.snapshot.recovery.status, 'resolved');
|
||||
assert.equal(resolved.snapshot.resolution.source, 'automatic_evidence');
|
||||
assert.equal(resolved.snapshot.resolution.evidenceDigest, 'e'.repeat(64));
|
||||
assert.deepEqual(await context.recoveryRepository.resolve(command), resolved);
|
||||
});
|
||||
|
||||
test('manual resolution cannot preempt a live execution and never resets it for retry', async (t) => {
|
||||
const context = await setup(t);
|
||||
const dispatch = await prepareDispatch(context.approvalService, 'manual');
|
||||
const execution = await startAction(context.actionRepository, dispatch, {
|
||||
leaseDurationMs: 500,
|
||||
});
|
||||
const armed = await context.recoveryRepository.findById(dispatch.id);
|
||||
const commandAt = (resolvedAtMs) => {
|
||||
const command = {
|
||||
dispatchId: dispatch.id,
|
||||
expectedExecutionVersion: armed.action.execution.version,
|
||||
expectedRecoveryVersion: armed.recovery.version,
|
||||
mutationId: 'resolution-manual-1',
|
||||
source: 'human',
|
||||
decision: 'abandon_unknown',
|
||||
reasonCode: 'operator_abandoned_unknown',
|
||||
resolvedBy: OWNER,
|
||||
resolvedAtMs,
|
||||
};
|
||||
return {
|
||||
...command,
|
||||
authorizationFact: createApprovedActionRecoveryAuthorizationFact({
|
||||
dispatchId: dispatch.id,
|
||||
projectId: PROJECT_ID,
|
||||
mutationId: command.mutationId,
|
||||
resolvedBy: OWNER,
|
||||
authenticationId: 'mfa-session-1',
|
||||
assurance: 'multi_factor',
|
||||
authenticatedAtMs: resolvedAtMs - 100,
|
||||
projectVersion: 1,
|
||||
bindingVersion: 1,
|
||||
authorizedAtMs: resolvedAtMs,
|
||||
}),
|
||||
};
|
||||
};
|
||||
await assert.rejects(
|
||||
context.recoveryRepository.resolve(
|
||||
commandAt(execution.leaseExpiresAtMs - 1),
|
||||
),
|
||||
ApprovedActionRecoveryFenceRejectedError,
|
||||
);
|
||||
const resolved = await context.recoveryRepository.resolve(
|
||||
commandAt(execution.leaseExpiresAtMs),
|
||||
);
|
||||
assert.equal(resolved.snapshot.action.execution.status, 'blocked');
|
||||
assert.equal(resolved.snapshot.resolution.decision, 'abandon_unknown');
|
||||
assert.equal(resolved.snapshot.resolution.resolvedBy.type, 'user');
|
||||
assert.equal(resolved.snapshot.action.execution.eligibleAtMs, null);
|
||||
});
|
||||
|
||||
test('late completion and recovery resolution have one durable terminal winner', async (t) => {
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-recovery-'));
|
||||
t.after(() => fs.rm(directory, { recursive: true, force: true }));
|
||||
const storage = path.join(directory, 'database.sqlite');
|
||||
const first = await setup(t, storage);
|
||||
const dispatch = await prepareDispatch(first.approvalService, 'race');
|
||||
const execution = await startAction(first.actionRepository, dispatch);
|
||||
const secondDatabase = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => secondDatabase.close());
|
||||
const secondRecovery = new LegacySequelizeApprovedActionRecoveryRepository(
|
||||
secondDatabase,
|
||||
);
|
||||
const claimed = await claimRecovery(secondRecovery, dispatch.id);
|
||||
const recoveryCommand = {
|
||||
dispatchId: dispatch.id,
|
||||
expectedExecutionVersion: claimed.snapshot.action.execution.version,
|
||||
expectedRecoveryVersion: claimed.snapshot.recovery.version,
|
||||
owner: 'resolver-1',
|
||||
leaseToken: 'recovery-lease-1',
|
||||
mutationId: 'resolution-race-1',
|
||||
source: 'automatic_evidence',
|
||||
decision: 'confirm_failed',
|
||||
evidenceDigest: 'd'.repeat(64),
|
||||
reasonCode: 'provider_failed',
|
||||
resolvedAtMs: BASE_TIME + 1_120,
|
||||
};
|
||||
await Promise.allSettled([
|
||||
first.actionRepository.complete({
|
||||
dispatchId: dispatch.id,
|
||||
owner: execution.owner,
|
||||
leaseToken: execution.leaseToken,
|
||||
expectedVersion: execution.started.execution.version,
|
||||
resultMutationId: 'normal-race-1',
|
||||
outcome: 'succeeded',
|
||||
resultCode: 'ok',
|
||||
completedAtMs: BASE_TIME + 1_120,
|
||||
}),
|
||||
secondRecovery.resolve(recoveryCommand),
|
||||
]);
|
||||
const final = await secondRecovery.findById(dispatch.id);
|
||||
assert.ok(['succeeded', 'failed'].includes(final.action.execution.status));
|
||||
assert.equal(final.recovery.status, 'resolved');
|
||||
assert.equal(
|
||||
final.recovery.resolutionMutationId,
|
||||
final.action.execution.resultMutationId,
|
||||
);
|
||||
if (final.resolution) {
|
||||
assert.equal(final.resolution.mutationId, 'resolution-race-1');
|
||||
assert.equal(final.action.execution.status, 'failed');
|
||||
} else {
|
||||
assert.equal(final.action.execution.resultMutationId, 'normal-race-1');
|
||||
assert.equal(final.action.execution.status, 'succeeded');
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,312 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
ApprovedActionRecoveryReconciler,
|
||||
} = require('../../back/runtime/application/approvedActionRecoveryReconciler');
|
||||
|
||||
function recoverySnapshot() {
|
||||
return {
|
||||
action: {
|
||||
dispatch: {
|
||||
id: 'dispatch-one',
|
||||
approvalRequestId: 'approval-one',
|
||||
approvalRequestVersion: 3,
|
||||
projectId: 'default',
|
||||
state: 'pending',
|
||||
action: {
|
||||
permission: 'tool.call:filesystem.write',
|
||||
actionType: 'tool_call',
|
||||
actionRef: 'planned-one',
|
||||
actionDigest: 'a'.repeat(64),
|
||||
previewDigest: 'f'.repeat(64),
|
||||
},
|
||||
requestedBy: { type: 'agent', id: 'agent-1' },
|
||||
consumedBy: { type: 'system', id: 'approval-dispatcher' },
|
||||
createdAtMs: 100,
|
||||
},
|
||||
execution: {
|
||||
dispatchId: 'dispatch-one',
|
||||
projectId: 'default',
|
||||
status: 'executing',
|
||||
version: 2,
|
||||
attemptCount: 1,
|
||||
maxAttempts: 5,
|
||||
eligibleAtMs: null,
|
||||
nextAttemptAtMs: null,
|
||||
leaseOwner: 'dispatcher-1',
|
||||
leaseToken: 'execution-lease-1',
|
||||
leaseExpiresAtMs: 200,
|
||||
startedAtMs: 110,
|
||||
resultMutationId: null,
|
||||
lastResultCode: null,
|
||||
completedAtMs: null,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 110,
|
||||
},
|
||||
},
|
||||
recovery: {
|
||||
dispatchId: 'dispatch-one',
|
||||
projectId: 'default',
|
||||
executionVersion: 2,
|
||||
status: 'armed',
|
||||
version: 0,
|
||||
nextScanAtMs: 200,
|
||||
leaseOwner: null,
|
||||
leaseToken: null,
|
||||
leaseExpiresAtMs: null,
|
||||
findingCount: 0,
|
||||
lastFindingMutationId: null,
|
||||
lastFinding: null,
|
||||
lastResultCode: null,
|
||||
lastEvidenceDigest: null,
|
||||
resolutionMutationId: null,
|
||||
createdAtMs: 110,
|
||||
updatedAtMs: 110,
|
||||
},
|
||||
resolution: null,
|
||||
};
|
||||
}
|
||||
|
||||
function fakeRepository() {
|
||||
const calls = [];
|
||||
const initial = recoverySnapshot();
|
||||
return {
|
||||
calls,
|
||||
async listDue(query) {
|
||||
calls.push(['list', query]);
|
||||
return { recoveries: [initial], truncated: false };
|
||||
},
|
||||
async claim(command) {
|
||||
calls.push(['claim', command]);
|
||||
return {
|
||||
status: 'claimed',
|
||||
snapshot: {
|
||||
...initial,
|
||||
recovery: {
|
||||
...initial.recovery,
|
||||
status: 'leased',
|
||||
version: 1,
|
||||
nextScanAtMs: command.nowMs + command.leaseDurationMs,
|
||||
leaseOwner: command.owner,
|
||||
leaseToken: command.leaseToken,
|
||||
leaseExpiresAtMs: command.nowMs + command.leaseDurationMs,
|
||||
updatedAtMs: command.nowMs,
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
async recordFinding(command) {
|
||||
calls.push(['finding', command]);
|
||||
return {
|
||||
...initial,
|
||||
recovery: {
|
||||
...initial.recovery,
|
||||
status: command.retryAtMs ? 'armed' : 'manual_required',
|
||||
version: command.expectedRecoveryVersion + 1,
|
||||
nextScanAtMs: command.retryAtMs ?? null,
|
||||
findingCount: 1,
|
||||
lastFindingMutationId: command.findingMutationId,
|
||||
lastFinding: command.finding,
|
||||
lastResultCode: command.resultCode,
|
||||
lastEvidenceDigest: command.evidenceDigest ?? null,
|
||||
updatedAtMs: command.observedAtMs,
|
||||
},
|
||||
};
|
||||
},
|
||||
async resolve(command) {
|
||||
calls.push(['resolve', command]);
|
||||
const status =
|
||||
command.decision === 'confirm_succeeded' ? 'succeeded' : 'failed';
|
||||
return {
|
||||
status: 'resolved',
|
||||
snapshot: {
|
||||
...initial,
|
||||
action: {
|
||||
...initial.action,
|
||||
execution: {
|
||||
...initial.action.execution,
|
||||
status,
|
||||
version: command.expectedExecutionVersion + 1,
|
||||
leaseOwner: null,
|
||||
leaseToken: null,
|
||||
leaseExpiresAtMs: null,
|
||||
resultMutationId: command.mutationId,
|
||||
lastResultCode: command.reasonCode,
|
||||
completedAtMs: command.resolvedAtMs,
|
||||
updatedAtMs: command.resolvedAtMs,
|
||||
},
|
||||
},
|
||||
recovery: {
|
||||
...initial.recovery,
|
||||
status: 'resolved',
|
||||
version: command.expectedRecoveryVersion + 1,
|
||||
executionVersion: command.expectedExecutionVersion + 1,
|
||||
nextScanAtMs: null,
|
||||
resolutionMutationId: command.mutationId,
|
||||
updatedAtMs: command.resolvedAtMs,
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function options() {
|
||||
let now = 200;
|
||||
let id = 0;
|
||||
return {
|
||||
owner: 'resolver-1',
|
||||
leaseDurationMs: 1_000,
|
||||
retryBaseMs: 10,
|
||||
retryMaxMs: 100,
|
||||
clock: () => ++now,
|
||||
createId: () => `recovery-mutation-${++id}`,
|
||||
};
|
||||
}
|
||||
|
||||
function provider(overrides = {}) {
|
||||
return {
|
||||
actionType: 'tool_call',
|
||||
capability: 'automatic',
|
||||
async inspect() {
|
||||
return {
|
||||
finding: 'verified_succeeded',
|
||||
resultCode: 'provider_receipt_verified',
|
||||
evidenceDigest: 'e'.repeat(64),
|
||||
};
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('verified evidence resolves without exposing an execute capability', async () => {
|
||||
const repository = fakeRepository();
|
||||
let inspected = 0;
|
||||
const reconciler = new ApprovedActionRecoveryReconciler(
|
||||
repository,
|
||||
[
|
||||
provider({
|
||||
async inspect(context) {
|
||||
inspected += 1;
|
||||
assert.equal(context.idempotencyKey, 'dispatch-one');
|
||||
assert.equal(context.snapshot.action.execution.status, 'executing');
|
||||
assert.equal('execute' in context, false);
|
||||
return {
|
||||
finding: 'verified_succeeded',
|
||||
resultCode: 'provider_receipt_verified',
|
||||
evidenceDigest: 'e'.repeat(64),
|
||||
};
|
||||
},
|
||||
}),
|
||||
],
|
||||
options(),
|
||||
);
|
||||
const summary = await reconciler.reconcileBatch({ limit: 1 });
|
||||
assert.equal(inspected, 1);
|
||||
assert.deepEqual(
|
||||
repository.calls.map((entry) => entry[0]),
|
||||
['list', 'claim', 'resolve'],
|
||||
);
|
||||
assert.equal(summary.verifiedSucceeded, 1);
|
||||
assert.equal(summary.manualRequired, 0);
|
||||
});
|
||||
|
||||
test('manual-only providers are never inspected and move to manual-required', async () => {
|
||||
const repository = fakeRepository();
|
||||
let inspected = false;
|
||||
const reconciler = new ApprovedActionRecoveryReconciler(
|
||||
repository,
|
||||
[
|
||||
provider({
|
||||
capability: 'manual_only',
|
||||
async inspect() {
|
||||
inspected = true;
|
||||
throw new Error('must not be called');
|
||||
},
|
||||
}),
|
||||
],
|
||||
options(),
|
||||
);
|
||||
const summary = await reconciler.reconcileBatch();
|
||||
assert.equal(inspected, false);
|
||||
const finding = repository.calls.find((entry) => entry[0] === 'finding')[1];
|
||||
assert.equal(finding.finding, 'unsupported');
|
||||
assert.equal('retryAtMs' in finding, false);
|
||||
assert.equal(summary.manualRequired, 1);
|
||||
});
|
||||
|
||||
test('missing and unavailable evidence defer with bounded retry instead of resolving', async () => {
|
||||
for (const inspect of [
|
||||
async () => ({ finding: 'missing', resultCode: 'receipt_missing' }),
|
||||
async () => {
|
||||
throw new Error('provider unavailable');
|
||||
},
|
||||
]) {
|
||||
const repository = fakeRepository();
|
||||
const reconciler = new ApprovedActionRecoveryReconciler(
|
||||
repository,
|
||||
[provider({ inspect })],
|
||||
options(),
|
||||
);
|
||||
const summary = await reconciler.reconcileBatch();
|
||||
const finding = repository.calls.find((entry) => entry[0] === 'finding')[1];
|
||||
assert.ok(['missing', 'unavailable'].includes(finding.finding));
|
||||
assert.ok(finding.retryAtMs > finding.observedAtMs);
|
||||
assert.equal(
|
||||
repository.calls.some((entry) => entry[0] === 'resolve'),
|
||||
false,
|
||||
);
|
||||
assert.equal(summary.deferred, 1);
|
||||
}
|
||||
});
|
||||
|
||||
test('extensible or malformed evidence fails closed into manual review', async () => {
|
||||
const repository = fakeRepository();
|
||||
const reconciler = new ApprovedActionRecoveryReconciler(
|
||||
repository,
|
||||
[
|
||||
provider({
|
||||
async inspect() {
|
||||
return {
|
||||
finding: 'verified_succeeded',
|
||||
resultCode: 'ok',
|
||||
evidenceDigest: 'e'.repeat(64),
|
||||
injected: true,
|
||||
};
|
||||
},
|
||||
}),
|
||||
],
|
||||
options(),
|
||||
);
|
||||
const summary = await reconciler.reconcileBatch();
|
||||
const finding = repository.calls.find((entry) => entry[0] === 'finding')[1];
|
||||
assert.equal(finding.finding, 'conflict');
|
||||
assert.equal(finding.resultCode, 'recovery_evidence_invalid');
|
||||
assert.equal('retryAtMs' in finding, false);
|
||||
assert.equal(summary.manualRequired, 1);
|
||||
});
|
||||
|
||||
test('rejects duplicate providers and keeps each batch to one bounded page', async () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
new ApprovedActionRecoveryReconciler(
|
||||
fakeRepository(),
|
||||
[provider(), provider()],
|
||||
options(),
|
||||
),
|
||||
/Duplicate approved action recovery provider/,
|
||||
);
|
||||
const repository = fakeRepository();
|
||||
const reconciler = new ApprovedActionRecoveryReconciler(
|
||||
repository,
|
||||
[provider()],
|
||||
options(),
|
||||
);
|
||||
await reconciler.reconcileBatch({ limit: 1 });
|
||||
assert.equal(
|
||||
repository.calls.filter((entry) => entry[0] === 'list').length,
|
||||
1,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,353 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
ApprovedActionRuntimeLifecycle,
|
||||
} = require('../../back/runtime/application/approvedActionRuntimeLifecycle');
|
||||
const {
|
||||
ApprovedActionRuntimeSupervisor,
|
||||
} = require('../../back/runtime/application/approvedActionRuntimeSupervisor');
|
||||
const {
|
||||
localPrimaryResourcePolicy,
|
||||
} = require('../../back/runtime/domain/deploymentProfile');
|
||||
|
||||
function dispatchPage(overrides = {}) {
|
||||
return {
|
||||
scanned: 1,
|
||||
claimed: 1,
|
||||
started: 1,
|
||||
succeeded: 1,
|
||||
failed: 0,
|
||||
blocked: 0,
|
||||
retrying: 0,
|
||||
deferred: 0,
|
||||
recoveryRequired: 0,
|
||||
alreadyTerminal: 0,
|
||||
unavailable: 0,
|
||||
truncated: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function recoveryPage(overrides = {}) {
|
||||
return {
|
||||
scanned: 1,
|
||||
claimed: 1,
|
||||
verifiedSucceeded: 1,
|
||||
verifiedFailed: 0,
|
||||
deferred: 0,
|
||||
manualRequired: 0,
|
||||
executionActive: 0,
|
||||
alreadyResolved: 0,
|
||||
unavailable: 0,
|
||||
truncated: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeSummary(overrides = {}) {
|
||||
return {
|
||||
recovery: {
|
||||
pages: 1,
|
||||
...recoveryPage(),
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
...overrides.recovery,
|
||||
},
|
||||
dispatch: {
|
||||
pages: 1,
|
||||
...dispatchPage(),
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
...overrides.dispatch,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function deferred() {
|
||||
let resolve;
|
||||
const promise = new Promise((resolvePromise) => {
|
||||
resolve = resolvePromise;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
function fakeScheduler() {
|
||||
let id = 0;
|
||||
const pending = new Map();
|
||||
const cleared = [];
|
||||
return {
|
||||
pending,
|
||||
cleared,
|
||||
scheduler: {
|
||||
setTimeout(callback, delayMs) {
|
||||
const timer = {
|
||||
id: ++id,
|
||||
delayMs,
|
||||
unrefCalls: 0,
|
||||
unref() {
|
||||
this.unrefCalls += 1;
|
||||
},
|
||||
};
|
||||
pending.set(timer.id, { timer, callback });
|
||||
return timer;
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
cleared.push(timer.id);
|
||||
pending.delete(timer.id);
|
||||
},
|
||||
},
|
||||
fireNext() {
|
||||
const next = pending.values().next().value;
|
||||
assert.ok(next, 'expected a scheduled timer');
|
||||
pending.delete(next.timer.id);
|
||||
next.callback();
|
||||
return next.timer;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function flush() {
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
}
|
||||
|
||||
test('supervisor runs recovery first and aggregates bounded dispatch pages', async () => {
|
||||
const calls = [];
|
||||
let dispatchPageNumber = 0;
|
||||
const supervisor = new ApprovedActionRuntimeSupervisor(
|
||||
{
|
||||
async dispatchBatch(options) {
|
||||
calls.push(['dispatch', options]);
|
||||
dispatchPageNumber += 1;
|
||||
if (dispatchPageNumber === 1) {
|
||||
return dispatchPage({
|
||||
truncated: true,
|
||||
nextCursor: { eligibleAtMs: 11, dispatchId: 'dispatch-1' },
|
||||
});
|
||||
}
|
||||
return dispatchPage({ claimed: 0, succeeded: 0, unavailable: 1 });
|
||||
},
|
||||
},
|
||||
{
|
||||
async reconcileBatch(options) {
|
||||
calls.push(['recovery', options]);
|
||||
return recoveryPage();
|
||||
},
|
||||
},
|
||||
);
|
||||
const summary = await supervisor.runCycle({
|
||||
recovery: { pageSize: 4, maxPages: 1 },
|
||||
dispatch: { pageSize: 8, maxPages: 2 },
|
||||
});
|
||||
assert.deepEqual(calls, [
|
||||
['recovery', { limit: 4 }],
|
||||
['dispatch', { limit: 8 }],
|
||||
[
|
||||
'dispatch',
|
||||
{
|
||||
cursor: { eligibleAtMs: 11, dispatchId: 'dispatch-1' },
|
||||
limit: 8,
|
||||
},
|
||||
],
|
||||
]);
|
||||
assert.equal(summary.recovery.verifiedSucceeded, 1);
|
||||
assert.equal(summary.dispatch.pages, 2);
|
||||
assert.equal(summary.dispatch.scanned, 2);
|
||||
assert.equal(summary.dispatch.claimed, 1);
|
||||
assert.equal(summary.dispatch.succeeded, 1);
|
||||
assert.equal(summary.dispatch.unavailable, 1);
|
||||
assert.equal(summary.dispatch.remaining, false);
|
||||
});
|
||||
|
||||
test('supervisor exposes page-limit cursors and refuses stalled scans', async () => {
|
||||
const dispatchCursor = { eligibleAtMs: 20, dispatchId: 'dispatch-2' };
|
||||
const recoveryCursor = { nextScanAtMs: 30, dispatchId: 'dispatch-3' };
|
||||
const limited = new ApprovedActionRuntimeSupervisor(
|
||||
{
|
||||
async dispatchBatch() {
|
||||
return dispatchPage({ truncated: true, nextCursor: dispatchCursor });
|
||||
},
|
||||
},
|
||||
{
|
||||
async reconcileBatch() {
|
||||
return recoveryPage({ truncated: true, nextCursor: recoveryCursor });
|
||||
},
|
||||
},
|
||||
);
|
||||
const limitedSummary = await limited.runCycle({
|
||||
dispatch: { pageSize: 1, maxPages: 1 },
|
||||
recovery: { pageSize: 1, maxPages: 1 },
|
||||
});
|
||||
assert.equal(limitedSummary.dispatch.stopReason, 'page_limit');
|
||||
assert.deepEqual(limitedSummary.dispatch.nextCursor, dispatchCursor);
|
||||
assert.equal(limitedSummary.recovery.stopReason, 'page_limit');
|
||||
assert.deepEqual(limitedSummary.recovery.nextCursor, recoveryCursor);
|
||||
|
||||
const stalled = new ApprovedActionRuntimeSupervisor(
|
||||
{
|
||||
async dispatchBatch(options) {
|
||||
return dispatchPage({ truncated: true, nextCursor: options.cursor });
|
||||
},
|
||||
},
|
||||
{
|
||||
async reconcileBatch() {
|
||||
return recoveryPage();
|
||||
},
|
||||
},
|
||||
);
|
||||
const stalledSummary = await stalled.runCycle({
|
||||
dispatch: { cursor: dispatchCursor, pageSize: 1, maxPages: 2 },
|
||||
});
|
||||
assert.equal(stalledSummary.dispatch.stopReason, 'cursor_stalled');
|
||||
assert.equal(stalledSummary.dispatch.remaining, true);
|
||||
});
|
||||
|
||||
test('recovery storage failure prevents new dispatch work in the same cycle', async () => {
|
||||
let dispatchCalls = 0;
|
||||
const supervisor = new ApprovedActionRuntimeSupervisor(
|
||||
{
|
||||
async dispatchBatch() {
|
||||
dispatchCalls += 1;
|
||||
return dispatchPage();
|
||||
},
|
||||
},
|
||||
{
|
||||
async reconcileBatch() {
|
||||
throw new Error('recovery index unavailable');
|
||||
},
|
||||
},
|
||||
);
|
||||
await assert.rejects(supervisor.runCycle(), /recovery index unavailable/);
|
||||
assert.equal(dispatchCalls, 0);
|
||||
});
|
||||
|
||||
test('lifecycle uses one unref timer, never overlaps, and resumes both cursors', async () => {
|
||||
const timers = fakeScheduler();
|
||||
const first = deferred();
|
||||
const calls = [];
|
||||
const policy = localPrimaryResourcePolicy('edge').approvedAction;
|
||||
const lifecycle = new ApprovedActionRuntimeLifecycle(
|
||||
{
|
||||
async runCycle(options) {
|
||||
calls.push(options);
|
||||
if (calls.length === 1) return first.promise;
|
||||
return runtimeSummary();
|
||||
},
|
||||
},
|
||||
{
|
||||
intervalMs: policy.intervalMs,
|
||||
initialDelayMs: policy.initialDelayMs,
|
||||
stopTimeoutMs: policy.stopTimeoutMs,
|
||||
cycle: {
|
||||
dispatch: policy.dispatch,
|
||||
recovery: policy.recovery,
|
||||
},
|
||||
scheduler: timers.scheduler,
|
||||
},
|
||||
);
|
||||
assert.equal(timers.pending.size, 0);
|
||||
assert.equal(lifecycle.start(), true);
|
||||
assert.equal(lifecycle.start(), false);
|
||||
const initial = timers.fireNext();
|
||||
assert.equal(initial.delayMs, 0);
|
||||
assert.equal(initial.unrefCalls, 1);
|
||||
await flush();
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(timers.pending.size, 0);
|
||||
first.resolve(
|
||||
runtimeSummary({
|
||||
recovery: {
|
||||
remaining: true,
|
||||
stopReason: 'page_limit',
|
||||
nextCursor: { nextScanAtMs: 40, dispatchId: 'recovery-resume' },
|
||||
},
|
||||
dispatch: {
|
||||
remaining: true,
|
||||
stopReason: 'page_limit',
|
||||
nextCursor: { eligibleAtMs: 50, dispatchId: 'dispatch-resume' },
|
||||
},
|
||||
}),
|
||||
);
|
||||
await flush();
|
||||
const next = timers.fireNext();
|
||||
assert.equal(next.delayMs, 30_000);
|
||||
assert.equal(next.unrefCalls, 1);
|
||||
await flush();
|
||||
assert.deepEqual(calls[1], {
|
||||
recovery: {
|
||||
pageSize: 8,
|
||||
maxPages: 1,
|
||||
cursor: { nextScanAtMs: 40, dispatchId: 'recovery-resume' },
|
||||
},
|
||||
dispatch: {
|
||||
pageSize: 8,
|
||||
maxPages: 1,
|
||||
cursor: { eligibleAtMs: 50, dispatchId: 'dispatch-resume' },
|
||||
},
|
||||
});
|
||||
assert.equal(await lifecycle.stop(), 'drained');
|
||||
});
|
||||
|
||||
test('lifecycle isolates diagnostics and bounds an in-flight shutdown', async () => {
|
||||
const timers = fakeScheduler();
|
||||
const running = deferred();
|
||||
const errors = [];
|
||||
const lifecycle = new ApprovedActionRuntimeLifecycle(
|
||||
{
|
||||
async runCycle() {
|
||||
return running.promise;
|
||||
},
|
||||
},
|
||||
{
|
||||
intervalMs: 500,
|
||||
stopTimeoutMs: 5,
|
||||
scheduler: timers.scheduler,
|
||||
onError(error) {
|
||||
errors.push(error.message);
|
||||
},
|
||||
},
|
||||
);
|
||||
lifecycle.start();
|
||||
timers.fireNext();
|
||||
await flush();
|
||||
assert.equal(await lifecycle.stop(), 'timed_out');
|
||||
assert.equal(timers.pending.size, 0);
|
||||
assert.equal(lifecycle.start(), false);
|
||||
running.resolve(runtimeSummary());
|
||||
await flush();
|
||||
assert.equal(lifecycle.start(), true);
|
||||
assert.equal(await lifecycle.stop(), 'drained');
|
||||
assert.deepEqual(errors, []);
|
||||
});
|
||||
|
||||
test('supervisor and lifecycle reject hot loops and unbounded pages', async () => {
|
||||
const supervisor = new ApprovedActionRuntimeSupervisor(
|
||||
{
|
||||
async dispatchBatch() {
|
||||
return dispatchPage();
|
||||
},
|
||||
},
|
||||
{
|
||||
async reconcileBatch() {
|
||||
return recoveryPage();
|
||||
},
|
||||
},
|
||||
);
|
||||
await assert.rejects(
|
||||
supervisor.runCycle({ dispatch: { maxPages: 65 } }),
|
||||
RangeError,
|
||||
);
|
||||
assert.throws(
|
||||
() => new ApprovedActionRuntimeLifecycle(supervisor, { intervalMs: 249 }),
|
||||
RangeError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
new ApprovedActionRuntimeLifecycle(supervisor, {
|
||||
intervalMs: 500,
|
||||
stopTimeoutMs: 60_001,
|
||||
}),
|
||||
RangeError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,618 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const { QueryTypes, Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const { runSchemaMigration } = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
runCancellationRequestMigration,
|
||||
} = require('../../back/migrations/0004-run-cancellation-request');
|
||||
const {
|
||||
runAttemptDeadlineMigration,
|
||||
} = require('../../back/migrations/0006-run-attempt-deadline');
|
||||
const {
|
||||
runRetryPolicyMigration,
|
||||
} = require('../../back/migrations/0011-run-retry-policy');
|
||||
const {
|
||||
projectPolicyMigration,
|
||||
} = require('../../back/migrations/0017-project-policy');
|
||||
const {
|
||||
approvalRequestMigration,
|
||||
} = require('../../back/migrations/0020-approval-requests');
|
||||
const {
|
||||
approvedActionDispatchExecutionMigration,
|
||||
} = require('../../back/migrations/0021-approved-action-dispatch-executions');
|
||||
const {
|
||||
approvedActionRecoveryMigration,
|
||||
} = require('../../back/migrations/0022-approved-action-recovery');
|
||||
const {
|
||||
APPROVED_RUN_ACTION_RECEIPT_PROJECT_INDEX,
|
||||
APPROVED_RUN_ACTION_RECEIPT_RESOURCE_UNIQUE_INDEX,
|
||||
APPROVED_RUN_ACTION_RECEIPT_TABLE,
|
||||
approvedRunActionReceiptMigration,
|
||||
} = require('../../back/migrations/0023-approved-run-action-receipts');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeApprovedActionDispatchRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/approvedActionDispatchRepository');
|
||||
const {
|
||||
LegacySequelizeApprovedActionRecoveryRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/approvedActionRecoveryRepository');
|
||||
const {
|
||||
LegacySequelizeApprovedRunActionRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/approvedRunActionRepository');
|
||||
const {
|
||||
LegacySequelizeApprovedRunRecoveryEvidenceProvider,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/approvedRunRecoveryEvidenceProvider');
|
||||
const {
|
||||
LegacySequelizeApprovalRequestRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/approvalRequestRepository');
|
||||
const {
|
||||
LegacySequelizeProjectPolicyRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/projectPolicyRepository');
|
||||
const {
|
||||
LegacySequelizeRunRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/runRepository');
|
||||
const {
|
||||
ApprovedActionDispatcher,
|
||||
} = require('../../back/runtime/application/approvedActionDispatcher');
|
||||
const {
|
||||
ApprovedActionRecoveryReconciler,
|
||||
} = require('../../back/runtime/application/approvedActionRecoveryReconciler');
|
||||
const {
|
||||
ApprovedRunActionHandler,
|
||||
} = require('../../back/runtime/application/approvedRunActionHandler');
|
||||
const {
|
||||
ApprovalRequestService,
|
||||
} = require('../../back/runtime/application/approvalRequestService');
|
||||
const {
|
||||
PrimaryRunCreator,
|
||||
} = require('../../back/runtime/application/primaryRunCreator');
|
||||
const {
|
||||
ProjectPolicyEngine,
|
||||
} = require('../../back/runtime/application/projectPolicyEngine');
|
||||
const {
|
||||
ApprovedRunActionBindingConflictError,
|
||||
ApprovedRunActionRepositoryError,
|
||||
InvalidApprovedRunActionError,
|
||||
digestApprovedRunCreationPlan,
|
||||
normalizeApprovedRunCreationPlan,
|
||||
} = require('../../back/runtime/domain/approvedRunAction');
|
||||
|
||||
const PROJECT_ID = 'default';
|
||||
const AGENT = Object.freeze({ type: 'agent', id: 'agent-1' });
|
||||
const OWNER = Object.freeze({ type: 'user', id: 'owner-1' });
|
||||
const SYSTEM = Object.freeze({ type: 'system', id: 'approval-dispatcher' });
|
||||
const BASE_TIME = 200_000;
|
||||
|
||||
function plan(name = 'one') {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
actionRef: `approved-run-${name}`,
|
||||
projectId: PROJECT_ID,
|
||||
taskId: `task-${name}`,
|
||||
taskRevision: `revision-${name}`,
|
||||
executorType: 'local_process',
|
||||
priority: 3,
|
||||
taskName: `Task ${name}`,
|
||||
taskSnapshotRef: `task-snapshot:${name}`,
|
||||
inputRef: `input:${name}`,
|
||||
};
|
||||
}
|
||||
|
||||
async function migrate(database) {
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
runCancellationRequestMigration,
|
||||
runAttemptDeadlineMigration,
|
||||
runRetryPolicyMigration,
|
||||
projectPolicyMigration,
|
||||
approvalRequestMigration,
|
||||
approvedActionDispatchExecutionMigration,
|
||||
approvedActionRecoveryMigration,
|
||||
approvedRunActionReceiptMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
}
|
||||
|
||||
async function bind(repository, subject, role, mutationId) {
|
||||
await repository.append({
|
||||
expectedCurrentVersion: 0,
|
||||
binding: {
|
||||
projectId: PROJECT_ID,
|
||||
subject,
|
||||
version: 1,
|
||||
state: 'active',
|
||||
role,
|
||||
mutationId,
|
||||
changedBy: OWNER,
|
||||
createdAtMs: BASE_TIME - 100,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function setup(t) {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
await migrate(database);
|
||||
const policy = new LegacySequelizeProjectPolicyRepository(database);
|
||||
await bind(policy, OWNER, 'owner', 'bind-owner');
|
||||
await bind(policy, AGENT, 'operator', 'bind-agent');
|
||||
const approvals = new ApprovalRequestService(
|
||||
new LegacySequelizeApprovalRequestRepository(database),
|
||||
new ProjectPolicyEngine(policy),
|
||||
);
|
||||
return {
|
||||
database,
|
||||
approvals,
|
||||
executions: new LegacySequelizeApprovedActionDispatchRepository(database),
|
||||
};
|
||||
}
|
||||
|
||||
async function prepareApprovedRun(
|
||||
setupResult,
|
||||
name,
|
||||
approvedPlan = plan(name),
|
||||
) {
|
||||
const actionDigest = digestApprovedRunCreationPlan(approvedPlan);
|
||||
const action = {
|
||||
permission: 'run.start',
|
||||
actionType: 'run.create',
|
||||
actionRef: approvedPlan.actionRef,
|
||||
actionDigest,
|
||||
previewDigest: 'f'.repeat(64),
|
||||
};
|
||||
const requestedAtMs = BASE_TIME + 100;
|
||||
await setupResult.approvals.create({
|
||||
id: `approval-${name}`,
|
||||
projectId: PROJECT_ID,
|
||||
action,
|
||||
risk: 'high',
|
||||
requestedBy: AGENT,
|
||||
requestedAtMs,
|
||||
expiresAtMs: requestedAtMs + 60_000,
|
||||
});
|
||||
await setupResult.approvals.decide({
|
||||
requestId: `approval-${name}`,
|
||||
expectedVersion: 1,
|
||||
decisionId: `decision-${name}`,
|
||||
decision: 'approved',
|
||||
reasonCode: 'reviewed_action',
|
||||
decidedBy: OWNER,
|
||||
decidedAtMs: requestedAtMs + 10,
|
||||
});
|
||||
const consumed = await setupResult.approvals.consume({
|
||||
requestId: `approval-${name}`,
|
||||
expectedVersion: 2,
|
||||
consumptionId: `consumption-${name}`,
|
||||
dispatchId: `dispatch-${name}`,
|
||||
action,
|
||||
requestedBy: AGENT,
|
||||
consumedBy: SYSTEM,
|
||||
consumedAtMs: requestedAtMs + 20,
|
||||
});
|
||||
return consumed.dispatch;
|
||||
}
|
||||
|
||||
async function startApprovedRun(setupResult, name, approvedPlan = plan(name)) {
|
||||
const dispatch = await prepareApprovedRun(setupResult, name, approvedPlan);
|
||||
const claimed = await setupResult.executions.claim({
|
||||
dispatchId: dispatch.id,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: `lease-${name}`,
|
||||
nowMs: BASE_TIME + 130,
|
||||
leaseDurationMs: 1_000,
|
||||
});
|
||||
assert.equal(claimed.status, 'claimed');
|
||||
return setupResult.executions.start({
|
||||
dispatchId: dispatch.id,
|
||||
approvalRequestId: dispatch.approvalRequestId,
|
||||
actionDigest: dispatch.action.actionDigest,
|
||||
owner: 'dispatcher-1',
|
||||
leaseToken: `lease-${name}`,
|
||||
expectedVersion: claimed.snapshot.execution.version,
|
||||
startedAtMs: BASE_TIME + 140,
|
||||
});
|
||||
}
|
||||
|
||||
function recoveryContext(snapshot) {
|
||||
return {
|
||||
snapshot: { action: snapshot },
|
||||
idempotencyKey: snapshot.dispatch.id,
|
||||
observedAtMs: BASE_TIME + 10_000,
|
||||
};
|
||||
}
|
||||
|
||||
test('canonical Run action plans are exact-shape and digest-stable', () => {
|
||||
const first = plan('canonical');
|
||||
const reordered = {
|
||||
inputRef: first.inputRef,
|
||||
priority: first.priority,
|
||||
executorType: first.executorType,
|
||||
taskRevision: first.taskRevision,
|
||||
taskId: first.taskId,
|
||||
projectId: first.projectId,
|
||||
actionRef: first.actionRef,
|
||||
schemaVersion: first.schemaVersion,
|
||||
taskSnapshotRef: first.taskSnapshotRef,
|
||||
taskName: first.taskName,
|
||||
};
|
||||
assert.equal(
|
||||
digestApprovedRunCreationPlan(first),
|
||||
digestApprovedRunCreationPlan(reordered),
|
||||
);
|
||||
assert.throws(
|
||||
() => normalizeApprovedRunCreationPlan({ ...first, hidden: true }),
|
||||
InvalidApprovedRunActionError,
|
||||
);
|
||||
});
|
||||
|
||||
test('0023 owns bounded receipt indexes and enforces immutable tuple checks', async (t) => {
|
||||
const state = await setup(t);
|
||||
const { database } = state;
|
||||
const indexes = await database
|
||||
.getQueryInterface()
|
||||
.showIndex(APPROVED_RUN_ACTION_RECEIPT_TABLE);
|
||||
assert.ok(
|
||||
indexes.some(
|
||||
(index) => index.name === APPROVED_RUN_ACTION_RECEIPT_PROJECT_INDEX,
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
indexes.some(
|
||||
(index) =>
|
||||
index.name === APPROVED_RUN_ACTION_RECEIPT_RESOURCE_UNIQUE_INDEX &&
|
||||
index.unique,
|
||||
),
|
||||
);
|
||||
const approvedPlan = plan('constraints');
|
||||
const snapshot = await startApprovedRun(state, 'constraints', approvedPlan);
|
||||
await new LegacySequelizeApprovedRunActionRepository(database, {
|
||||
clock: () => snapshot.execution.startedAtMs + 5,
|
||||
}).create({ snapshot, plan: approvedPlan });
|
||||
await assert.rejects(
|
||||
database.query(
|
||||
`UPDATE "${APPROVED_RUN_ACTION_RECEIPT_TABLE}"
|
||||
SET outcome = 'failed'
|
||||
WHERE dispatch_id = :dispatchId`,
|
||||
{ replacements: { dispatchId: snapshot.dispatch.id } },
|
||||
),
|
||||
);
|
||||
await assert.rejects(
|
||||
database.query(
|
||||
`UPDATE "${APPROVED_RUN_ACTION_RECEIPT_TABLE}"
|
||||
SET idempotency_key = 'different-dispatch'
|
||||
WHERE dispatch_id = :dispatchId`,
|
||||
{ replacements: { dispatchId: snapshot.dispatch.id } },
|
||||
),
|
||||
);
|
||||
await assert.rejects(
|
||||
database.query(
|
||||
`UPDATE "${APPROVED_RUN_ACTION_RECEIPT_TABLE}"
|
||||
SET created_at_ms = finished_at_ms + 1
|
||||
WHERE dispatch_id = :dispatchId`,
|
||||
{ replacements: { dispatchId: snapshot.dispatch.id } },
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('handler atomically creates one queued Run and a fully bound receipt', async (t) => {
|
||||
const state = await setup(t);
|
||||
const approvedPlan = plan('atomic');
|
||||
const snapshot = await startApprovedRun(state, 'atomic', approvedPlan);
|
||||
const repository = new LegacySequelizeApprovedRunActionRepository(
|
||||
state.database,
|
||||
{
|
||||
clock: () => snapshot.execution.startedAtMs + 5,
|
||||
createId: (() => {
|
||||
const ids = [
|
||||
'019f8000-0000-7000-8000-000000000001',
|
||||
'019f8000-0000-7000-8000-000000000002',
|
||||
'019f8000-0000-7000-8000-000000000003',
|
||||
'019f8000-0000-7000-8000-000000000004',
|
||||
];
|
||||
return () => ids.shift();
|
||||
})(),
|
||||
},
|
||||
);
|
||||
const resolver = {
|
||||
async resolve() {
|
||||
return approvedPlan;
|
||||
},
|
||||
};
|
||||
const handler = new ApprovedRunActionHandler(resolver, repository);
|
||||
const inspection = await handler.inspect(snapshot.dispatch);
|
||||
assert.deepEqual(inspection, {
|
||||
status: 'ready',
|
||||
actionDigest: snapshot.dispatch.action.actionDigest,
|
||||
});
|
||||
const context = {
|
||||
dispatch: snapshot.dispatch,
|
||||
execution: snapshot.execution,
|
||||
idempotencyKey: snapshot.dispatch.id,
|
||||
fence: {
|
||||
owner: snapshot.execution.leaseOwner,
|
||||
leaseToken: snapshot.execution.leaseToken,
|
||||
version: snapshot.execution.version,
|
||||
},
|
||||
};
|
||||
assert.deepEqual(await handler.execute(context), {
|
||||
outcome: 'succeeded',
|
||||
resultCode: 'approved_run_created',
|
||||
});
|
||||
assert.deepEqual(await handler.execute(context), {
|
||||
outcome: 'succeeded',
|
||||
resultCode: 'approved_run_created',
|
||||
});
|
||||
|
||||
const runs = await state.database.query('SELECT * FROM "Runs"', {
|
||||
type: QueryTypes.SELECT,
|
||||
});
|
||||
const receipts = await state.database.query(
|
||||
`SELECT * FROM "${APPROVED_RUN_ACTION_RECEIPT_TABLE}"`,
|
||||
{ type: QueryTypes.SELECT },
|
||||
);
|
||||
assert.equal(runs.length, 1);
|
||||
assert.equal(receipts.length, 1);
|
||||
assert.equal(runs[0].status, 'queued');
|
||||
assert.equal(runs[0].idempotency_key, snapshot.dispatch.id);
|
||||
assert.equal(runs[0].request_id, snapshot.dispatch.approvalRequestId);
|
||||
assert.equal(receipts[0].resource_id, runs[0].id);
|
||||
assert.equal(receipts[0].execution_attempt, snapshot.execution.attemptCount);
|
||||
assert.equal(receipts[0].execution_version, snapshot.execution.version);
|
||||
assert.equal(receipts[0].started_at_ms, snapshot.execution.startedAtMs);
|
||||
});
|
||||
|
||||
test('receipt insertion failure rolls the Run aggregate back with it', async (t) => {
|
||||
const state = await setup(t);
|
||||
const approvedPlan = plan('rollback');
|
||||
const snapshot = await startApprovedRun(state, 'rollback', approvedPlan);
|
||||
await state.database.query(
|
||||
`CREATE TRIGGER reject_approved_run_receipt
|
||||
BEFORE INSERT ON "${APPROVED_RUN_ACTION_RECEIPT_TABLE}"
|
||||
BEGIN SELECT RAISE(ABORT, 'receipt rejected'); END`,
|
||||
);
|
||||
const repository = new LegacySequelizeApprovedRunActionRepository(
|
||||
state.database,
|
||||
{ clock: () => snapshot.execution.startedAtMs + 5 },
|
||||
);
|
||||
await assert.rejects(
|
||||
repository.create({ snapshot, plan: approvedPlan }),
|
||||
ApprovedRunActionRepositoryError,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await state.database.query('SELECT id FROM "Runs"', {
|
||||
type: QueryTypes.SELECT,
|
||||
})
|
||||
).length,
|
||||
0,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await state.database.query(
|
||||
'SELECT dispatch_id FROM "ApprovedRunActionReceipts"',
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
},
|
||||
)
|
||||
).length,
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
test('atomic creation accepts same-fence renew and records the current version', async (t) => {
|
||||
const state = await setup(t);
|
||||
const approvedPlan = plan('renewed');
|
||||
const started = await startApprovedRun(state, 'renewed', approvedPlan);
|
||||
const renewed = await state.executions.renew({
|
||||
dispatchId: started.dispatch.id,
|
||||
owner: started.execution.leaseOwner,
|
||||
leaseToken: started.execution.leaseToken,
|
||||
expectedVersion: started.execution.version,
|
||||
nowMs: started.execution.startedAtMs + 10,
|
||||
leaseDurationMs: 1_000,
|
||||
});
|
||||
const repository = new LegacySequelizeApprovedRunActionRepository(
|
||||
state.database,
|
||||
{ clock: () => started.execution.startedAtMs + 20 },
|
||||
);
|
||||
await repository.create({ snapshot: started, plan: approvedPlan });
|
||||
const [receipt] = await state.database.query(
|
||||
`SELECT execution_version FROM "${APPROVED_RUN_ACTION_RECEIPT_TABLE}"`,
|
||||
{ type: QueryTypes.SELECT },
|
||||
);
|
||||
assert.equal(receipt.execution_version, renewed.execution.version);
|
||||
const provider = new LegacySequelizeApprovedRunRecoveryEvidenceProvider(
|
||||
state.database,
|
||||
);
|
||||
assert.equal(
|
||||
(await provider.inspect(recoveryContext(renewed))).finding,
|
||||
'verified_succeeded',
|
||||
);
|
||||
});
|
||||
|
||||
test('terminal resolution fences stale handler context before any Run write', async (t) => {
|
||||
const state = await setup(t);
|
||||
const approvedPlan = plan('fenced');
|
||||
const started = await startApprovedRun(state, 'fenced', approvedPlan);
|
||||
await state.executions.complete({
|
||||
dispatchId: started.dispatch.id,
|
||||
owner: started.execution.leaseOwner,
|
||||
leaseToken: started.execution.leaseToken,
|
||||
expectedVersion: started.execution.version,
|
||||
resultMutationId: 'terminal-before-action',
|
||||
outcome: 'failed',
|
||||
resultCode: 'fenced_before_action',
|
||||
completedAtMs: started.execution.startedAtMs + 10,
|
||||
});
|
||||
const repository = new LegacySequelizeApprovedRunActionRepository(
|
||||
state.database,
|
||||
{ clock: () => started.execution.startedAtMs + 20 },
|
||||
);
|
||||
await assert.rejects(
|
||||
repository.create({ snapshot: started, plan: approvedPlan }),
|
||||
ApprovedRunActionBindingConflictError,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await state.database.query('SELECT id FROM "Runs"', {
|
||||
type: QueryTypes.SELECT,
|
||||
})
|
||||
).length,
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
test('evidence provider verifies only the atomic receipt and bound Run fact', async (t) => {
|
||||
const state = await setup(t);
|
||||
const approvedPlan = plan('evidence');
|
||||
const snapshot = await startApprovedRun(state, 'evidence', approvedPlan);
|
||||
const provider = new LegacySequelizeApprovedRunRecoveryEvidenceProvider(
|
||||
state.database,
|
||||
);
|
||||
assert.deepEqual(await provider.inspect(recoveryContext(snapshot)), {
|
||||
finding: 'missing',
|
||||
resultCode: 'approved_run_receipt_missing',
|
||||
});
|
||||
const repository = new LegacySequelizeApprovedRunActionRepository(
|
||||
state.database,
|
||||
{ clock: () => snapshot.execution.startedAtMs + 5 },
|
||||
);
|
||||
await repository.create({ snapshot, plan: approvedPlan });
|
||||
const evidence = await provider.inspect(recoveryContext(snapshot));
|
||||
assert.equal(evidence.finding, 'verified_succeeded');
|
||||
assert.equal(evidence.resultCode, 'approved_run_receipt_verified');
|
||||
assert.match(evidence.evidenceDigest, /^[0-9a-f]{64}$/);
|
||||
|
||||
await state.database.query(
|
||||
`UPDATE "${APPROVED_RUN_ACTION_RECEIPT_TABLE}"
|
||||
SET action_digest = :digest
|
||||
WHERE dispatch_id = :dispatchId`,
|
||||
{
|
||||
replacements: {
|
||||
digest: '0'.repeat(64),
|
||||
dispatchId: snapshot.dispatch.id,
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.deepEqual(await provider.inspect(recoveryContext(snapshot)), {
|
||||
finding: 'conflict',
|
||||
resultCode: 'approved_run_receipt_conflict',
|
||||
});
|
||||
});
|
||||
|
||||
test('a Run idempotency collision without an atomic receipt is conflict, not success', async (t) => {
|
||||
const state = await setup(t);
|
||||
const approvedPlan = plan('collision');
|
||||
const snapshot = await startApprovedRun(state, 'collision', approvedPlan);
|
||||
const creator = new PrimaryRunCreator(
|
||||
new LegacySequelizeRunRepository(state.database),
|
||||
);
|
||||
await creator.create(
|
||||
{
|
||||
projectId: PROJECT_ID,
|
||||
taskId: approvedPlan.taskId,
|
||||
taskRevision: approvedPlan.taskRevision,
|
||||
triggerType: 'untrusted_collision',
|
||||
executionOrigin: 'system',
|
||||
requestId: snapshot.dispatch.approvalRequestId,
|
||||
idempotencyKey: snapshot.dispatch.id,
|
||||
acceptedAtMs: snapshot.execution.startedAtMs,
|
||||
actor: { type: 'system' },
|
||||
},
|
||||
'local_process',
|
||||
);
|
||||
const provider = new LegacySequelizeApprovedRunRecoveryEvidenceProvider(
|
||||
state.database,
|
||||
);
|
||||
assert.deepEqual(await provider.inspect(recoveryContext(snapshot)), {
|
||||
finding: 'conflict',
|
||||
resultCode: 'approved_run_receipt_conflict',
|
||||
});
|
||||
});
|
||||
|
||||
test('receipt recovers a dispatcher crash after the Run commit but before completion', async (t) => {
|
||||
const state = await setup(t);
|
||||
const approvedPlan = plan('crash-window');
|
||||
const dispatch = await prepareApprovedRun(
|
||||
state,
|
||||
'crash-window',
|
||||
approvedPlan,
|
||||
);
|
||||
const crashingRepository = {
|
||||
findById: (...args) => state.executions.findById(...args),
|
||||
listDue: (...args) => state.executions.listDue(...args),
|
||||
claim: (...args) => state.executions.claim(...args),
|
||||
start: (...args) => state.executions.start(...args),
|
||||
renew: (...args) => state.executions.renew(...args),
|
||||
releaseBeforeStart: (...args) =>
|
||||
state.executions.releaseBeforeStart(...args),
|
||||
async complete() {
|
||||
throw new Error('simulated completion persistence outage');
|
||||
},
|
||||
};
|
||||
const actionRepository = new LegacySequelizeApprovedRunActionRepository(
|
||||
state.database,
|
||||
{ clock: () => BASE_TIME + 1_010 },
|
||||
);
|
||||
const handler = new ApprovedRunActionHandler(
|
||||
{
|
||||
async resolve() {
|
||||
return approvedPlan;
|
||||
},
|
||||
},
|
||||
actionRepository,
|
||||
);
|
||||
let dispatchNow = BASE_TIME + 1_000;
|
||||
let dispatchId = 0;
|
||||
const dispatcher = new ApprovedActionDispatcher(
|
||||
crashingRepository,
|
||||
[handler],
|
||||
{
|
||||
owner: 'dispatcher-crash-test',
|
||||
leaseDurationMs: 1_000,
|
||||
clock: () => ++dispatchNow,
|
||||
createId: () => `dispatch-mutation-${++dispatchId}`,
|
||||
},
|
||||
);
|
||||
const dispatched = await dispatcher.dispatchBatch({ limit: 1 });
|
||||
assert.equal(dispatched.started, 1);
|
||||
assert.equal(dispatched.succeeded, 0);
|
||||
assert.equal(dispatched.unavailable, 1);
|
||||
assert.equal(dispatched.recoveryRequired, 1);
|
||||
assert.equal(
|
||||
(await state.executions.findById(dispatch.id)).execution.status,
|
||||
'executing',
|
||||
);
|
||||
|
||||
let recoveryNow = BASE_TIME + 3_000;
|
||||
let recoveryId = 0;
|
||||
const reconciler = new ApprovedActionRecoveryReconciler(
|
||||
new LegacySequelizeApprovedActionRecoveryRepository(state.database),
|
||||
[new LegacySequelizeApprovedRunRecoveryEvidenceProvider(state.database)],
|
||||
{
|
||||
owner: 'recovery-crash-test',
|
||||
leaseDurationMs: 1_000,
|
||||
clock: () => ++recoveryNow,
|
||||
createId: () => `recovery-mutation-${++recoveryId}`,
|
||||
},
|
||||
);
|
||||
const recovered = await reconciler.reconcileBatch({ limit: 1 });
|
||||
assert.equal(recovered.verifiedSucceeded, 1);
|
||||
assert.equal(
|
||||
(await state.executions.findById(dispatch.id)).execution.status,
|
||||
'succeeded',
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,227 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
bootstrapDefaultManualPrimaryRuntime,
|
||||
} = require('../../back/runtime/adapters/legacy/bootstrapDefaultManualPrimaryRuntime');
|
||||
const {
|
||||
RuntimeRolloutPolicy,
|
||||
} = require('../../back/runtime/domain/runtimeRollout');
|
||||
|
||||
const NOW = 1_750_000_000_000;
|
||||
|
||||
function loadResult(status, mode = 'off') {
|
||||
return {
|
||||
status,
|
||||
policy: new RuntimeRolloutPolicy({
|
||||
defaultMode: 'off',
|
||||
origins: mode === 'off' ? {} : { manual: mode },
|
||||
allowLegacyFallbackBeforeStart: false,
|
||||
}),
|
||||
audit: {
|
||||
event: 'runtime.rollout_config_evaluated',
|
||||
evaluatedAtMs: NOW,
|
||||
sourcePath: '/data/config/qinglong3-rollout.json',
|
||||
status,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('default-off bootstrap does not import or construct the Primary stack', async () => {
|
||||
const calls = [];
|
||||
const result = await bootstrapDefaultManualPrimaryRuntime({
|
||||
load: async () => loadResult('missing'),
|
||||
async loadStack() {
|
||||
calls.push('load-stack');
|
||||
throw new Error('must remain lazy');
|
||||
},
|
||||
install() {
|
||||
calls.push('install');
|
||||
return () => undefined;
|
||||
},
|
||||
audit(record) {
|
||||
calls.push(`audit:${record.activation}`);
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.active, false);
|
||||
assert.deepEqual(calls, ['audit:not_activated']);
|
||||
});
|
||||
|
||||
test('accepted bootstrap lazily loads the stack and delegates activation', async () => {
|
||||
const calls = [];
|
||||
const result = await bootstrapDefaultManualPrimaryRuntime({
|
||||
load: async () => loadResult('accepted', 'primary'),
|
||||
async loadStack() {
|
||||
calls.push('load-stack');
|
||||
return {
|
||||
createDefaultManualPrimaryActivationStack(_rollout, options) {
|
||||
calls.push('create-stack');
|
||||
calls.push(`profile:${options.deploymentProfile}`);
|
||||
return {
|
||||
router: {
|
||||
ownsNewRuns: () => true,
|
||||
async start() {
|
||||
throw new Error('not used');
|
||||
},
|
||||
async stopCron() {
|
||||
return { matched: 0, failed: 0 };
|
||||
},
|
||||
async stopAttempt() {
|
||||
return { matched: 0, failed: 0 };
|
||||
},
|
||||
},
|
||||
async reconcile() {
|
||||
calls.push('reconcile');
|
||||
return {
|
||||
pages: 1,
|
||||
scanned: 0,
|
||||
verifiedRunning: 0,
|
||||
recoveredRunning: 0,
|
||||
completedFromReceipt: 0,
|
||||
quarantinedReceipts: 0,
|
||||
publishGraceWaits: 0,
|
||||
markedLost: 0,
|
||||
skipped: 0,
|
||||
ambiguous: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
},
|
||||
startCompletion() {
|
||||
calls.push('start-completion');
|
||||
return true;
|
||||
},
|
||||
async stopCompletion() {
|
||||
calls.push('stop-completion');
|
||||
return 'drained';
|
||||
},
|
||||
startTimeout() {
|
||||
calls.push('start-timeout');
|
||||
return true;
|
||||
},
|
||||
async stopTimeout() {
|
||||
calls.push('stop-timeout');
|
||||
return 'drained';
|
||||
},
|
||||
startCancellation() {
|
||||
calls.push('start-cancellation');
|
||||
return true;
|
||||
},
|
||||
async stopCancellation() {
|
||||
calls.push('stop-cancellation');
|
||||
return 'drained';
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
},
|
||||
install() {
|
||||
calls.push('install');
|
||||
return () => calls.push('dispose');
|
||||
},
|
||||
audit(record) {
|
||||
calls.push(`audit:${record.activation}`);
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.active, true);
|
||||
await result.stop();
|
||||
assert.deepEqual(calls, [
|
||||
'load-stack',
|
||||
'audit:selected',
|
||||
'create-stack',
|
||||
'profile:standalone',
|
||||
'reconcile',
|
||||
'audit:reconciled',
|
||||
'start-completion',
|
||||
'start-timeout',
|
||||
'start-cancellation',
|
||||
'install',
|
||||
'audit:activated',
|
||||
'dispose',
|
||||
'stop-timeout',
|
||||
'stop-cancellation',
|
||||
'stop-completion',
|
||||
'audit:stopped',
|
||||
]);
|
||||
});
|
||||
|
||||
test('accepted bootstrap audits a lazy stack import failure without installing', async () => {
|
||||
const calls = [];
|
||||
await assert.rejects(
|
||||
bootstrapDefaultManualPrimaryRuntime({
|
||||
load: async () => loadResult('accepted', 'primary'),
|
||||
async loadStack() {
|
||||
calls.push('load-stack');
|
||||
throw new Error('stack import failed');
|
||||
},
|
||||
install() {
|
||||
calls.push('install');
|
||||
return () => calls.push('dispose');
|
||||
},
|
||||
audit(record) {
|
||||
calls.push(`audit:${record.activation}`);
|
||||
},
|
||||
}),
|
||||
/stack import failed/,
|
||||
);
|
||||
assert.deepEqual(calls, ['load-stack', 'audit:failed']);
|
||||
});
|
||||
|
||||
test('disabled bootstrap stays inert even with an invalid deployment profile', async () => {
|
||||
const previous = process.env.QL_DEPLOYMENT_PROFILE;
|
||||
process.env.QL_DEPLOYMENT_PROFILE = 'invalid-profile';
|
||||
try {
|
||||
const result = await bootstrapDefaultManualPrimaryRuntime({
|
||||
load: async () => loadResult('disabled'),
|
||||
async loadStack() {
|
||||
throw new Error('must remain lazy');
|
||||
},
|
||||
install() {
|
||||
throw new Error('must remain uninstalled');
|
||||
},
|
||||
audit() {},
|
||||
});
|
||||
assert.equal(result.active, false);
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.QL_DEPLOYMENT_PROFILE;
|
||||
else process.env.QL_DEPLOYMENT_PROFILE = previous;
|
||||
}
|
||||
});
|
||||
|
||||
test('accepted bootstrap rejects and audits an invalid deployment profile', async () => {
|
||||
const previous = process.env.QL_DEPLOYMENT_PROFILE;
|
||||
process.env.QL_DEPLOYMENT_PROFILE = 'invalid-profile';
|
||||
const calls = [];
|
||||
try {
|
||||
await assert.rejects(
|
||||
bootstrapDefaultManualPrimaryRuntime({
|
||||
load: async () => loadResult('accepted', 'primary'),
|
||||
async loadStack() {
|
||||
calls.push('load-stack');
|
||||
return {
|
||||
createDefaultManualPrimaryActivationStack() {
|
||||
calls.push('create-stack');
|
||||
throw new Error('profile must be rejected first');
|
||||
},
|
||||
};
|
||||
},
|
||||
install() {
|
||||
calls.push('install');
|
||||
return () => calls.push('dispose');
|
||||
},
|
||||
audit(record) {
|
||||
calls.push(`audit:${record.activation}`);
|
||||
},
|
||||
}),
|
||||
/QL_DEPLOYMENT_PROFILE is invalid/,
|
||||
);
|
||||
assert.deepEqual(calls, ['load-stack', 'audit:selected', 'audit:failed']);
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.QL_DEPLOYMENT_PROFILE;
|
||||
else process.env.QL_DEPLOYMENT_PROFILE = previous;
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,447 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const { QueryTypes, Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_EVENT_TABLE,
|
||||
RUN_TABLE,
|
||||
runSchemaMigration,
|
||||
} = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
runCancellationRequestMigration,
|
||||
} = require('../../back/migrations/0004-run-cancellation-request');
|
||||
const {
|
||||
runCancellationDispatchMigration,
|
||||
RUN_CANCELLATION_DISPATCH_TABLE,
|
||||
} = require('../../back/migrations/0005-run-cancellation-dispatch');
|
||||
const {
|
||||
runAttemptDeadlineMigration,
|
||||
} = require('../../back/migrations/0006-run-attempt-deadline');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeCancellationDispatchRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/cancellationDispatchRepository');
|
||||
const {
|
||||
CancellationDispatchBindingConflictError,
|
||||
CancellationDispatchFenceRejectedError,
|
||||
CancellationDispatchRepositoryError,
|
||||
} = require('../../back/runtime/domain/cancellationDispatchErrors');
|
||||
const {
|
||||
PrimaryCancellationDispatcher,
|
||||
} = require('../../back/runtime/application/primaryCancellationDispatcher');
|
||||
|
||||
const databases = [];
|
||||
let idSequence = 200;
|
||||
|
||||
function nextId() {
|
||||
idSequence += 1;
|
||||
return `019f71c0-0000-7000-8000-${String(idSequence).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
async function createRepository() {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
runCancellationRequestMigration,
|
||||
runCancellationDispatchMigration,
|
||||
runAttemptDeadlineMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
databases.push(database);
|
||||
return {
|
||||
database,
|
||||
repository: new LegacySequelizeCancellationDispatchRepository(database),
|
||||
};
|
||||
}
|
||||
|
||||
async function insertCandidate(database, overrides = {}) {
|
||||
const queryInterface = database.getQueryInterface();
|
||||
const runId = overrides.runId ?? nextId();
|
||||
const attemptId = overrides.attemptId ?? nextId();
|
||||
const requestedAtMs = overrides.requestedAtMs ?? 1_750_000_000_100;
|
||||
await queryInterface.bulkInsert(RUN_TABLE, [
|
||||
{
|
||||
id: runId,
|
||||
project_id: 'default',
|
||||
task_id: `task:${runId}`,
|
||||
task_revision: 'revision-1',
|
||||
trigger_type: 'manual',
|
||||
execution_origin: 'manual',
|
||||
execution_owner: overrides.executionOwner ?? 'runtime',
|
||||
status: overrides.runStatus ?? 'running',
|
||||
version: 2,
|
||||
event_sequence: 0,
|
||||
priority: 0,
|
||||
created_at_ms: 1_750_000_000_000,
|
||||
started_at_ms: 1_750_000_000_010,
|
||||
cancel_requested_at_ms:
|
||||
overrides.cancelRequestedAtMs === undefined
|
||||
? requestedAtMs
|
||||
: overrides.cancelRequestedAtMs,
|
||||
cancel_reason: 'user',
|
||||
},
|
||||
]);
|
||||
await queryInterface.bulkInsert(RUN_ATTEMPT_TABLE, [
|
||||
{
|
||||
id: attemptId,
|
||||
run_id: runId,
|
||||
attempt: 1,
|
||||
status: overrides.attemptStatus ?? 'running',
|
||||
executor_type: 'local_process',
|
||||
callback_sequence: 0,
|
||||
created_at_ms: 1_750_000_000_010,
|
||||
},
|
||||
]);
|
||||
return { runId, attemptId, requestedAtMs };
|
||||
}
|
||||
|
||||
function claim(candidate, overrides = {}) {
|
||||
return {
|
||||
...candidate,
|
||||
owner: 'worker-a',
|
||||
leaseToken: 'lease-a',
|
||||
nowMs: candidate.requestedAtMs,
|
||||
leaseDurationMs: 50,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(databases.splice(0).map((database) => database.close()));
|
||||
});
|
||||
|
||||
test('fences two workers and lets a second worker recover an expired lease', async () => {
|
||||
const { database, repository } = await createRepository();
|
||||
const candidate = await insertCandidate(database);
|
||||
|
||||
const first = await repository.claim(claim(candidate));
|
||||
assert.equal(first.status, 'claimed');
|
||||
assert.equal(first.dispatch.version, 1);
|
||||
assert.equal(first.dispatch.dispatchCount, 1);
|
||||
assert.equal(first.dispatch.leaseOwner, 'worker-a');
|
||||
|
||||
const competing = await repository.claim(
|
||||
claim(candidate, {
|
||||
owner: 'worker-b',
|
||||
leaseToken: 'lease-b',
|
||||
nowMs: candidate.requestedAtMs + 25,
|
||||
}),
|
||||
);
|
||||
assert.equal(competing.status, 'leased');
|
||||
assert.equal(competing.dispatch.leaseOwner, 'worker-a');
|
||||
|
||||
const recovered = await repository.claim(
|
||||
claim(candidate, {
|
||||
owner: 'worker-b',
|
||||
leaseToken: 'lease-b',
|
||||
nowMs: candidate.requestedAtMs + 50,
|
||||
}),
|
||||
);
|
||||
assert.equal(recovered.status, 'claimed');
|
||||
assert.equal(recovered.dispatch.version, 2);
|
||||
assert.equal(recovered.dispatch.dispatchCount, 2);
|
||||
assert.equal(recovered.dispatch.leaseOwner, 'worker-b');
|
||||
|
||||
await assert.rejects(
|
||||
repository.recordResult({
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
owner: 'worker-a',
|
||||
leaseToken: 'lease-a',
|
||||
expectedVersion: 1,
|
||||
result: 'termination_requested',
|
||||
atMs: candidate.requestedAtMs + 51,
|
||||
eventId: nextId(),
|
||||
}),
|
||||
CancellationDispatchFenceRejectedError,
|
||||
);
|
||||
|
||||
const recorded = await repository.recordResult({
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
owner: 'worker-b',
|
||||
leaseToken: 'lease-b',
|
||||
expectedVersion: recovered.dispatch.version,
|
||||
result: 'termination_requested',
|
||||
atMs: candidate.requestedAtMs + 52,
|
||||
eventId: nextId(),
|
||||
});
|
||||
assert.equal(recorded.dispatch.status, 'dispatched');
|
||||
assert.equal(recorded.dispatch.version, 3);
|
||||
assert.equal(recorded.event.type, 'run.cancel_dispatched');
|
||||
assert.deepEqual(recorded.event.payload, {
|
||||
attempt_id: candidate.attemptId,
|
||||
dispatch_count: 2,
|
||||
result: 'termination_requested',
|
||||
});
|
||||
|
||||
const terminal = await repository.claim(
|
||||
claim(candidate, {
|
||||
owner: 'worker-c',
|
||||
leaseToken: 'lease-c',
|
||||
nowMs: candidate.requestedAtMs + 100,
|
||||
}),
|
||||
);
|
||||
assert.equal(terminal.status, 'dispatched');
|
||||
});
|
||||
|
||||
test('persists retry backoff and only reclaims when it becomes due', async () => {
|
||||
const { database, repository } = await createRepository();
|
||||
const candidate = await insertCandidate(database);
|
||||
const leased = await repository.claim(claim(candidate));
|
||||
const retryAtMs = candidate.requestedAtMs + 1_000;
|
||||
|
||||
const failed = await repository.recordResult({
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
owner: 'worker-a',
|
||||
leaseToken: 'lease-a',
|
||||
expectedVersion: leased.dispatch.version,
|
||||
result: 'dispatch_error',
|
||||
atMs: candidate.requestedAtMs + 1,
|
||||
nextAttemptAtMs: retryAtMs,
|
||||
eventId: nextId(),
|
||||
});
|
||||
assert.equal(failed.dispatch.status, 'retry_wait');
|
||||
assert.equal(failed.dispatch.nextAttemptAtMs, retryAtMs);
|
||||
assert.equal(failed.event.type, 'run.cancel_dispatch_failed');
|
||||
|
||||
const early = await repository.claim(
|
||||
claim(candidate, {
|
||||
owner: 'worker-b',
|
||||
leaseToken: 'lease-b',
|
||||
nowMs: retryAtMs - 1,
|
||||
}),
|
||||
);
|
||||
assert.equal(early.status, 'not_due');
|
||||
|
||||
const retry = await repository.claim(
|
||||
claim(candidate, {
|
||||
owner: 'worker-b',
|
||||
leaseToken: 'lease-b',
|
||||
nowMs: retryAtMs,
|
||||
}),
|
||||
);
|
||||
assert.equal(retry.status, 'claimed');
|
||||
assert.equal(retry.dispatch.dispatchCount, 2);
|
||||
|
||||
const missingController = await repository.recordResult({
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
owner: 'worker-b',
|
||||
leaseToken: 'lease-b',
|
||||
expectedVersion: retry.dispatch.version,
|
||||
result: 'controller_missing',
|
||||
atMs: retryAtMs + 1,
|
||||
nextAttemptAtMs: retryAtMs + 2_000,
|
||||
eventId: nextId(),
|
||||
});
|
||||
assert.equal(missingController.dispatch.status, 'retry_wait');
|
||||
assert.equal(
|
||||
missingController.dispatch.lastDispatchedAtMs,
|
||||
failed.dispatch.lastDispatchedAtMs,
|
||||
);
|
||||
});
|
||||
|
||||
test('fails closed for stale candidates and conflicting Attempt bindings', async () => {
|
||||
const { database, repository } = await createRepository();
|
||||
const stale = await insertCandidate(database, { runStatus: 'succeeded' });
|
||||
assert.deepEqual(await repository.claim(claim(stale)), {
|
||||
status: 'not_eligible',
|
||||
});
|
||||
assert.equal(await repository.findByRunId(stale.runId), null);
|
||||
|
||||
const candidate = await insertCandidate(database);
|
||||
await repository.claim(claim(candidate));
|
||||
const secondAttemptId = nextId();
|
||||
await database.getQueryInterface().bulkInsert(RUN_ATTEMPT_TABLE, [
|
||||
{
|
||||
id: secondAttemptId,
|
||||
run_id: candidate.runId,
|
||||
attempt: 2,
|
||||
status: 'running',
|
||||
executor_type: 'local_process',
|
||||
callback_sequence: 0,
|
||||
created_at_ms: candidate.requestedAtMs + 1,
|
||||
},
|
||||
]);
|
||||
await assert.rejects(
|
||||
repository.claim(
|
||||
claim(
|
||||
{ ...candidate, attemptId: secondAttemptId },
|
||||
{
|
||||
owner: 'worker-b',
|
||||
leaseToken: 'lease-b',
|
||||
nowMs: candidate.requestedAtMs + 50,
|
||||
},
|
||||
),
|
||||
),
|
||||
CancellationDispatchBindingConflictError,
|
||||
);
|
||||
});
|
||||
|
||||
test('rolls back dispatch state and Run version when event append fails', async () => {
|
||||
const { database, repository } = await createRepository();
|
||||
const candidate = await insertCandidate(database);
|
||||
const leased = await repository.claim(claim(candidate));
|
||||
const duplicateEventId = nextId();
|
||||
await database.getQueryInterface().bulkInsert(RUN_EVENT_TABLE, [
|
||||
{
|
||||
id: duplicateEventId,
|
||||
run_id: candidate.runId,
|
||||
sequence: 99,
|
||||
type: 'fixture.event',
|
||||
dedupe_key: 'fixture-event',
|
||||
actor_type: 'system',
|
||||
payload: '{}',
|
||||
created_at_ms: candidate.requestedAtMs,
|
||||
},
|
||||
]);
|
||||
|
||||
await assert.rejects(
|
||||
repository.recordResult({
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
owner: 'worker-a',
|
||||
leaseToken: 'lease-a',
|
||||
expectedVersion: leased.dispatch.version,
|
||||
result: 'already_exited',
|
||||
atMs: candidate.requestedAtMs + 1,
|
||||
eventId: duplicateEventId,
|
||||
}),
|
||||
);
|
||||
const afterFailure = await repository.findByRunId(candidate.runId);
|
||||
assert.equal(afterFailure.status, 'leased');
|
||||
assert.equal(afterFailure.version, leased.dispatch.version);
|
||||
|
||||
const [run] = await database.query(
|
||||
`SELECT version, event_sequence FROM ${RUN_TABLE} WHERE id = :runId`,
|
||||
{
|
||||
replacements: { runId: candidate.runId },
|
||||
type: QueryTypes.SELECT,
|
||||
},
|
||||
);
|
||||
assert.deepEqual(run, { version: 2, event_sequence: 0 });
|
||||
|
||||
const recovered = await repository.recordResult({
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
owner: 'worker-a',
|
||||
leaseToken: 'lease-a',
|
||||
expectedVersion: leased.dispatch.version,
|
||||
result: 'already_exited',
|
||||
atMs: candidate.requestedAtMs + 2,
|
||||
eventId: nextId(),
|
||||
});
|
||||
assert.equal(recovered.dispatch.status, 'dispatched');
|
||||
});
|
||||
|
||||
test('allows only one of two dispatchers to signal the same persisted Attempt', async () => {
|
||||
const { database, repository } = await createRepository();
|
||||
const candidate = await insertCandidate(database);
|
||||
const source = {
|
||||
async listCandidates() {
|
||||
return {
|
||||
candidates: [
|
||||
{
|
||||
runId: candidate.runId,
|
||||
requestedAtMs: candidate.requestedAtMs,
|
||||
reason: 'user',
|
||||
attempts: [
|
||||
{
|
||||
attemptId: candidate.attemptId,
|
||||
executorType: 'local_process',
|
||||
executorHandle: 'durable-handle',
|
||||
pid: 4100,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
truncated: false,
|
||||
unsafeAttemptOverflow: false,
|
||||
};
|
||||
},
|
||||
};
|
||||
let stopCalls = 0;
|
||||
const controller = {
|
||||
executorType: 'local_process',
|
||||
async stop() {
|
||||
stopCalls += 1;
|
||||
return {
|
||||
status: 'termination_requested',
|
||||
termSignalSent: true,
|
||||
killSignalSent: false,
|
||||
};
|
||||
},
|
||||
};
|
||||
function options(owner) {
|
||||
return {
|
||||
owner,
|
||||
clock: () => candidate.requestedAtMs + 1,
|
||||
createId: nextId,
|
||||
};
|
||||
}
|
||||
const workerA = new PrimaryCancellationDispatcher(
|
||||
source,
|
||||
repository,
|
||||
[controller],
|
||||
options('worker-a'),
|
||||
);
|
||||
const workerB = new PrimaryCancellationDispatcher(
|
||||
source,
|
||||
repository,
|
||||
[controller],
|
||||
options('worker-b'),
|
||||
);
|
||||
|
||||
const first = await workerA.dispatchBatch();
|
||||
const second = await workerB.dispatchBatch();
|
||||
assert.equal(first.terminationRequested, 1);
|
||||
assert.equal(second.alreadyResolved, 1);
|
||||
assert.equal(stopCalls, 1);
|
||||
});
|
||||
|
||||
test('fails closed instead of reclaiming a corrupt persisted lease', async () => {
|
||||
const { database, repository } = await createRepository();
|
||||
const candidate = await insertCandidate(database);
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.bulkInsert(RUN_CANCELLATION_DISPATCH_TABLE, [
|
||||
{
|
||||
run_id: candidate.runId,
|
||||
attempt_id: candidate.attemptId,
|
||||
status: 'leased',
|
||||
version: 1,
|
||||
dispatch_count: 1,
|
||||
lease_owner: 'worker-a',
|
||||
lease_token: null,
|
||||
lease_expires_at_ms: candidate.requestedAtMs - 1,
|
||||
created_at_ms: candidate.requestedAtMs - 100,
|
||||
updated_at_ms: candidate.requestedAtMs - 50,
|
||||
},
|
||||
]);
|
||||
|
||||
await assert.rejects(
|
||||
repository.claim(
|
||||
claim(candidate, {
|
||||
owner: 'worker-b',
|
||||
leaseToken: 'lease-b',
|
||||
nowMs: candidate.requestedAtMs,
|
||||
}),
|
||||
),
|
||||
CancellationDispatchRepositoryError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,204 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
activateClusterControlRuntime,
|
||||
} = require('../../back/runtime/application/clusterControlRuntimeActivation');
|
||||
|
||||
const EVIDENCE = Object.freeze({
|
||||
contractName: 'control-core',
|
||||
contractVersion: 1,
|
||||
serverMajor: 16,
|
||||
migrationIds: Object.freeze([
|
||||
'pg-0001-schema-capability',
|
||||
'pg-0002-run-core',
|
||||
]),
|
||||
});
|
||||
|
||||
function stack(events, recovery = { safe: true, remaining: 0, failed: 0 }) {
|
||||
return {
|
||||
async reconcile() {
|
||||
events.push('reconcile');
|
||||
return recovery;
|
||||
},
|
||||
async startLifecycles() {
|
||||
events.push('start-lifecycles');
|
||||
return true;
|
||||
},
|
||||
installAdmission() {
|
||||
events.push('install-admission');
|
||||
return () => events.push('dispose-admission');
|
||||
},
|
||||
async stop() {
|
||||
events.push('stop-stack');
|
||||
return 'stopped';
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function options(events, overrides = {}) {
|
||||
return {
|
||||
enabled: true,
|
||||
profile: 'cluster-control',
|
||||
readiness: {
|
||||
async assertReady() {
|
||||
events.push('readiness');
|
||||
return EVIDENCE;
|
||||
},
|
||||
},
|
||||
create() {
|
||||
events.push('create');
|
||||
return stack(events);
|
||||
},
|
||||
audit(record) {
|
||||
events.push(`audit:${record.state}`);
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('does nothing when cluster-control activation is not explicitly enabled', async () => {
|
||||
const events = [];
|
||||
const result = await activateClusterControlRuntime(
|
||||
options(events, { enabled: false }),
|
||||
);
|
||||
assert.equal(result.status, 'disabled');
|
||||
assert.deepEqual(events, ['audit:disabled']);
|
||||
assert.equal(await result.stop(), 'stopped');
|
||||
});
|
||||
|
||||
test('rejects the wrong deployment profile before probing the database', async () => {
|
||||
const events = [];
|
||||
await assert.rejects(
|
||||
activateClusterControlRuntime(options(events, { profile: 'standalone' })),
|
||||
/cannot activate cluster-control/,
|
||||
);
|
||||
assert.deepEqual(events, []);
|
||||
});
|
||||
|
||||
test('never constructs repositories when schema readiness fails', async () => {
|
||||
const events = [];
|
||||
const unavailable = new Error('database unavailable');
|
||||
await assert.rejects(
|
||||
activateClusterControlRuntime(
|
||||
options(events, {
|
||||
readiness: {
|
||||
async assertReady() {
|
||||
events.push('readiness');
|
||||
throw unavailable;
|
||||
},
|
||||
},
|
||||
}),
|
||||
),
|
||||
(error) => error === unavailable,
|
||||
);
|
||||
assert.deepEqual(events, ['readiness', 'audit:failed']);
|
||||
});
|
||||
|
||||
test('orders readiness, recovery, lifecycles and admission and stops idempotently', async () => {
|
||||
const events = [];
|
||||
const result = await activateClusterControlRuntime(options(events));
|
||||
assert.equal(result.status, 'active');
|
||||
assert.deepEqual(events, [
|
||||
'readiness',
|
||||
'audit:schema_ready',
|
||||
'create',
|
||||
'reconcile',
|
||||
'audit:reconciled',
|
||||
'start-lifecycles',
|
||||
'install-admission',
|
||||
'audit:active',
|
||||
]);
|
||||
const first = result.stop();
|
||||
const second = result.stop();
|
||||
assert.equal(first, second);
|
||||
assert.equal(await first, 'stopped');
|
||||
assert.deepEqual(events.slice(-3), [
|
||||
'dispose-admission',
|
||||
'stop-stack',
|
||||
'audit:stopped',
|
||||
]);
|
||||
});
|
||||
|
||||
test('cleans up a constructed stack when startup recovery is unsafe', async () => {
|
||||
const events = [];
|
||||
await assert.rejects(
|
||||
activateClusterControlRuntime(
|
||||
options(events, {
|
||||
create() {
|
||||
events.push('create');
|
||||
return stack(events, { safe: false, remaining: 1, failed: 0 });
|
||||
},
|
||||
}),
|
||||
),
|
||||
/did not converge safely/,
|
||||
);
|
||||
assert.equal(events.includes('start-lifecycles'), false);
|
||||
assert.equal(events.includes('install-admission'), false);
|
||||
assert.deepEqual(events.slice(-2), ['stop-stack', 'audit:failed']);
|
||||
});
|
||||
|
||||
test('continues stopping the stack when admission cleanup fails', async () => {
|
||||
const events = [];
|
||||
const cleanupFailure = new Error('admission cleanup failed');
|
||||
const result = await activateClusterControlRuntime(
|
||||
options(events, {
|
||||
create() {
|
||||
events.push('create');
|
||||
return {
|
||||
...stack(events),
|
||||
installAdmission() {
|
||||
events.push('install-admission');
|
||||
return () => {
|
||||
events.push('dispose-admission');
|
||||
throw cleanupFailure;
|
||||
};
|
||||
},
|
||||
};
|
||||
},
|
||||
}),
|
||||
);
|
||||
const first = result.stop();
|
||||
assert.equal(first, result.stop());
|
||||
await assert.rejects(first, (error) => error === cleanupFailure);
|
||||
assert.deepEqual(events.slice(-3), [
|
||||
'dispose-admission',
|
||||
'stop-stack',
|
||||
'audit:failed',
|
||||
]);
|
||||
});
|
||||
|
||||
test('preserves an activation failure when admission rollback also fails', async () => {
|
||||
const events = [];
|
||||
const activationFailure = new Error('active audit failed');
|
||||
await assert.rejects(
|
||||
activateClusterControlRuntime(
|
||||
options(events, {
|
||||
create() {
|
||||
events.push('create');
|
||||
return {
|
||||
...stack(events),
|
||||
installAdmission() {
|
||||
events.push('install-admission');
|
||||
return () => {
|
||||
events.push('dispose-admission');
|
||||
throw new Error('rollback failed');
|
||||
};
|
||||
},
|
||||
};
|
||||
},
|
||||
audit(record) {
|
||||
events.push(`audit:${record.state}`);
|
||||
if (record.state === 'active') throw activationFailure;
|
||||
},
|
||||
}),
|
||||
),
|
||||
(error) => error === activationFailure,
|
||||
);
|
||||
assert.deepEqual(events.slice(-3), [
|
||||
'dispose-admission',
|
||||
'stop-stack',
|
||||
'audit:failed',
|
||||
]);
|
||||
});
|
||||
@@ -0,0 +1,301 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
postgresqlMainMigrationStream,
|
||||
} = require('../../back/migrations/postgresql');
|
||||
const {
|
||||
postgresqlControlSchemaContract,
|
||||
} = require('../../back/migrations/postgresql/schemaContract');
|
||||
const {
|
||||
PostgresSchemaReadinessError,
|
||||
} = require('../../back/migrations/postgresql/schemaReadiness');
|
||||
const {
|
||||
PostgresRunRepository,
|
||||
} = require('../../back/runtime/adapters/postgresql/runRepository');
|
||||
const {
|
||||
bootstrapClusterControlRuntime,
|
||||
} = require('../../back/runtime/adapters/postgresql/clusterControlRuntimeBootstrap');
|
||||
|
||||
function migrationHistory() {
|
||||
return postgresqlMainMigrationStream.migrations.map((migration, index) => ({
|
||||
streamId: 'postgresql-main',
|
||||
dialect: 'postgresql',
|
||||
migrationId: migration.id,
|
||||
checksum: migration.checksum,
|
||||
appliedAtMs: index + 1,
|
||||
}));
|
||||
}
|
||||
|
||||
function runtimePrivileges() {
|
||||
const privileges = {
|
||||
schema_migrations: [true, false, false, false],
|
||||
schema_capabilities: [true, false, false, false],
|
||||
runs: [true, true, true, false],
|
||||
run_attempts: [true, true, true, false],
|
||||
run_events: [true, true, false, false],
|
||||
run_retry_policies: [true, true, true, false],
|
||||
};
|
||||
return Object.entries(privileges).map(
|
||||
([
|
||||
tableName,
|
||||
[selectAllowed, insertAllowed, updateAllowed, deleteAllowed],
|
||||
]) => ({
|
||||
tableName,
|
||||
selectAllowed,
|
||||
insertAllowed,
|
||||
updateAllowed,
|
||||
deleteAllowed,
|
||||
isOwner: false,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function databaseResource(events, overrides = {}) {
|
||||
const contract = postgresqlControlSchemaContract;
|
||||
const pool = {
|
||||
async query(text) {
|
||||
events.push(
|
||||
`query:${
|
||||
events.filter((event) => event.startsWith('query:')).length + 1
|
||||
}`,
|
||||
);
|
||||
if (text.includes("current_setting('server_version_num')")) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
serverVersionNum: overrides.serverVersionNum ?? '160014',
|
||||
currentUser: 'ql3_runtime',
|
||||
inRecovery: false,
|
||||
transactionReadOnly: 'off',
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."schema_migrations"')) {
|
||||
return { rows: migrationHistory() };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."schema_capabilities"')) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
contractName: contract.contractName,
|
||||
contractVersion: contract.contractVersion,
|
||||
migrationId: contract.migrationId,
|
||||
capabilities: contract.capabilities,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM information_schema.columns')) {
|
||||
return {
|
||||
rows: contract.tables.flatMap((table) =>
|
||||
table.columns.map((columnName) => ({
|
||||
tableName: table.name,
|
||||
columnName,
|
||||
})),
|
||||
),
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM pg_indexes')) {
|
||||
return {
|
||||
rows: contract.indexes.map((indexName) => ({ indexName })),
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM pg_constraint')) {
|
||||
return {
|
||||
rows: [
|
||||
...contract.checks.map((constraintName) => ({
|
||||
constraintName,
|
||||
constraintType: 'check',
|
||||
})),
|
||||
...contract.foreignKeys.map((constraintName) => ({
|
||||
constraintName,
|
||||
constraintType: 'foreign_key',
|
||||
})),
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM pg_catalog.pg_roles')) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
canLogin: true,
|
||||
superuser: false,
|
||||
createDatabase: false,
|
||||
createRole: false,
|
||||
replication: false,
|
||||
bypassRowLevelSecurity: false,
|
||||
databaseConnect: true,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('has_schema_privilege')) {
|
||||
return { rows: [{ schemaUsage: true, schemaCreate: false }] };
|
||||
}
|
||||
if (text.includes('has_table_privilege')) {
|
||||
return { rows: runtimePrivileges() };
|
||||
}
|
||||
throw new Error(`unexpected query: ${text}`);
|
||||
},
|
||||
async connect() {
|
||||
throw new Error('Repository connections are not used during bootstrap');
|
||||
},
|
||||
};
|
||||
return {
|
||||
pool,
|
||||
async close() {
|
||||
events.push('close-database');
|
||||
if (overrides.closeError) throw overrides.closeError;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function activationStack(
|
||||
events,
|
||||
recovery = { safe: true, remaining: 0, failed: 0 },
|
||||
) {
|
||||
return {
|
||||
async reconcile() {
|
||||
events.push('reconcile');
|
||||
return recovery;
|
||||
},
|
||||
async startLifecycles() {
|
||||
events.push('start-lifecycles');
|
||||
return true;
|
||||
},
|
||||
installAdmission() {
|
||||
events.push('install-admission');
|
||||
return () => events.push('dispose-admission');
|
||||
},
|
||||
async stop() {
|
||||
events.push('stop-stack');
|
||||
return 'stopped';
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function bootstrapOptions(events, overrides = {}) {
|
||||
return {
|
||||
enabled: true,
|
||||
profile: 'cluster-control',
|
||||
async openDatabase() {
|
||||
events.push('open-database');
|
||||
return databaseResource(events);
|
||||
},
|
||||
create({ evidence, runs }) {
|
||||
events.push('create-stack');
|
||||
assert.equal(evidence.contractVersion, 2);
|
||||
assert.equal(runs instanceof PostgresRunRepository, true);
|
||||
return activationStack(events);
|
||||
},
|
||||
audit(record) {
|
||||
events.push(`audit:${record.state}`);
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('disabled and wrong-profile bootstrap never opens PostgreSQL', async () => {
|
||||
const disabledEvents = [];
|
||||
const disabled = await bootstrapClusterControlRuntime(
|
||||
bootstrapOptions(disabledEvents, { enabled: false }),
|
||||
);
|
||||
assert.equal(disabled.status, 'disabled');
|
||||
assert.deepEqual(disabledEvents, ['audit:disabled']);
|
||||
|
||||
const wrongProfileEvents = [];
|
||||
await assert.rejects(
|
||||
bootstrapClusterControlRuntime(
|
||||
bootstrapOptions(wrongProfileEvents, { profile: 'edge' }),
|
||||
),
|
||||
/cannot activate cluster-control/,
|
||||
);
|
||||
assert.deepEqual(wrongProfileEvents, []);
|
||||
});
|
||||
|
||||
test('readiness failure closes the database before returning the root error', async () => {
|
||||
const events = [];
|
||||
await assert.rejects(
|
||||
bootstrapClusterControlRuntime(
|
||||
bootstrapOptions(events, {
|
||||
async openDatabase() {
|
||||
events.push('open-database');
|
||||
return databaseResource(events, { serverVersionNum: '150018' });
|
||||
},
|
||||
}),
|
||||
),
|
||||
(error) =>
|
||||
error instanceof PostgresSchemaReadinessError &&
|
||||
error.code === 'server_version_unsupported',
|
||||
);
|
||||
assert.equal(events.includes('create-stack'), false);
|
||||
assert.deepEqual(events.slice(-2), ['audit:failed', 'close-database']);
|
||||
});
|
||||
|
||||
test('opens once, assembles after readiness, and closes after stack shutdown', async () => {
|
||||
const events = [];
|
||||
const result = await bootstrapClusterControlRuntime(bootstrapOptions(events));
|
||||
assert.equal(result.status, 'active');
|
||||
assert.equal(events.filter((event) => event === 'open-database').length, 1);
|
||||
assert.ok(events.indexOf('create-stack') > events.lastIndexOf('query:7'));
|
||||
const first = result.stop();
|
||||
assert.equal(first, result.stop());
|
||||
assert.equal(await first, 'stopped');
|
||||
assert.deepEqual(events.slice(-4), [
|
||||
'dispose-admission',
|
||||
'stop-stack',
|
||||
'audit:stopped',
|
||||
'close-database',
|
||||
]);
|
||||
});
|
||||
|
||||
test('unsafe recovery stops the stack and closes the database', async () => {
|
||||
const events = [];
|
||||
await assert.rejects(
|
||||
bootstrapClusterControlRuntime(
|
||||
bootstrapOptions(events, {
|
||||
create({ runs }) {
|
||||
events.push('create-stack');
|
||||
assert.equal(runs instanceof PostgresRunRepository, true);
|
||||
return activationStack(events, {
|
||||
safe: false,
|
||||
remaining: 1,
|
||||
failed: 0,
|
||||
});
|
||||
},
|
||||
}),
|
||||
),
|
||||
/did not converge safely/,
|
||||
);
|
||||
assert.equal(events.includes('install-admission'), false);
|
||||
assert.deepEqual(events.slice(-3), [
|
||||
'stop-stack',
|
||||
'audit:failed',
|
||||
'close-database',
|
||||
]);
|
||||
});
|
||||
|
||||
test('database close failure does not skip stack shutdown and remains idempotent', async () => {
|
||||
const events = [];
|
||||
const closeError = new Error('database close failed');
|
||||
const result = await bootstrapClusterControlRuntime(
|
||||
bootstrapOptions(events, {
|
||||
async openDatabase() {
|
||||
events.push('open-database');
|
||||
return databaseResource(events, { closeError });
|
||||
},
|
||||
}),
|
||||
);
|
||||
const first = result.stop();
|
||||
assert.equal(first, result.stop());
|
||||
await assert.rejects(first, (error) => error === closeError);
|
||||
assert.deepEqual(events.slice(-4), [
|
||||
'dispose-admission',
|
||||
'stop-stack',
|
||||
'audit:stopped',
|
||||
'close-database',
|
||||
]);
|
||||
});
|
||||
@@ -0,0 +1,220 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const { Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const { runSchemaMigration } = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
runCancellationRequestMigration,
|
||||
} = require('../../back/migrations/0004-run-cancellation-request');
|
||||
const {
|
||||
runAttemptDeadlineMigration,
|
||||
} = require('../../back/migrations/0006-run-attempt-deadline');
|
||||
const {
|
||||
completionReceiptJournalMigration,
|
||||
} = require('../../back/migrations/0007-completion-receipt-journal');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeCompletionReceiptJournal,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/completionReceiptJournal');
|
||||
const {
|
||||
LegacySequelizeRunRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/runRepository');
|
||||
const {
|
||||
CompletionReceiptFileStore,
|
||||
} = require('../../back/runtime/adapters/fs/completionReceiptFileStore');
|
||||
const {
|
||||
PrimaryCompletionReceiptConsumer,
|
||||
} = require('../../back/runtime/application/primaryCompletionReceiptConsumer');
|
||||
const {
|
||||
PrimaryCompletionReceiptJournalScanner,
|
||||
} = require('../../back/runtime/application/primaryCompletionReceiptJournalScanner');
|
||||
const {
|
||||
PrimaryRunCompletionService,
|
||||
hashPrimaryCompletionToken,
|
||||
} = require('../../back/runtime/application/primaryRunCompletionService');
|
||||
|
||||
const NOW = 1_750_900_000_000;
|
||||
const TOKEN = 'j'.repeat(43);
|
||||
let sequence = 2_400;
|
||||
|
||||
function nextId() {
|
||||
sequence += 1;
|
||||
return `019f7900-0000-7000-8000-${String(sequence).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
async function setup(t) {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
runCancellationRequestMigration,
|
||||
runAttemptDeadlineMigration,
|
||||
completionReceiptJournalMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
return {
|
||||
repository: new LegacySequelizeRunRepository(database),
|
||||
journal: new LegacySequelizeCompletionReceiptJournal(database),
|
||||
};
|
||||
}
|
||||
|
||||
async function seed(repository, status = 'running', finishedAtMs) {
|
||||
const run = {
|
||||
id: nextId(),
|
||||
projectId: 'default',
|
||||
taskId: 'journal-test',
|
||||
taskRevision: 'revision-1',
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
status: status === 'running' ? 'running' : 'succeeded',
|
||||
version: 0,
|
||||
eventSequence: 0,
|
||||
priority: 0,
|
||||
createdAtMs: NOW,
|
||||
...(finishedAtMs === undefined ? {} : { finishedAtMs }),
|
||||
};
|
||||
const attempt = {
|
||||
id: nextId(),
|
||||
runId: run.id,
|
||||
attempt: 1,
|
||||
status,
|
||||
executorType: 'local_process',
|
||||
callbackSequence: status === 'running' ? 0 : 1,
|
||||
callbackTokenHash: hashPrimaryCompletionToken(TOKEN),
|
||||
createdAtMs: NOW,
|
||||
...(finishedAtMs === undefined ? {} : { finishedAtMs }),
|
||||
};
|
||||
await repository.transaction(async (transaction) => {
|
||||
await transaction.insertRun(run);
|
||||
await transaction.insertAttempt(attempt);
|
||||
});
|
||||
return { run, attempt };
|
||||
}
|
||||
|
||||
test('registers idempotently and lists active and terminal pending receipts', async (t) => {
|
||||
const { repository, journal } = await setup(t);
|
||||
const active = await seed(repository);
|
||||
const terminal = await seed(repository, 'succeeded', NOW + 20);
|
||||
for (const aggregate of [active, terminal]) {
|
||||
await journal.register({
|
||||
runId: aggregate.run.id,
|
||||
attemptId: aggregate.attempt.id,
|
||||
registeredAtMs: NOW + 1,
|
||||
});
|
||||
}
|
||||
await journal.register({
|
||||
runId: active.run.id,
|
||||
attemptId: active.attempt.id,
|
||||
registeredAtMs: NOW + 1,
|
||||
});
|
||||
|
||||
const page = await journal.listCandidates({
|
||||
observedAtMs: NOW + 100,
|
||||
limit: 8,
|
||||
});
|
||||
assert.equal(page.candidates.length, 2);
|
||||
assert.deepEqual(
|
||||
page.candidates.map((candidate) => candidate.attemptStatus).sort(),
|
||||
['running', 'succeeded'],
|
||||
);
|
||||
});
|
||||
|
||||
test('defers quarantined entries until purge time and resolves idempotently', async (t) => {
|
||||
const { repository, journal } = await setup(t);
|
||||
const aggregate = await seed(repository);
|
||||
await journal.register({
|
||||
runId: aggregate.run.id,
|
||||
attemptId: aggregate.attempt.id,
|
||||
registeredAtMs: NOW,
|
||||
});
|
||||
await journal.markQuarantined({
|
||||
attemptId: aggregate.attempt.id,
|
||||
quarantineRef: `.quarantine/${aggregate.attempt.id.slice(0, 2)}/${
|
||||
aggregate.attempt.id
|
||||
}.json`,
|
||||
updatedAtMs: NOW + 1,
|
||||
purgeAfterMs: NOW + 100,
|
||||
});
|
||||
|
||||
assert.equal(
|
||||
(await journal.listCandidates({ observedAtMs: NOW + 99 })).candidates
|
||||
.length,
|
||||
0,
|
||||
);
|
||||
const due = await journal.listCandidates({ observedAtMs: NOW + 100 });
|
||||
assert.equal(due.candidates[0].state, 'quarantined');
|
||||
assert.equal(await journal.resolve(aggregate.attempt.id), true);
|
||||
assert.equal(await journal.resolve(aggregate.attempt.id), false);
|
||||
});
|
||||
|
||||
test('finds and cleans a receipt after the Run already became terminal', async (t) => {
|
||||
const { repository, journal } = await setup(t);
|
||||
const aggregate = await seed(repository);
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-journal-replay-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
const store = new CompletionReceiptFileStore(root);
|
||||
await journal.register({
|
||||
runId: aggregate.run.id,
|
||||
attemptId: aggregate.attempt.id,
|
||||
registeredAtMs: aggregate.attempt.createdAtMs,
|
||||
});
|
||||
await store.publish({
|
||||
schemaVersion: 1,
|
||||
runId: aggregate.run.id,
|
||||
attemptId: aggregate.attempt.id,
|
||||
callbackSequence: 1,
|
||||
token: TOKEN,
|
||||
startedAtMs: NOW + 1,
|
||||
finishedAtMs: NOW + 20,
|
||||
exitCode: 0,
|
||||
});
|
||||
const completions = new PrimaryRunCompletionService(repository, nextId);
|
||||
await completions.complete({
|
||||
runId: aggregate.run.id,
|
||||
attemptId: aggregate.attempt.id,
|
||||
callbackSequence: 1,
|
||||
source: { kind: 'executor', executorType: 'local_process' },
|
||||
result: {
|
||||
outcome: 'succeeded',
|
||||
startedAtMs: NOW + 1,
|
||||
finishedAtMs: NOW + 20,
|
||||
exitCode: 0,
|
||||
},
|
||||
});
|
||||
|
||||
const consumer = new PrimaryCompletionReceiptConsumer(store, completions, {
|
||||
journal,
|
||||
clock: { now: () => NOW + 30 },
|
||||
});
|
||||
const scanner = new PrimaryCompletionReceiptJournalScanner(
|
||||
journal,
|
||||
store,
|
||||
consumer,
|
||||
{ clock: { now: () => NOW + 30 } },
|
||||
);
|
||||
const summary = await scanner.scanBatch();
|
||||
assert.equal(summary.alreadyTerminal, 1);
|
||||
assert.equal(summary.failed, 0);
|
||||
assert.equal(await store.read(aggregate.attempt.id), undefined);
|
||||
assert.equal(
|
||||
(await journal.listCandidates({ observedAtMs: NOW + 30 })).candidates
|
||||
.length,
|
||||
0,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,281 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { spawnSync } = require('node:child_process');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const {
|
||||
CompletionReceiptOrphanFileDirectory,
|
||||
} = require('../../back/runtime/adapters/fs/completionReceiptOrphanDirectory');
|
||||
const {
|
||||
CompletionReceiptOrphanAuditor,
|
||||
} = require('../../back/runtime/application/completionReceiptOrphanAuditor');
|
||||
const { parseArguments } = require('../../scripts/ql3-receipt-audit.cjs');
|
||||
|
||||
const REPOSITORY_ROOT = path.resolve(__dirname, '../..');
|
||||
const CLI_PATH = path.join(REPOSITORY_ROOT, 'scripts', 'ql3-receipt-audit.cjs');
|
||||
const OBSERVED_AT_MS = 1_800_000_000_000;
|
||||
const OLD_MTIME_MS = OBSERVED_AT_MS - 10 * 60_000;
|
||||
const YOUNG_MTIME_MS = OBSERVED_AT_MS - 1_000;
|
||||
const OWNED_ID = '019f7400-0000-7000-8000-000000000001';
|
||||
const ACTIVE_ID = '019f7400-0000-7000-8000-000000000002';
|
||||
const TERMINAL_ID = '019f7400-0000-7000-8000-000000000003';
|
||||
const UNKNOWN_ID = '019f7400-0000-7000-8000-000000000004';
|
||||
const YOUNG_ID = '019f7400-0000-7000-8000-000000000005';
|
||||
const JOURNAL_ONLY_ID = '019f7400-0000-7000-8000-000000000006';
|
||||
const roots = [];
|
||||
|
||||
async function temporaryRoot() {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-orphan-audit-'));
|
||||
roots.push(root);
|
||||
return root;
|
||||
}
|
||||
|
||||
async function writeAt(root, shard, name, modifiedAtMs = OLD_MTIME_MS) {
|
||||
const directory = path.join(root, shard);
|
||||
const target = path.join(directory, name);
|
||||
await fs.mkdir(directory, { recursive: true });
|
||||
await fs.writeFile(target, '{}');
|
||||
const timestamp = new Date(modifiedAtMs);
|
||||
await fs.utimes(target, timestamp, timestamp);
|
||||
return target;
|
||||
}
|
||||
|
||||
function ownershipSource(values) {
|
||||
const ownership = new Map(values.map((value) => [value.attemptId, value]));
|
||||
return {
|
||||
async lookup(attemptIds) {
|
||||
return new Map(
|
||||
attemptIds
|
||||
.filter((attemptId) => ownership.has(attemptId))
|
||||
.map((attemptId) => [attemptId, ownership.get(attemptId)]),
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
roots
|
||||
.splice(0)
|
||||
.map((root) => fs.rm(root, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
test('audits Journal ownership and quarantines only old regular orphans', async () => {
|
||||
const root = await temporaryRoot();
|
||||
const shard = '01';
|
||||
const names = {
|
||||
owned: `${OWNED_ID}.json`,
|
||||
active: `${ACTIVE_ID}.json`,
|
||||
terminal: `${TERMINAL_ID}.json`,
|
||||
unknown: `${UNKNOWN_ID}.json`,
|
||||
young: `${YOUNG_ID}.json`,
|
||||
temporary: `.${UNKNOWN_ID}.${'a'.repeat(32)}.tmp`,
|
||||
malformed: 'unexpected.bin',
|
||||
unsafe: 'unsafe-link',
|
||||
};
|
||||
for (const [key, name] of Object.entries(names)) {
|
||||
if (key !== 'unsafe') {
|
||||
await writeAt(
|
||||
root,
|
||||
shard,
|
||||
name,
|
||||
key === 'young' ? YOUNG_MTIME_MS : OLD_MTIME_MS,
|
||||
);
|
||||
}
|
||||
}
|
||||
await fs.symlink(
|
||||
path.join(root, shard, names.malformed),
|
||||
path.join(root, shard, names.unsafe),
|
||||
);
|
||||
const auditor = new CompletionReceiptOrphanAuditor(
|
||||
new CompletionReceiptOrphanFileDirectory(root),
|
||||
ownershipSource([
|
||||
{
|
||||
attemptId: OWNED_ID,
|
||||
attemptStatus: 'running',
|
||||
journalState: 'pending',
|
||||
},
|
||||
{ attemptId: ACTIVE_ID, attemptStatus: 'running' },
|
||||
{ attemptId: TERMINAL_ID, attemptStatus: 'succeeded' },
|
||||
]),
|
||||
);
|
||||
|
||||
const audit = await auditor.run({
|
||||
observedAtMs: OBSERVED_AT_MS,
|
||||
startShard: 1,
|
||||
shardCount: 1,
|
||||
maxEntriesPerShard: 16,
|
||||
minimumAgeMs: 5 * 60_000,
|
||||
});
|
||||
assert.equal(audit.scannedEntries, 8);
|
||||
assert.deepEqual(audit.overflowShards, []);
|
||||
assert.equal(audit.counts.journaled, 1);
|
||||
assert.equal(audit.counts.active_attempt, 1);
|
||||
assert.equal(audit.counts.terminal_orphan, 1);
|
||||
assert.equal(audit.counts.unknown_receipt, 1);
|
||||
assert.equal(audit.counts.young_unknown_receipt, 1);
|
||||
assert.equal(audit.counts.stale_temporary, 1);
|
||||
assert.equal(audit.counts.unknown_entry, 1);
|
||||
assert.equal(audit.counts.unsafe_entry, 1);
|
||||
assert.equal(
|
||||
audit.entries.filter((entry) => entry.action === 'eligible').length,
|
||||
4,
|
||||
);
|
||||
|
||||
const quarantine = await auditor.run({
|
||||
mode: 'quarantine',
|
||||
observedAtMs: OBSERVED_AT_MS,
|
||||
startShard: 1,
|
||||
shardCount: 1,
|
||||
maxEntriesPerShard: 16,
|
||||
minimumAgeMs: 5 * 60_000,
|
||||
});
|
||||
assert.equal(
|
||||
quarantine.entries.filter((entry) => entry.action === 'quarantined').length,
|
||||
4,
|
||||
);
|
||||
for (const name of [
|
||||
names.terminal,
|
||||
names.unknown,
|
||||
names.temporary,
|
||||
names.malformed,
|
||||
]) {
|
||||
await assert.rejects(fs.lstat(path.join(root, shard, name)), /ENOENT/);
|
||||
}
|
||||
for (const name of [names.owned, names.active, names.young, names.unsafe]) {
|
||||
assert.ok(await fs.lstat(path.join(root, shard, name)));
|
||||
}
|
||||
const quarantineFiles = await fs.readdir(
|
||||
path.join(root, '.orphan-quarantine', shard),
|
||||
);
|
||||
assert.equal(quarantineFiles.length, 4);
|
||||
});
|
||||
|
||||
test('fails closed when a shard exceeds its fixed entry capacity', async () => {
|
||||
const root = await temporaryRoot();
|
||||
for (const name of ['a.bin', 'b.bin', 'c.bin']) {
|
||||
await writeAt(root, '02', name);
|
||||
}
|
||||
const auditor = new CompletionReceiptOrphanAuditor(
|
||||
new CompletionReceiptOrphanFileDirectory(root),
|
||||
ownershipSource([]),
|
||||
);
|
||||
const report = await auditor.run({
|
||||
mode: 'quarantine',
|
||||
observedAtMs: OBSERVED_AT_MS,
|
||||
startShard: 2,
|
||||
shardCount: 1,
|
||||
maxEntriesPerShard: 2,
|
||||
minimumAgeMs: 0,
|
||||
});
|
||||
|
||||
assert.deepEqual(report.overflowShards, ['02']);
|
||||
assert.equal(report.scannedEntries, 2);
|
||||
assert.ok(
|
||||
report.entries.every((entry) => entry.action === 'blocked_overflow'),
|
||||
);
|
||||
assert.equal((await fs.readdir(path.join(root, '02'))).length, 3);
|
||||
await assert.rejects(
|
||||
auditor.run({ shardCount: 33 }),
|
||||
/shardCount must be between 1 and 32/,
|
||||
);
|
||||
await assert.rejects(
|
||||
auditor.run({ maxEntriesPerShard: 65 }),
|
||||
/maxEntriesPerShard must be between 1 and 64/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects shard and quarantine directory symlink escapes', async () => {
|
||||
const root = await temporaryRoot();
|
||||
const outside = await temporaryRoot();
|
||||
await fs.symlink(outside, path.join(root, '03'));
|
||||
const directory = new CompletionReceiptOrphanFileDirectory(root);
|
||||
await assert.rejects(directory.inspectShard('03', 4), /not a safe directory/);
|
||||
|
||||
await writeAt(root, '04', 'orphan.bin');
|
||||
const snapshot = await directory.inspectShard('04', 4);
|
||||
await fs.symlink(outside, path.join(root, '.orphan-quarantine'));
|
||||
await assert.rejects(
|
||||
directory.quarantine(snapshot.entries[0]),
|
||||
/not a safe directory/,
|
||||
);
|
||||
assert.ok(await fs.lstat(path.join(root, '04', 'orphan.bin')));
|
||||
assert.throws(
|
||||
() => new CompletionReceiptOrphanFileDirectory(path.parse(root).root),
|
||||
/must not be a filesystem root/,
|
||||
);
|
||||
});
|
||||
|
||||
test('parses a bounded shard cursor and keeps audit as the default mode', () => {
|
||||
const options = parseArguments([
|
||||
'--start-shard=fe',
|
||||
'--shards=2',
|
||||
'--entries-per-shard=4',
|
||||
'--minimum-age-ms=1000',
|
||||
]);
|
||||
assert.equal(options.mode, 'audit');
|
||||
assert.equal(options.startShard, 254);
|
||||
assert.equal(options.shardCount, 2);
|
||||
assert.equal(options.maxEntriesPerShard, 4);
|
||||
assert.throws(() => parseArguments(['--shards=33']), /between 1 and 32/);
|
||||
assert.throws(() => parseArguments(['--start-shard=FF']), /lowercase hex/);
|
||||
});
|
||||
|
||||
test(
|
||||
'Node 24 CLI reads ownership without opening the database for writes',
|
||||
{ skip: Number(process.versions.node.split('.')[0]) < 24 },
|
||||
async () => {
|
||||
const root = await temporaryRoot();
|
||||
const databasePath = path.join(root, 'database.sqlite');
|
||||
const receiptRoot = path.join(root, 'receipts');
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
const database = new DatabaseSync(databasePath);
|
||||
database.exec(`
|
||||
CREATE TABLE RunAttempts (
|
||||
id TEXT PRIMARY KEY,
|
||||
status TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE CompletionReceiptJournals (
|
||||
attempt_id TEXT PRIMARY KEY,
|
||||
state TEXT NOT NULL
|
||||
);
|
||||
INSERT INTO RunAttempts (id, status)
|
||||
VALUES ('${TERMINAL_ID}', 'succeeded');
|
||||
INSERT INTO CompletionReceiptJournals (attempt_id, state)
|
||||
VALUES ('${JOURNAL_ONLY_ID}', 'pending');
|
||||
`);
|
||||
database.close();
|
||||
await writeAt(receiptRoot, '01', `${TERMINAL_ID}.json`);
|
||||
await writeAt(receiptRoot, '01', `${JOURNAL_ONLY_ID}.json`);
|
||||
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
CLI_PATH,
|
||||
'--json',
|
||||
`--database=${databasePath}`,
|
||||
`--root=${receiptRoot}`,
|
||||
'--start-shard=01',
|
||||
'--shards=1',
|
||||
'--entries-per-shard=4',
|
||||
'--minimum-age-ms=0',
|
||||
],
|
||||
{ cwd: REPOSITORY_ROOT, encoding: 'utf8' },
|
||||
);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const report = JSON.parse(result.stdout);
|
||||
assert.equal(report.counts.terminal_orphan, 1);
|
||||
assert.equal(report.counts.journaled, 1);
|
||||
assert.equal(
|
||||
report.entries.find((entry) => entry.attemptId === TERMINAL_ID).action,
|
||||
'eligible',
|
||||
);
|
||||
assert.ok(
|
||||
await fs.lstat(path.join(receiptRoot, '01', `${TERMINAL_ID}.json`)),
|
||||
);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,158 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const {
|
||||
InvalidCompletionReceiptError,
|
||||
MAX_COMPLETION_RECEIPT_BYTES,
|
||||
parseCompletionReceipt,
|
||||
serializeCompletionReceipt,
|
||||
} = require('../../back/runtime/domain/completionReceipt');
|
||||
const {
|
||||
CompletionReceiptAlreadyExistsError,
|
||||
CompletionReceiptFileStore,
|
||||
} = require('../../back/runtime/adapters/fs/completionReceiptFileStore');
|
||||
|
||||
const roots = [];
|
||||
const RUN_ID = '019f7200-0000-7000-8000-000000000001';
|
||||
const ATTEMPT_ID = '019f7200-0000-7000-8000-000000000002';
|
||||
|
||||
function receipt(overrides = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
runId: RUN_ID,
|
||||
attemptId: ATTEMPT_ID,
|
||||
callbackSequence: 1,
|
||||
token: 'a'.repeat(43),
|
||||
startedAtMs: 1_750_200_000_000,
|
||||
finishedAtMs: 1_750_200_000_100,
|
||||
exitCode: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function createStore() {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-receipts-'));
|
||||
roots.push(root);
|
||||
return { root, store: new CompletionReceiptFileStore(root) };
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
roots
|
||||
.splice(0)
|
||||
.map((root) => fs.rm(root, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
test('completion receipt codec round-trips one canonical bounded payload', () => {
|
||||
const serialized = serializeCompletionReceipt(receipt());
|
||||
assert.ok(Buffer.byteLength(serialized) < MAX_COMPLETION_RECEIPT_BYTES);
|
||||
assert.equal(
|
||||
serializeCompletionReceipt(parseCompletionReceipt(serialized)),
|
||||
serialized,
|
||||
);
|
||||
assert.deepEqual(parseCompletionReceipt(serialized), receipt());
|
||||
});
|
||||
|
||||
test('completion receipt codec rejects ambiguous and extensible payloads', () => {
|
||||
const value = receipt();
|
||||
const unknown = JSON.stringify({ ...value, command: 'secret' });
|
||||
assert.throws(
|
||||
() => parseCompletionReceipt(unknown),
|
||||
InvalidCompletionReceiptError,
|
||||
);
|
||||
|
||||
const duplicate = serializeCompletionReceipt(value).replace(
|
||||
'"attemptId"',
|
||||
`"runId":"${RUN_ID}","attemptId"`,
|
||||
);
|
||||
assert.throws(() => parseCompletionReceipt(duplicate), /duplicate key/);
|
||||
assert.throws(
|
||||
() => serializeCompletionReceipt(receipt({ finishedAtMs: 1 })),
|
||||
/finishedAtMs/,
|
||||
);
|
||||
assert.throws(
|
||||
() => serializeCompletionReceipt(receipt({ exitCode: 256 })),
|
||||
/exitCode/,
|
||||
);
|
||||
assert.throws(
|
||||
() => parseCompletionReceipt('x'.repeat(MAX_COMPLETION_RECEIPT_BYTES + 1)),
|
||||
/size/,
|
||||
);
|
||||
});
|
||||
|
||||
test('file store publishes without overwrite and removes only a known receipt', async () => {
|
||||
const { root, store } = await createStore();
|
||||
await store.publish(receipt());
|
||||
assert.deepEqual(await store.read(ATTEMPT_ID), receipt());
|
||||
const target = path.join(root, ATTEMPT_ID.slice(0, 2), `${ATTEMPT_ID}.json`);
|
||||
assert.equal((await fs.stat(target)).mode & 0o777, 0o600);
|
||||
|
||||
await assert.rejects(
|
||||
store.publish(receipt({ exitCode: 1 })),
|
||||
CompletionReceiptAlreadyExistsError,
|
||||
);
|
||||
assert.equal((await store.read(ATTEMPT_ID)).exitCode, 0);
|
||||
assert.equal(await store.remove(ATTEMPT_ID), true);
|
||||
assert.equal(await store.remove(ATTEMPT_ID), false);
|
||||
assert.equal(await store.read(ATTEMPT_ID), undefined);
|
||||
});
|
||||
|
||||
test('does not expose a final receipt when storage reports ENOSPC', async () => {
|
||||
const { store } = await createStore();
|
||||
const originalOpen = fs.open;
|
||||
fs.open = async (target, ...args) => {
|
||||
if (String(target).endsWith('.tmp')) {
|
||||
throw Object.assign(new Error('simulated disk full'), { code: 'ENOSPC' });
|
||||
}
|
||||
return originalOpen(target, ...args);
|
||||
};
|
||||
try {
|
||||
await assert.rejects(store.publish(receipt()), { code: 'ENOSPC' });
|
||||
assert.equal(await store.read(ATTEMPT_ID), undefined);
|
||||
} finally {
|
||||
fs.open = originalOpen;
|
||||
}
|
||||
});
|
||||
|
||||
test('file store moves a known receipt to a private quarantine path', async () => {
|
||||
const { root, store } = await createStore();
|
||||
await store.publish(receipt());
|
||||
|
||||
const quarantineRef = await store.quarantine(ATTEMPT_ID);
|
||||
assert.match(quarantineRef, /^\.quarantine\/[0-9a-f]{2}\/019f7200-.+\.json$/);
|
||||
assert.equal(await store.read(ATTEMPT_ID), undefined);
|
||||
const quarantined = path.join(root, ...quarantineRef.split('/'));
|
||||
assert.equal((await fs.stat(quarantined)).mode & 0o777, 0o600);
|
||||
assert.equal(await store.quarantine(ATTEMPT_ID), quarantineRef);
|
||||
assert.equal(await store.purgeQuarantine(ATTEMPT_ID), true);
|
||||
assert.equal(await store.purgeQuarantine(ATTEMPT_ID), false);
|
||||
});
|
||||
|
||||
test('file store rejects traversal, oversized files, symlinks, and path mismatch', async () => {
|
||||
const { root, store } = await createStore();
|
||||
await assert.rejects(store.read('../../etc/passwd'), /UUIDv7/);
|
||||
|
||||
const directory = path.join(root, ATTEMPT_ID.slice(0, 2));
|
||||
const target = path.join(directory, `${ATTEMPT_ID}.json`);
|
||||
await fs.mkdir(directory, { recursive: true });
|
||||
await fs.writeFile(target, 'x'.repeat(MAX_COMPLETION_RECEIPT_BYTES + 1));
|
||||
await assert.rejects(store.read(ATTEMPT_ID), /byte limit/);
|
||||
|
||||
await fs.unlink(target);
|
||||
const other = receipt({
|
||||
attemptId: '019f7200-0000-7000-8000-000000000003',
|
||||
});
|
||||
await fs.writeFile(target, serializeCompletionReceipt(other));
|
||||
await assert.rejects(store.read(ATTEMPT_ID), /do not match/);
|
||||
|
||||
await fs.unlink(target);
|
||||
const source = path.join(directory, 'source.json');
|
||||
await fs.writeFile(source, serializeCompletionReceipt(receipt()));
|
||||
await fs.symlink(source, target);
|
||||
await assert.rejects(store.read(ATTEMPT_ID), InvalidCompletionReceiptError);
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const { DataTypes, Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const { migrations } = require('../../back/migrations');
|
||||
const {
|
||||
createDefaultManualPrimaryActivationStack,
|
||||
} = require('../../back/runtime/adapters/legacy/defaultManualPrimaryActivation');
|
||||
const {
|
||||
RuntimeRolloutPolicy,
|
||||
} = require('../../back/runtime/domain/runtimeRollout');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
|
||||
async function createDatabase(t) {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
const queryInterface = database.getQueryInterface();
|
||||
for (const table of ['CrontabViews', 'Subscriptions', 'Crontabs', 'Envs']) {
|
||||
await queryInterface.createTable(table, {
|
||||
id: { type: DataTypes.INTEGER, primaryKey: true },
|
||||
});
|
||||
}
|
||||
await queryInterface.createTable('RunningInstances', {
|
||||
id: { type: DataTypes.INTEGER, primaryKey: true },
|
||||
started_at: { type: DataTypes.INTEGER, allowNull: false },
|
||||
});
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations,
|
||||
logger: { info() {} },
|
||||
});
|
||||
return database;
|
||||
}
|
||||
|
||||
test('real Primary activation stack reconciles empty state and stops cleanly', async (t) => {
|
||||
const database = await createDatabase(t);
|
||||
const rollout = new RuntimeRolloutPolicy({
|
||||
defaultMode: 'off',
|
||||
origins: { manual: 'primary' },
|
||||
allowLegacyFallbackBeforeStart: false,
|
||||
});
|
||||
const stack = createDefaultManualPrimaryActivationStack(rollout, {
|
||||
database,
|
||||
owner: 'test-http-worker',
|
||||
recovery: { pageSize: 8, maxPages: 2 },
|
||||
cancellation: {
|
||||
intervalMs: 1_000,
|
||||
initialDelayMs: 60_000,
|
||||
stopTimeoutMs: 1_000,
|
||||
cycle: { pageSize: 8, maxPages: 2 },
|
||||
},
|
||||
timeout: {
|
||||
intervalMs: 5_000,
|
||||
initialDelayMs: 60_000,
|
||||
stopTimeoutMs: 1_000,
|
||||
cycle: { pageSize: 8, maxPages: 2 },
|
||||
},
|
||||
});
|
||||
|
||||
assert.deepEqual(await stack.reconcile(), {
|
||||
pages: 1,
|
||||
scanned: 0,
|
||||
verifiedRunning: 0,
|
||||
recoveredRunning: 0,
|
||||
completedFromReceipt: 0,
|
||||
quarantinedReceipts: 0,
|
||||
publishGraceWaits: 0,
|
||||
markedLost: 0,
|
||||
skipped: 0,
|
||||
ambiguous: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
});
|
||||
assert.equal(stack.startCompletion(), true);
|
||||
assert.equal(stack.startCompletion(), false);
|
||||
assert.equal(stack.startTimeout(), true);
|
||||
assert.equal(stack.startTimeout(), false);
|
||||
assert.equal(stack.startCancellation(), true);
|
||||
assert.equal(stack.startCancellation(), false);
|
||||
assert.equal(await stack.stopTimeout(), 'drained');
|
||||
assert.equal(await stack.stopCancellation(), 'drained');
|
||||
assert.equal(await stack.stopCompletion(), 'drained');
|
||||
});
|
||||
|
||||
test('real Primary activation stack rejects invalid worker ownership', async (t) => {
|
||||
const database = await createDatabase(t);
|
||||
const rollout = new RuntimeRolloutPolicy({
|
||||
defaultMode: 'off',
|
||||
origins: { manual: 'primary' },
|
||||
allowLegacyFallbackBeforeStart: false,
|
||||
});
|
||||
|
||||
assert.throws(
|
||||
() =>
|
||||
createDefaultManualPrimaryActivationStack(rollout, {
|
||||
database,
|
||||
owner: '',
|
||||
}),
|
||||
RangeError,
|
||||
);
|
||||
});
|
||||
|
||||
test('local SQLite Primary stack refuses cluster-control and worker profiles', async (t) => {
|
||||
const database = await createDatabase(t);
|
||||
const rollout = new RuntimeRolloutPolicy({
|
||||
defaultMode: 'off',
|
||||
origins: { manual: 'primary' },
|
||||
allowLegacyFallbackBeforeStart: false,
|
||||
});
|
||||
|
||||
for (const deploymentProfile of ['cluster-control', 'worker']) {
|
||||
assert.throws(
|
||||
() =>
|
||||
createDefaultManualPrimaryActivationStack(rollout, {
|
||||
database,
|
||||
deploymentProfile,
|
||||
}),
|
||||
/cannot host the local SQLite Primary stack/,
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,100 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
localPrimaryResourcePolicy,
|
||||
parseDeploymentProfile,
|
||||
} = require('../../back/runtime/domain/deploymentProfile');
|
||||
|
||||
test('uses standalone by default and parses only explicit deployment profiles', () => {
|
||||
assert.equal(parseDeploymentProfile(undefined), 'standalone');
|
||||
assert.equal(parseDeploymentProfile(''), 'standalone');
|
||||
for (const profile of ['edge', 'standalone', 'cluster-control', 'worker']) {
|
||||
assert.equal(parseDeploymentProfile(profile), profile);
|
||||
}
|
||||
for (const profile of ['EDGE', ' edge', 'router', 'cluster']) {
|
||||
assert.throws(() => parseDeploymentProfile(profile), TypeError);
|
||||
}
|
||||
});
|
||||
|
||||
test('keeps edge scans smaller and slower than standalone scans', () => {
|
||||
const edge = localPrimaryResourcePolicy('edge');
|
||||
const standalone = localPrimaryResourcePolicy('standalone');
|
||||
|
||||
assert.ok(edge.completion.intervalMs > standalone.completion.intervalMs);
|
||||
assert.ok(edge.cancellation.intervalMs > standalone.cancellation.intervalMs);
|
||||
assert.ok(edge.timeout.intervalMs > standalone.timeout.intervalMs);
|
||||
assert.ok(edge.retry.intervalMs > standalone.retry.intervalMs);
|
||||
assert.ok(
|
||||
edge.approvedAction.intervalMs > standalone.approvedAction.intervalMs,
|
||||
);
|
||||
assert.ok(
|
||||
edge.artifactRetention.intervalMs > standalone.artifactRetention.intervalMs,
|
||||
);
|
||||
assert.ok(edge.cancellation.pageSize < standalone.cancellation.pageSize);
|
||||
assert.ok(edge.timeout.maxPages < standalone.timeout.maxPages);
|
||||
assert.ok(edge.completion.pageSize < standalone.completion.pageSize);
|
||||
assert.ok(edge.retry.pageSize < standalone.retry.pageSize);
|
||||
assert.ok(
|
||||
edge.approvedAction.dispatch.pageSize <
|
||||
standalone.approvedAction.dispatch.pageSize,
|
||||
);
|
||||
assert.ok(
|
||||
edge.approvedAction.recovery.maxPages <
|
||||
standalone.approvedAction.recovery.maxPages,
|
||||
);
|
||||
assert.ok(
|
||||
edge.artifactRetention.pageSize < standalone.artifactRetention.pageSize,
|
||||
);
|
||||
assert.ok(
|
||||
edge.artifactRetention.maximumDeletions <
|
||||
standalone.artifactRetention.maximumDeletions,
|
||||
);
|
||||
assert.equal(edge.retry.maxPages, 1);
|
||||
assert.equal(standalone.retry.maxPages, 1);
|
||||
assert.equal(edge.receiptPublishGraceMs, 50);
|
||||
assert.equal(standalone.receiptPublishGraceMs, 100);
|
||||
assert.equal(edge.receiptTerminalMissingRetentionMs, 120_000);
|
||||
assert.equal(standalone.receiptTerminalMissingRetentionMs, 60_000);
|
||||
assert.ok(
|
||||
edge.receiptQuarantineRetentionMs < standalone.receiptQuarantineRetentionMs,
|
||||
);
|
||||
|
||||
edge.completion.pageSize = 64;
|
||||
edge.retry.pageSize = 64;
|
||||
edge.approvedAction.dispatch.pageSize = 64;
|
||||
edge.approvedAction.recovery.pageSize = 64;
|
||||
edge.artifactRetention.pageSize = 64;
|
||||
assert.notEqual(
|
||||
localPrimaryResourcePolicy('edge').completion.pageSize,
|
||||
edge.completion.pageSize,
|
||||
);
|
||||
assert.notEqual(
|
||||
localPrimaryResourcePolicy('edge').retry.pageSize,
|
||||
edge.retry.pageSize,
|
||||
);
|
||||
assert.notEqual(
|
||||
localPrimaryResourcePolicy('edge').approvedAction.dispatch.pageSize,
|
||||
edge.approvedAction.dispatch.pageSize,
|
||||
);
|
||||
assert.notEqual(
|
||||
localPrimaryResourcePolicy('edge').approvedAction.recovery.pageSize,
|
||||
edge.approvedAction.recovery.pageSize,
|
||||
);
|
||||
assert.notEqual(
|
||||
localPrimaryResourcePolicy('edge').artifactRetention.pageSize,
|
||||
edge.artifactRetention.pageSize,
|
||||
);
|
||||
});
|
||||
|
||||
test('refuses unsupported local control-plane topologies', () => {
|
||||
assert.throws(
|
||||
() => localPrimaryResourcePolicy('cluster-control'),
|
||||
/cannot host the local SQLite Primary stack/,
|
||||
);
|
||||
assert.throws(
|
||||
() => localPrimaryResourcePolicy('worker'),
|
||||
/cannot host the local SQLite Primary stack/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,498 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { spawn } = require('node:child_process');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const { setTimeout: delay } = require('node:timers/promises');
|
||||
const {
|
||||
CompletionReceiptFileStore,
|
||||
} = require('../../back/runtime/adapters/fs/completionReceiptFileStore');
|
||||
const {
|
||||
LocalArtifactTruncationFactStore,
|
||||
localArtifactTruncationFactFileName,
|
||||
} = require('../../back/runtime/adapters/fs/localArtifactTruncationFactStore');
|
||||
const {
|
||||
enableDurableLocalProcessOutput,
|
||||
} = require('../../back/runtime/adapters/local-process/durableLocalProcessOutput');
|
||||
const {
|
||||
LocalProcessExecutor,
|
||||
} = require('../../back/runtime/adapters/local-process/localProcessExecutor');
|
||||
const {
|
||||
encodeLocalArtifactTruncationFact,
|
||||
} = require('../../back/runtime/domain/localArtifactTruncation');
|
||||
|
||||
const REPOSITORY_ROOT = path.resolve(__dirname, '../..');
|
||||
const LAUNCHER_PATH = path.join(REPOSITORY_ROOT, 'shell', 'ql3-launcher.sh');
|
||||
const RUN_ID = '019f7300-0000-7000-8000-000000000001';
|
||||
const ATTEMPT_ID = '019f7300-0000-7000-8000-000000000002';
|
||||
const CALLBACK_TOKEN = 'a'.repeat(43);
|
||||
const LOG_ARTIFACT_ID = `local-${'b'.repeat(30)}`;
|
||||
const roots = [];
|
||||
|
||||
async function temporaryRoot() {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-launcher-'));
|
||||
roots.push(root);
|
||||
return root;
|
||||
}
|
||||
|
||||
function specification(overrides = {}) {
|
||||
return {
|
||||
runId: RUN_ID,
|
||||
attemptId: ATTEMPT_ID,
|
||||
projectId: 'default',
|
||||
taskId: 'durable-launcher-test',
|
||||
taskRevision: 'revision-1',
|
||||
command: {
|
||||
kind: 'argv',
|
||||
file: process.execPath,
|
||||
args: ['-e', "process.stdout.write('durable-output')"],
|
||||
},
|
||||
environmentPolicy: 'isolated',
|
||||
terminationGraceMs: 100,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function durableContext(root, outputFilePath, write, capability = {}) {
|
||||
return {
|
||||
environment: {},
|
||||
completionCallback: {
|
||||
token: CALLBACK_TOKEN,
|
||||
callbackSequence: 1,
|
||||
},
|
||||
output: enableDurableLocalProcessOutput(
|
||||
{
|
||||
async write(value) {
|
||||
await write?.(value);
|
||||
},
|
||||
},
|
||||
{
|
||||
outputFilePath,
|
||||
completionReceiptRoot: path.join(root, 'receipts'),
|
||||
...capability,
|
||||
},
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForReceipt(store, attemptId, timeoutMs = 5_000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
const value = await store.read(attemptId);
|
||||
if (value) return value;
|
||||
await delay(25);
|
||||
}
|
||||
throw new Error('Timed out waiting for completion receipt');
|
||||
}
|
||||
|
||||
async function waitForFileContent(filePath, expected, timeoutMs = 5_000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
if ((await fs.readFile(filePath, 'utf8')).includes(expected)) return;
|
||||
} catch (error) {
|
||||
if (error?.code !== 'ENOENT') throw error;
|
||||
}
|
||||
await delay(25);
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${expected} in ${filePath}`);
|
||||
}
|
||||
|
||||
function waitForExit(child) {
|
||||
return new Promise((resolve, reject) => {
|
||||
child.once('error', reject);
|
||||
child.once('exit', (code, signal) => resolve({ code, signal }));
|
||||
});
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
roots
|
||||
.splice(0)
|
||||
.map((root) => fs.rm(root, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
test(
|
||||
'writes stdout and stderr directly and publishes one canonical receipt',
|
||||
{ timeout: 10_000 },
|
||||
async () => {
|
||||
const root = await temporaryRoot();
|
||||
const outputFilePath = path.join(root, 'logs', 'attempt.log');
|
||||
let sinkWrites = 0;
|
||||
const executor = new LocalProcessExecutor({
|
||||
durableLauncherPath: LAUNCHER_PATH,
|
||||
});
|
||||
const handle = await executor.start(
|
||||
specification({
|
||||
command: {
|
||||
kind: 'shell',
|
||||
command:
|
||||
"test -z \"${QL3_RECEIPT_CALLBACK_TOKEN+x}\" || exit 91; printf 'stdout-value'; printf 'stderr-value' >&2; exit 7",
|
||||
shell: '/bin/sh',
|
||||
},
|
||||
}),
|
||||
durableContext(root, outputFilePath, async () => {
|
||||
sinkWrites += 1;
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await handle.completion;
|
||||
assert.equal(result.outcome, 'failed');
|
||||
assert.equal(result.exitCode, 7);
|
||||
assert.equal(sinkWrites, 0);
|
||||
assert.equal(
|
||||
await fs.readFile(outputFilePath, 'utf8'),
|
||||
'stdout-valuestderr-value',
|
||||
);
|
||||
assert.equal((await fs.stat(outputFilePath)).mode & 0o777, 0o600);
|
||||
|
||||
const receipt = await new CompletionReceiptFileStore(
|
||||
path.join(root, 'receipts'),
|
||||
).read(ATTEMPT_ID);
|
||||
assert.equal(receipt.runId, RUN_ID);
|
||||
assert.equal(receipt.attemptId, ATTEMPT_ID);
|
||||
assert.equal(receipt.callbackSequence, 1);
|
||||
assert.equal(receipt.token, CALLBACK_TOKEN);
|
||||
assert.equal(receipt.exitCode, 7);
|
||||
assert.ok(receipt.finishedAtMs >= receipt.startedAtMs);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'hard-caps durable output while draining the child to successful completion',
|
||||
{ timeout: 10_000 },
|
||||
async () => {
|
||||
const root = await temporaryRoot();
|
||||
const outputDirectory = path.join(root, 'logs');
|
||||
const outputShard = path.join(outputDirectory, LOG_ARTIFACT_ID.slice(6, 8));
|
||||
const outputFilePath = path.join(outputShard, `${LOG_ARTIFACT_ID}.log`);
|
||||
const maximumBytes = 64 * 1024;
|
||||
const executor = new LocalProcessExecutor({
|
||||
durableLauncherPath: LAUNCHER_PATH,
|
||||
});
|
||||
const handle = await executor.start(
|
||||
specification({
|
||||
command: {
|
||||
kind: 'argv',
|
||||
file: process.execPath,
|
||||
args: [
|
||||
'-e',
|
||||
[
|
||||
'if (process.env.QL3_OUTPUT_QUOTA_FIFO) process.exit(91);',
|
||||
'if (process.env.QL3_OUTPUT_TRUNCATION_TARGET) process.exit(92);',
|
||||
'const chunk = Buffer.alloc(256 * 1024, 0x61);',
|
||||
'process.stdout.write(chunk, () => process.exit(0));',
|
||||
].join(''),
|
||||
],
|
||||
},
|
||||
}),
|
||||
durableContext(root, outputFilePath, undefined, {
|
||||
maximumBytes,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
}),
|
||||
);
|
||||
const result = await handle.completion;
|
||||
assert.equal(result.outcome, 'succeeded');
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal((await fs.stat(outputFilePath)).size, maximumBytes);
|
||||
const truncation = await new LocalArtifactTruncationFactStore(
|
||||
outputDirectory,
|
||||
).read(LOG_ARTIFACT_ID);
|
||||
assert.deepEqual(
|
||||
{
|
||||
schemaVersion: truncation.schemaVersion,
|
||||
runId: truncation.runId,
|
||||
attemptId: truncation.attemptId,
|
||||
logArtifactId: truncation.logArtifactId,
|
||||
maximumBytes: truncation.maximumBytes,
|
||||
quotaReached: truncation.quotaReached,
|
||||
},
|
||||
{
|
||||
schemaVersion: 1,
|
||||
runId: RUN_ID,
|
||||
attemptId: ATTEMPT_ID,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
maximumBytes,
|
||||
quotaReached: true,
|
||||
},
|
||||
);
|
||||
assert.ok(Number.isSafeInteger(truncation.observedAtMs));
|
||||
assert.deepEqual(
|
||||
(await fs.readdir(outputShard)).filter((name) => name.endsWith('.fifo')),
|
||||
[],
|
||||
);
|
||||
assert.deepEqual(
|
||||
(await fs.readdir(outputShard)).filter((name) => name.endsWith('.tmp')),
|
||||
[],
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await waitForReceipt(
|
||||
new CompletionReceiptFileStore(path.join(root, 'receipts')),
|
||||
ATTEMPT_ID,
|
||||
)
|
||||
).exitCode,
|
||||
0,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test('publishes a negative truncation fact when output stays below quota', async () => {
|
||||
const root = await temporaryRoot();
|
||||
const outputDirectory = path.join(root, 'logs');
|
||||
const outputFilePath = path.join(
|
||||
outputDirectory,
|
||||
LOG_ARTIFACT_ID.slice(6, 8),
|
||||
`${LOG_ARTIFACT_ID}.log`,
|
||||
);
|
||||
const executor = new LocalProcessExecutor({
|
||||
durableLauncherPath: LAUNCHER_PATH,
|
||||
});
|
||||
const handle = await executor.start(
|
||||
specification(),
|
||||
durableContext(root, outputFilePath, undefined, {
|
||||
maximumBytes: 64 * 1024,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
}),
|
||||
);
|
||||
assert.equal((await handle.completion).exitCode, 0);
|
||||
const fact = await new LocalArtifactTruncationFactStore(outputDirectory).read(
|
||||
LOG_ARTIFACT_ID,
|
||||
);
|
||||
assert.equal(fact.quotaReached, false);
|
||||
assert.equal(fact.maximumBytes, 64 * 1024);
|
||||
});
|
||||
|
||||
test('never overwrites an already published truncation fact', async () => {
|
||||
const root = await temporaryRoot();
|
||||
const outputDirectory = path.join(root, 'logs');
|
||||
const outputShard = path.join(outputDirectory, LOG_ARTIFACT_ID.slice(6, 8));
|
||||
const outputFilePath = path.join(outputShard, `${LOG_ARTIFACT_ID}.log`);
|
||||
await fs.mkdir(outputShard, { recursive: true, mode: 0o700 });
|
||||
const existing = {
|
||||
schemaVersion: 1,
|
||||
runId: RUN_ID,
|
||||
attemptId: ATTEMPT_ID,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
maximumBytes: 64 * 1024,
|
||||
quotaReached: false,
|
||||
observedAtMs: 1,
|
||||
};
|
||||
await fs.writeFile(
|
||||
path.join(
|
||||
outputShard,
|
||||
localArtifactTruncationFactFileName(LOG_ARTIFACT_ID),
|
||||
),
|
||||
encodeLocalArtifactTruncationFact(existing),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
const executor = new LocalProcessExecutor({
|
||||
durableLauncherPath: LAUNCHER_PATH,
|
||||
});
|
||||
const handle = await executor.start(
|
||||
specification({
|
||||
command: {
|
||||
kind: 'argv',
|
||||
file: process.execPath,
|
||||
args: ['-e', 'process.stdout.write(Buffer.alloc(256 * 1024, 0x61))'],
|
||||
},
|
||||
}),
|
||||
durableContext(root, outputFilePath, undefined, {
|
||||
maximumBytes: 64 * 1024,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
}),
|
||||
);
|
||||
assert.equal((await handle.completion).exitCode, 0);
|
||||
assert.deepEqual(
|
||||
await new LocalArtifactTruncationFactStore(outputDirectory).read(
|
||||
LOG_ARTIFACT_ID,
|
||||
),
|
||||
existing,
|
||||
);
|
||||
});
|
||||
|
||||
test(
|
||||
'keeps the launcher alive through TERM until its child exits',
|
||||
{ timeout: 10_000 },
|
||||
async () => {
|
||||
const root = await temporaryRoot();
|
||||
const receiptRoot = path.join(root, 'receipts');
|
||||
const outputFilePath = path.join(root, 'cancelled.log');
|
||||
const executor = new LocalProcessExecutor({
|
||||
durableLauncherPath: LAUNCHER_PATH,
|
||||
});
|
||||
const handle = await executor.start(
|
||||
specification({
|
||||
command: {
|
||||
kind: 'argv',
|
||||
file: process.execPath,
|
||||
args: [
|
||||
'-e',
|
||||
"process.on('SIGTERM', () => setTimeout(() => process.exit(23), 100)); process.stdout.write('ready'); setInterval(() => undefined, 1000)",
|
||||
],
|
||||
},
|
||||
terminationGraceMs: 1_000,
|
||||
}),
|
||||
durableContext(root, outputFilePath),
|
||||
);
|
||||
await waitForFileContent(outputFilePath, 'ready');
|
||||
const stopped = await executor.stop(handle, {
|
||||
kind: 'user',
|
||||
requestedAtMs: Date.now(),
|
||||
});
|
||||
const result = await handle.completion;
|
||||
|
||||
assert.equal(stopped.killSignalSent, false);
|
||||
assert.equal(result.outcome, 'cancelled');
|
||||
assert.equal(result.exitCode, 23);
|
||||
assert.equal(
|
||||
(
|
||||
await waitForReceipt(
|
||||
new CompletionReceiptFileStore(receiptRoot),
|
||||
ATTEMPT_ID,
|
||||
)
|
||||
).exitCode,
|
||||
23,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'never overwrites an already published completion receipt',
|
||||
{ timeout: 10_000 },
|
||||
async () => {
|
||||
const root = await temporaryRoot();
|
||||
const receiptRoot = path.join(root, 'receipts');
|
||||
const store = new CompletionReceiptFileStore(receiptRoot);
|
||||
const original = {
|
||||
schemaVersion: 1,
|
||||
runId: RUN_ID,
|
||||
attemptId: ATTEMPT_ID,
|
||||
callbackSequence: 1,
|
||||
token: 'b'.repeat(43),
|
||||
startedAtMs: 1_750_000_000_000,
|
||||
finishedAtMs: 1_750_000_000_100,
|
||||
exitCode: 19,
|
||||
};
|
||||
await store.publish(original);
|
||||
|
||||
const executor = new LocalProcessExecutor({
|
||||
durableLauncherPath: LAUNCHER_PATH,
|
||||
});
|
||||
const handle = await executor.start(
|
||||
specification(),
|
||||
durableContext(root, path.join(root, 'attempt.log')),
|
||||
);
|
||||
assert.equal((await handle.completion).outcome, 'succeeded');
|
||||
assert.deepEqual(await store.read(ATTEMPT_ID), original);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'preserves the user exit code when receipt storage is unavailable',
|
||||
{ timeout: 10_000 },
|
||||
async () => {
|
||||
const root = await temporaryRoot();
|
||||
const blocked = path.join(root, 'blocked');
|
||||
await fs.mkdir(blocked, { mode: 0o500 });
|
||||
const target = path.join(blocked, `${ATTEMPT_ID}.json`);
|
||||
const temporary = path.join(
|
||||
blocked,
|
||||
`.${ATTEMPT_ID}.${'a'.repeat(32)}.tmp`,
|
||||
);
|
||||
const child = spawn(
|
||||
'/bin/sh',
|
||||
[LAUNCHER_PATH, 'argv', '/bin/sh', '-c', 'exit 17'],
|
||||
{
|
||||
cwd: REPOSITORY_ROOT,
|
||||
env: {
|
||||
PATH: process.env.PATH,
|
||||
QL3_RECEIPT_RUN_ID: RUN_ID,
|
||||
QL3_RECEIPT_ATTEMPT_ID: ATTEMPT_ID,
|
||||
QL3_RECEIPT_CALLBACK_SEQUENCE: '1',
|
||||
QL3_RECEIPT_CALLBACK_TOKEN: CALLBACK_TOKEN,
|
||||
QL3_RECEIPT_STARTED_AT_MS: String(Date.now()),
|
||||
QL3_RECEIPT_TARGET: target,
|
||||
QL3_RECEIPT_TEMPORARY: temporary,
|
||||
},
|
||||
stdio: 'ignore',
|
||||
},
|
||||
);
|
||||
assert.deepEqual(await waitForExit(child), { code: 17, signal: null });
|
||||
await assert.rejects(fs.lstat(target), /ENOENT/);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'keeps appending output and publishes completion after the parent exits',
|
||||
{ timeout: 15_000 },
|
||||
async () => {
|
||||
const root = await temporaryRoot();
|
||||
const outputFilePath = path.join(root, 'survives-parent.log');
|
||||
const receiptRoot = path.join(root, 'receipts');
|
||||
const durableOutputModule = path.join(
|
||||
REPOSITORY_ROOT,
|
||||
'back/runtime/adapters/local-process/durableLocalProcessOutput',
|
||||
);
|
||||
const executorModule = path.join(
|
||||
REPOSITORY_ROOT,
|
||||
'back/runtime/adapters/local-process/localProcessExecutor',
|
||||
);
|
||||
const childProgram = [
|
||||
"process.stdout.write('before-parent-exit\\n');",
|
||||
"setTimeout(() => { process.stdout.write('after-parent-exit\\n'); process.exit(0); }, 300);",
|
||||
].join('');
|
||||
const controller = `
|
||||
require('ts-node/register/transpile-only');
|
||||
const { enableDurableLocalProcessOutput } = require(${JSON.stringify(
|
||||
durableOutputModule,
|
||||
)});
|
||||
const { LocalProcessExecutor } = require(${JSON.stringify(
|
||||
executorModule,
|
||||
)});
|
||||
const executor = new LocalProcessExecutor({ durableLauncherPath: ${JSON.stringify(
|
||||
LAUNCHER_PATH,
|
||||
)} });
|
||||
executor.start(${JSON.stringify(
|
||||
specification({
|
||||
command: {
|
||||
kind: 'argv',
|
||||
file: process.execPath,
|
||||
args: ['-e', childProgram],
|
||||
},
|
||||
}),
|
||||
)}, {
|
||||
environment: {},
|
||||
completionCallback: { token: ${JSON.stringify(
|
||||
CALLBACK_TOKEN,
|
||||
)}, callbackSequence: 1 },
|
||||
output: enableDurableLocalProcessOutput({ async write() {} }, {
|
||||
outputFilePath: ${JSON.stringify(outputFilePath)},
|
||||
completionReceiptRoot: ${JSON.stringify(receiptRoot)},
|
||||
}),
|
||||
}).then(() => process.exit(0), () => process.exit(1));
|
||||
`;
|
||||
const controllerProcess = spawn(process.execPath, ['-e', controller], {
|
||||
cwd: REPOSITORY_ROOT,
|
||||
stdio: 'ignore',
|
||||
});
|
||||
assert.deepEqual(await waitForExit(controllerProcess), {
|
||||
code: 0,
|
||||
signal: null,
|
||||
});
|
||||
|
||||
const receipt = await waitForReceipt(
|
||||
new CompletionReceiptFileStore(receiptRoot),
|
||||
ATTEMPT_ID,
|
||||
);
|
||||
assert.equal(receipt.exitCode, 0);
|
||||
assert.equal(
|
||||
await fs.readFile(outputFilePath, 'utf8'),
|
||||
'before-parent-exit\nafter-parent-exit\n',
|
||||
);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,91 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
assertExecutionSpec,
|
||||
cloneExecutionSpec,
|
||||
} = require('../../back/runtime/domain/executionSpec');
|
||||
const {
|
||||
createExecutionSpecDigest,
|
||||
} = require('../../back/runtime/domain/runDispatchOffer');
|
||||
|
||||
function spec(overrides = {}) {
|
||||
return {
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
projectId: 'default',
|
||||
taskId: 'task-1',
|
||||
taskRevision: 'v1',
|
||||
command: { kind: 'argv', file: '/usr/bin/node', args: ['task.js'] },
|
||||
environmentPolicy: 'isolated',
|
||||
terminationGraceMs: 5_000,
|
||||
resourcePolicy: {
|
||||
memoryBytes: { value: 1024, enforcement: 'best_effort' },
|
||||
networkIsolation: 'best_effort',
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('validates and deep-clones a portable ExecutionSpec', () => {
|
||||
const source = spec();
|
||||
const cloned = cloneExecutionSpec(source);
|
||||
source.command.args[0] = 'changed.js';
|
||||
source.resourcePolicy.memoryBytes.value = 2048;
|
||||
assert.deepEqual(cloned.command.args, ['task.js']);
|
||||
assert.equal(cloned.resourcePolicy.memoryBytes.value, 1024);
|
||||
});
|
||||
|
||||
test('drops unknown data when cloning an ExecutionSpec for an offer', () => {
|
||||
const cloned = cloneExecutionSpec({
|
||||
...spec(),
|
||||
internalSecret: 'do-not-forward',
|
||||
resourcePolicy: {
|
||||
...spec().resourcePolicy,
|
||||
adapterPrivateField: 'do-not-forward',
|
||||
},
|
||||
});
|
||||
assert.equal(Object.hasOwn(cloned, 'internalSecret'), false);
|
||||
assert.equal(
|
||||
Object.hasOwn(cloned.resourcePolicy, 'adapterPrivateField'),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('digests only the canonical ExecutionSpec payload', () => {
|
||||
const reference = spec();
|
||||
assert.equal(
|
||||
createExecutionSpecDigest(reference),
|
||||
createExecutionSpecDigest({ ...reference, internalSecret: 'ignored' }),
|
||||
);
|
||||
assert.notEqual(
|
||||
createExecutionSpecDigest(reference),
|
||||
createExecutionSpecDigest({
|
||||
...reference,
|
||||
command: { kind: 'argv', file: '/usr/bin/node', args: ['changed.js'] },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects control characters, relative paths, and unsafe numeric limits', () => {
|
||||
assert.throws(
|
||||
() => assertExecutionSpec(spec({ taskId: 'task\n2' })),
|
||||
/control characters/,
|
||||
);
|
||||
assert.throws(
|
||||
() => assertExecutionSpec(spec({ workingDirectory: 'relative/path' })),
|
||||
/absolute path/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
assertExecutionSpec(
|
||||
spec({
|
||||
resourcePolicy: {
|
||||
memoryBytes: { value: 0, enforcement: 'required' },
|
||||
},
|
||||
}),
|
||||
),
|
||||
/positive safe integer/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,138 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
bootstrapHeadlessWorkerRuntime,
|
||||
} = require('../../back/runtime/application/headlessWorkerRuntime');
|
||||
|
||||
function heartbeat(calls, overrides = {}) {
|
||||
return {
|
||||
currentSession() {
|
||||
return undefined;
|
||||
},
|
||||
async start() {
|
||||
calls.push('heartbeat.start');
|
||||
return true;
|
||||
},
|
||||
async drain() {
|
||||
calls.push('heartbeat.drain');
|
||||
return undefined;
|
||||
},
|
||||
async stop() {
|
||||
calls.push('heartbeat.stop');
|
||||
return 'drained';
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('is default-off without touching Worker dependencies', async () => {
|
||||
const calls = [];
|
||||
const result = await bootstrapHeadlessWorkerRuntime({
|
||||
profile: 'worker',
|
||||
heartbeat: heartbeat(calls),
|
||||
executions: {
|
||||
async drain() {
|
||||
calls.push('executions.drain');
|
||||
return 'drained';
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.deepEqual(result, { status: 'disabled' });
|
||||
assert.deepEqual(calls, []);
|
||||
});
|
||||
|
||||
test('refuses to graft the headless topology onto a control-plane profile', async () => {
|
||||
const calls = [];
|
||||
await assert.rejects(
|
||||
bootstrapHeadlessWorkerRuntime({
|
||||
enabled: true,
|
||||
profile: 'cluster-control',
|
||||
heartbeat: heartbeat(calls),
|
||||
executions: {
|
||||
async drain() {
|
||||
return 'drained';
|
||||
},
|
||||
},
|
||||
}),
|
||||
/cannot activate the headless Worker runtime/,
|
||||
);
|
||||
assert.deepEqual(calls, []);
|
||||
});
|
||||
|
||||
test('advertises drain before waiting for tasks and disconnecting', async () => {
|
||||
const calls = [];
|
||||
const result = await bootstrapHeadlessWorkerRuntime({
|
||||
enabled: true,
|
||||
profile: 'worker',
|
||||
heartbeat: heartbeat(calls),
|
||||
executions: {
|
||||
async drain() {
|
||||
calls.push('executions.drain');
|
||||
return 'drained';
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, 'active');
|
||||
assert.equal(await result.runtime.drainAndStop(), 'stopped');
|
||||
assert.deepEqual(calls, [
|
||||
'heartbeat.start',
|
||||
'heartbeat.drain',
|
||||
'executions.drain',
|
||||
'heartbeat.stop',
|
||||
]);
|
||||
});
|
||||
|
||||
test('keeps a draining session alive when task shutdown reaches its bound', async () => {
|
||||
const calls = [];
|
||||
const result = await bootstrapHeadlessWorkerRuntime({
|
||||
enabled: true,
|
||||
profile: 'worker',
|
||||
heartbeat: heartbeat(calls),
|
||||
executions: {
|
||||
async drain() {
|
||||
calls.push('executions.drain');
|
||||
return 'timed_out';
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, 'active');
|
||||
assert.equal(await result.runtime.drainAndStop(), 'executions_timed_out');
|
||||
assert.deepEqual(calls, [
|
||||
'heartbeat.start',
|
||||
'heartbeat.drain',
|
||||
'executions.drain',
|
||||
]);
|
||||
});
|
||||
|
||||
test('reports a control-plane disconnect failure without claiming shutdown', async () => {
|
||||
const calls = [];
|
||||
const result = await bootstrapHeadlessWorkerRuntime({
|
||||
enabled: true,
|
||||
profile: 'worker',
|
||||
heartbeat: heartbeat(calls, {
|
||||
async stop() {
|
||||
calls.push('heartbeat.stop');
|
||||
return 'disconnect_failed';
|
||||
},
|
||||
}),
|
||||
executions: {
|
||||
async drain() {
|
||||
calls.push('executions.drain');
|
||||
return 'drained';
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, 'active');
|
||||
assert.equal(
|
||||
await result.runtime.drainAndStop(),
|
||||
'heartbeat_disconnect_failed',
|
||||
);
|
||||
assert.deepEqual(calls, [
|
||||
'heartbeat.start',
|
||||
'heartbeat.drain',
|
||||
'executions.drain',
|
||||
'heartbeat.stop',
|
||||
]);
|
||||
});
|
||||
@@ -0,0 +1,111 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
DEFAULT_LEGACY_TERMINATION_GRACE_MS,
|
||||
buildLegacyCronExecutionSpec,
|
||||
} = require('../../back/runtime/adapters/legacy/legacyCronExecutionSpec');
|
||||
const {
|
||||
InvalidExecutionSpecError,
|
||||
} = require('../../back/runtime/domain/executorErrors');
|
||||
|
||||
function createInput(overrides = {}) {
|
||||
return {
|
||||
runId: '019f70f0-0000-7000-8000-000000000001',
|
||||
attemptId: '019f70f0-0000-7000-8000-000000000002',
|
||||
projectId: 'default',
|
||||
taskRevision: 'legacy-revision-1',
|
||||
cron: {
|
||||
id: 7,
|
||||
command: 'demo/script.js now',
|
||||
},
|
||||
realTime: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('builds the minimal legacy task.sh shell contract', () => {
|
||||
assert.deepEqual(buildLegacyCronExecutionSpec(createInput()), {
|
||||
runId: '019f70f0-0000-7000-8000-000000000001',
|
||||
attemptId: '019f70f0-0000-7000-8000-000000000002',
|
||||
projectId: 'default',
|
||||
taskId: 'legacy-cron:7',
|
||||
taskRevision: 'legacy-revision-1',
|
||||
command: {
|
||||
kind: 'shell',
|
||||
command: "real_time='false' no_tee='true' ID='7' task demo/script.js now",
|
||||
shell: '/bin/bash',
|
||||
},
|
||||
environmentPolicy: 'inherit',
|
||||
terminationGraceMs: DEFAULT_LEGACY_TERMINATION_GRACE_MS,
|
||||
});
|
||||
});
|
||||
|
||||
test('preserves existing task/ql commands instead of adding a second prefix', () => {
|
||||
const task = buildLegacyCronExecutionSpec(
|
||||
createInput({ cron: { id: 8, command: ' task script.py now ' } }),
|
||||
);
|
||||
const ql = buildLegacyCronExecutionSpec(
|
||||
createInput({ cron: { id: 9, command: 'ql update' } }),
|
||||
);
|
||||
|
||||
assert.match(task.command.command, / ID='8' task script\.py now$/);
|
||||
assert.match(ql.command.command, / ID='9' ql update$/);
|
||||
});
|
||||
|
||||
test('quotes paths, hook commands, and log values without changing shell boundaries', () => {
|
||||
const spec = buildLegacyCronExecutionSpec(
|
||||
createInput({
|
||||
realTime: true,
|
||||
realLogPath: "folder with space/run's.log",
|
||||
noDelay: true,
|
||||
timeoutMs: 60_000,
|
||||
terminationGraceMs: 2_000,
|
||||
resourcePolicy: {
|
||||
memoryBytes: {
|
||||
value: 128 * 1024 * 1024,
|
||||
enforcement: 'best_effort',
|
||||
},
|
||||
},
|
||||
cron: {
|
||||
id: 10,
|
||||
command: "scripts/job.py now -- 'user value'",
|
||||
taskBefore: "echo 'before'\n echo second",
|
||||
taskAfter: 'echo after;\n echo done',
|
||||
workDirectory: "/data/project's worktree",
|
||||
logName: 'custom log',
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
spec.command.command,
|
||||
"real_log_path='folder with space/run'\\''s.log' no_delay='true' real_time='true' no_tee='true' ID='10' log_name='custom log' task_before='echo '\\''before'\\''; echo second' task_after='echo after; echo done' work_dir='/data/project'\\''s worktree' task scripts/job.py now -- 'user value'",
|
||||
);
|
||||
assert.equal(spec.timeoutMs, 60_000);
|
||||
assert.equal(spec.terminationGraceMs, 2_000);
|
||||
assert.deepEqual(spec.resourcePolicy, {
|
||||
memoryBytes: {
|
||||
value: 128 * 1024 * 1024,
|
||||
enforcement: 'best_effort',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test('rejects invalid legacy ids and empty commands before execution', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
buildLegacyCronExecutionSpec(
|
||||
createInput({ cron: { id: 0, command: 'script.py' } }),
|
||||
),
|
||||
InvalidExecutionSpecError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
buildLegacyCronExecutionSpec(
|
||||
createInput({ cron: { id: 1, command: ' ' } }),
|
||||
),
|
||||
InvalidExecutionSpecError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,160 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
installLegacyExecutionObserver,
|
||||
observeLegacyCancellation,
|
||||
observeLegacyExecution,
|
||||
observeLegacyExecutionCallback,
|
||||
shadowBridgeFailureSnapshot,
|
||||
} = require('../../back/runtime/compatibility/legacyExecutionBridge');
|
||||
|
||||
function acceptedFact(origin = 'manual') {
|
||||
return {
|
||||
origin,
|
||||
projectId: 'default',
|
||||
taskId: 'legacy-cron:1',
|
||||
taskRevision: 'sha256:revision',
|
||||
legacyCronId: 1,
|
||||
triggerType: origin,
|
||||
acceptedAtMs: 1_750_000_000_000,
|
||||
};
|
||||
}
|
||||
|
||||
test('routes only installed origins and preserves observation fact order', () => {
|
||||
const facts = [];
|
||||
const observer = {
|
||||
begin(fact) {
|
||||
facts.push(['accepted', fact]);
|
||||
return {
|
||||
spawned: (value) => facts.push(['spawned', value]),
|
||||
running: (value) => facts.push(['running', value]),
|
||||
startFailed: (value) => facts.push(['start_failed', value]),
|
||||
exited: (value) => facts.push(['exited', value]),
|
||||
cancelled: (value) => facts.push(['cancelled', value]),
|
||||
};
|
||||
},
|
||||
};
|
||||
const restore = installLegacyExecutionObserver(observer, ['manual']);
|
||||
try {
|
||||
const observation = observeLegacyExecution('manual', () => acceptedFact());
|
||||
observation.spawned({ atMs: 1, pid: 10 });
|
||||
observation.running({ atMs: 2 });
|
||||
observation.exited({ atMs: 3, exitCode: 0 });
|
||||
|
||||
let ignoredFactCreated = false;
|
||||
const ignored = observeLegacyExecution('boot', () => {
|
||||
ignoredFactCreated = true;
|
||||
return acceptedFact('boot');
|
||||
});
|
||||
assert.equal(ignored, undefined);
|
||||
assert.equal(ignoredFactCreated, false);
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
|
||||
assert.deepEqual(
|
||||
facts.map(([type]) => type),
|
||||
['accepted', 'spawned', 'running', 'exited'],
|
||||
);
|
||||
});
|
||||
|
||||
test('turns synchronous observer initialization failures into a no-op', () => {
|
||||
const before = shadowBridgeFailureSnapshot()['manual:begin:failed'] ?? 0;
|
||||
const restore = installLegacyExecutionObserver(
|
||||
{
|
||||
begin() {
|
||||
throw new Error('must not reach legacy execution');
|
||||
},
|
||||
},
|
||||
['manual'],
|
||||
);
|
||||
try {
|
||||
const observation = observeLegacyExecution('manual', () => acceptedFact());
|
||||
assert.doesNotThrow(() => {
|
||||
observation.spawned({ atMs: 1 });
|
||||
observation.running({ atMs: 2 });
|
||||
observation.exited({ atMs: 3, exitCode: 0 });
|
||||
});
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
|
||||
const after = shadowBridgeFailureSnapshot()['manual:begin:failed'];
|
||||
assert.equal(after, before + 1);
|
||||
});
|
||||
|
||||
test('routes local callback and cancellation facts without persistent lookup', () => {
|
||||
const facts = [];
|
||||
const restore = installLegacyExecutionObserver(
|
||||
{
|
||||
begin() {
|
||||
return {
|
||||
spawned: (fact) => facts.push(['spawned', fact]),
|
||||
running: (fact) => facts.push(['running', fact]),
|
||||
startFailed: (fact) => facts.push(['start_failed', fact]),
|
||||
exited: (fact) => facts.push(['exited', fact]),
|
||||
cancelled: (fact) => facts.push(['cancelled', fact]),
|
||||
};
|
||||
},
|
||||
},
|
||||
['manual'],
|
||||
);
|
||||
try {
|
||||
const observation = observeLegacyExecution('manual', () => ({
|
||||
...acceptedFact(),
|
||||
legacyCronId: 77,
|
||||
}));
|
||||
observation.spawned({ atMs: 1, pid: 700 });
|
||||
observeLegacyExecutionCallback({
|
||||
legacyCronId: 77,
|
||||
pid: 700,
|
||||
atMs: 2,
|
||||
phase: 'running',
|
||||
});
|
||||
observeLegacyCancellation({
|
||||
legacyCronId: 77,
|
||||
pid: 700,
|
||||
atMs: 3,
|
||||
scope: 'one',
|
||||
reason: 'user',
|
||||
});
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
|
||||
assert.deepEqual(
|
||||
facts.map(([operation]) => operation),
|
||||
['spawned', 'spawned', 'running', 'cancelled'],
|
||||
);
|
||||
});
|
||||
|
||||
test('keeps invalid shadow origin configuration fail-open when logging fails', () => {
|
||||
const Logger = require('../../back/loaders/logger').default;
|
||||
const previousWarn = Logger.warn;
|
||||
const previousOrigins = process.env.QL3_SHADOW_ORIGINS;
|
||||
let factCreated = false;
|
||||
process.env.QL3_SHADOW_ORIGINS = 'secret-invalid-origin';
|
||||
Logger.warn = () => {
|
||||
throw new Error('logger unavailable');
|
||||
};
|
||||
try {
|
||||
let observation;
|
||||
assert.doesNotThrow(() => {
|
||||
observation = observeLegacyExecution('manual', () => {
|
||||
factCreated = true;
|
||||
return acceptedFact();
|
||||
});
|
||||
});
|
||||
assert.equal(observation, undefined);
|
||||
assert.equal(factCreated, false);
|
||||
} finally {
|
||||
Logger.warn = previousWarn;
|
||||
if (previousOrigins === undefined) {
|
||||
delete process.env.QL3_SHADOW_ORIGINS;
|
||||
} else {
|
||||
process.env.QL3_SHADOW_ORIGINS = previousOrigins;
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,200 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
LegacyExecutionRegistry,
|
||||
} = require('../../back/runtime/compatibility/legacyExecutionRegistry');
|
||||
|
||||
function acceptedFact(legacyCronId) {
|
||||
return {
|
||||
origin: 'manual',
|
||||
projectId: 'default',
|
||||
taskId: `legacy-cron:${legacyCronId}`,
|
||||
taskRevision: 'sha256:revision',
|
||||
legacyCronId,
|
||||
triggerType: 'manual',
|
||||
acceptedAtMs: 100,
|
||||
};
|
||||
}
|
||||
|
||||
function recorder(label, calls) {
|
||||
return {
|
||||
spawned: (fact) => calls.push([label, 'spawned', fact]),
|
||||
running: (fact) => calls.push([label, 'running', fact]),
|
||||
startFailed: (fact) => calls.push([label, 'start_failed', fact]),
|
||||
exited: (fact) => calls.push([label, 'exited', fact]),
|
||||
cancelled: (fact) => calls.push([label, 'cancelled', fact]),
|
||||
};
|
||||
}
|
||||
|
||||
test('correlates local callbacks by log, then pid, and removes terminal entries', () => {
|
||||
const calls = [];
|
||||
const registry = new LegacyExecutionRegistry();
|
||||
const first = registry.register(acceptedFact(7), recorder('first', calls));
|
||||
const second = registry.register(acceptedFact(7), recorder('second', calls));
|
||||
first.spawned({ atMs: 101, pid: 11, logArtifactId: 'log-first' });
|
||||
second.spawned({ atMs: 101, pid: 22, logArtifactId: 'log-second' });
|
||||
|
||||
assert.equal(
|
||||
registry.callback({
|
||||
legacyCronId: 7,
|
||||
pid: 999,
|
||||
logArtifactId: 'log-second',
|
||||
atMs: 102,
|
||||
phase: 'running',
|
||||
}),
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
registry.callback({
|
||||
legacyCronId: 7,
|
||||
pid: 11,
|
||||
atMs: 103,
|
||||
phase: 'finished',
|
||||
exitCode: 0,
|
||||
}),
|
||||
1,
|
||||
);
|
||||
assert.equal(registry.size(), 1);
|
||||
assert.equal(
|
||||
registry.cancel({
|
||||
legacyCronId: 7,
|
||||
atMs: 104,
|
||||
scope: 'all',
|
||||
reason: 'user',
|
||||
}),
|
||||
1,
|
||||
);
|
||||
assert.equal(registry.size(), 0);
|
||||
assert.deepEqual(
|
||||
calls.map(([label, operation]) => [label, operation]),
|
||||
[
|
||||
['first', 'spawned'],
|
||||
['second', 'spawned'],
|
||||
['second', 'spawned'],
|
||||
['second', 'running'],
|
||||
['first', 'exited'],
|
||||
['second', 'cancelled'],
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('refuses ambiguous one-instance correlation and bounds local memory', () => {
|
||||
const calls = [];
|
||||
let overflows = 0;
|
||||
const registry = new LegacyExecutionRegistry({
|
||||
maxEntries: 1,
|
||||
onOverflow: () => {
|
||||
overflows += 1;
|
||||
},
|
||||
});
|
||||
registry.register(acceptedFact(8), recorder('tracked', calls));
|
||||
const untracked = registry.register(
|
||||
acceptedFact(8),
|
||||
recorder('untracked', calls),
|
||||
);
|
||||
|
||||
assert.equal(overflows, 1);
|
||||
assert.equal(registry.size(), 1);
|
||||
untracked.running({ atMs: 200 });
|
||||
assert.equal(
|
||||
registry.callback({
|
||||
legacyCronId: 8,
|
||||
atMs: 201,
|
||||
phase: 'finished',
|
||||
}),
|
||||
1,
|
||||
);
|
||||
assert.equal(registry.size(), 0);
|
||||
assert.deepEqual(
|
||||
calls.map(([label, operation]) => [label, operation]),
|
||||
[
|
||||
['untracked', 'running'],
|
||||
['tracked', 'exited'],
|
||||
],
|
||||
);
|
||||
|
||||
const ambiguous = new LegacyExecutionRegistry();
|
||||
ambiguous.register(acceptedFact(9), recorder('a', calls));
|
||||
ambiguous.register(acceptedFact(9), recorder('b', calls));
|
||||
assert.equal(
|
||||
ambiguous.callback({
|
||||
legacyCronId: 9,
|
||||
atMs: 300,
|
||||
phase: 'running',
|
||||
}),
|
||||
0,
|
||||
);
|
||||
|
||||
const conflicting = new LegacyExecutionRegistry();
|
||||
const left = conflicting.register(acceptedFact(12), recorder('left', calls));
|
||||
const right = conflicting.register(
|
||||
acceptedFact(12),
|
||||
recorder('right', calls),
|
||||
);
|
||||
left.spawned({ atMs: 1, pid: 1, logArtifactId: 'left-log' });
|
||||
right.spawned({ atMs: 1, pid: 2, logArtifactId: 'right-log' });
|
||||
assert.equal(
|
||||
conflicting.callback({
|
||||
legacyCronId: 12,
|
||||
pid: 1,
|
||||
logArtifactId: 'right-log',
|
||||
atMs: 2,
|
||||
phase: 'finished',
|
||||
}),
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
test('cleans up entries after start failures and direct terminal observations', () => {
|
||||
const calls = [];
|
||||
const registry = new LegacyExecutionRegistry();
|
||||
const failed = registry.register(acceptedFact(10), recorder('failed', calls));
|
||||
failed.startFailed({ atMs: 2, errorCode: 'SPAWN_FAILED' });
|
||||
const exited = registry.register(acceptedFact(11), recorder('exited', calls));
|
||||
exited.exited({ atMs: 3, exitCode: 1 });
|
||||
|
||||
assert.equal(registry.size(), 0);
|
||||
});
|
||||
|
||||
test('swallows local observer failures and reports them out of band', () => {
|
||||
let failures = 0;
|
||||
const registry = new LegacyExecutionRegistry({
|
||||
onDispatchFailure: () => {
|
||||
failures += 1;
|
||||
},
|
||||
});
|
||||
const throwing = {
|
||||
spawned() {
|
||||
throw new Error('spawned failed');
|
||||
},
|
||||
running() {
|
||||
throw new Error('running failed');
|
||||
},
|
||||
startFailed() {
|
||||
throw new Error('start failed');
|
||||
},
|
||||
exited() {
|
||||
throw new Error('exit failed');
|
||||
},
|
||||
cancelled() {
|
||||
throw new Error('cancel failed');
|
||||
},
|
||||
};
|
||||
const observation = registry.register(acceptedFact(13), throwing);
|
||||
|
||||
assert.doesNotThrow(() => {
|
||||
observation.spawned({ atMs: 1, pid: 13 });
|
||||
observation.running({ atMs: 2 });
|
||||
registry.cancel({
|
||||
legacyCronId: 13,
|
||||
pid: 13,
|
||||
atMs: 3,
|
||||
scope: 'one',
|
||||
reason: 'user',
|
||||
});
|
||||
});
|
||||
assert.equal(failures, 3);
|
||||
assert.equal(registry.size(), 0);
|
||||
});
|
||||
@@ -0,0 +1,58 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
createLegacyLogOutputRef,
|
||||
LEGACY_LOG_OUTPUT_REF_PREFIX,
|
||||
MAX_LEGACY_LOG_PATH_BYTES,
|
||||
parseLegacyLogOutputRef,
|
||||
} = require('../../back/runtime/compatibility/legacyLogOutputRef');
|
||||
|
||||
test('round-trips a bounded relative legacy log path', () => {
|
||||
const logPath = 'task-name/2026-07-18-12-00-00.log';
|
||||
const outputRef = createLegacyLogOutputRef(logPath);
|
||||
|
||||
assert.match(outputRef, /^legacy-log-v1\.[A-Za-z0-9_-]+$/);
|
||||
assert.equal(parseLegacyLogOutputRef(outputRef), logPath);
|
||||
});
|
||||
|
||||
test('normalizes producer paths but requires a canonical encoded reference', () => {
|
||||
const canonical = createLegacyLogOutputRef('task//nested/./run.log');
|
||||
assert.equal(parseLegacyLogOutputRef(canonical), 'task/nested/run.log');
|
||||
|
||||
const nonCanonical =
|
||||
LEGACY_LOG_OUTPUT_REF_PREFIX +
|
||||
Buffer.from('task//run.log').toString('base64url');
|
||||
assert.equal(parseLegacyLogOutputRef(nonCanonical), null);
|
||||
});
|
||||
|
||||
test('rejects absolute, traversing, Windows and NUL-containing paths', () => {
|
||||
for (const value of [
|
||||
'/var/log/secret.log',
|
||||
'../secret.log',
|
||||
'task/../../secret.log',
|
||||
'C:\\secret.log',
|
||||
'task\\secret.log',
|
||||
'task/' + String.fromCharCode(0) + 'secret.log',
|
||||
]) {
|
||||
assert.throws(() => createLegacyLogOutputRef(value));
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects oversized, malformed and non-canonical references', () => {
|
||||
assert.throws(() =>
|
||||
createLegacyLogOutputRef('a'.repeat(MAX_LEGACY_LOG_PATH_BYTES + 1)),
|
||||
);
|
||||
assert.equal(parseLegacyLogOutputRef('legacy-log-v1.'), null);
|
||||
assert.equal(parseLegacyLogOutputRef('legacy-log-v1.***'), null);
|
||||
assert.equal(parseLegacyLogOutputRef('legacy-log-v2.dGFzay5sb2c'), null);
|
||||
assert.equal(
|
||||
parseLegacyLogOutputRef(LEGACY_LOG_OUTPUT_REF_PREFIX + 'Zh'),
|
||||
null,
|
||||
);
|
||||
assert.equal(
|
||||
parseLegacyLogOutputRef(LEGACY_LOG_OUTPUT_REF_PREFIX + 'a'.repeat(513)),
|
||||
null,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,416 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { test } = require('node:test');
|
||||
const jwt = require('jsonwebtoken');
|
||||
const { Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const {
|
||||
IDENTITY_AUTHENTICATION_BINDING_CURRENT_INDEX,
|
||||
IDENTITY_AUTHENTICATION_BINDING_SUBJECT_INDEX,
|
||||
IDENTITY_AUTHENTICATION_BINDING_TABLE,
|
||||
IDENTITY_SUBJECT_STATUS_INDEX,
|
||||
IDENTITY_SUBJECT_TABLE,
|
||||
identityDirectoryMigration,
|
||||
} = require('../../back/migrations/0019-identity-directory');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacyAuthInfoSessionSource,
|
||||
LegacyPanelSessionUnavailableError,
|
||||
MAX_LEGACY_PANEL_TOKENS_PER_PLATFORM,
|
||||
} = require('../../back/runtime/adapters/authentication/legacyAuthInfoSessionSource');
|
||||
const {
|
||||
LegacySequelizeIdentityDirectoryRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/identityDirectoryRepository');
|
||||
const {
|
||||
LegacyPanelAuthenticationRejectedError,
|
||||
LegacyPanelAuthenticationService,
|
||||
LegacyPanelAuthenticationUnavailableError,
|
||||
} = require('../../back/runtime/application/legacyPanelAuthenticationService');
|
||||
const {
|
||||
IdentityDirectoryUnavailableError,
|
||||
LEGACY_PANEL_IDENTITY_PROVIDER,
|
||||
LEGACY_PANEL_PROVIDER_SUBJECT,
|
||||
LEGACY_PRIMARY_USER_SUBJECT_ID,
|
||||
} = require('../../back/runtime/domain/identityDirectory');
|
||||
|
||||
const SECRET = 'test-only-legacy-jwt-secret';
|
||||
const ISSUED_AT_SECONDS = 100;
|
||||
const EXPIRES_AT_SECONDS = 200;
|
||||
const NOW_MS = 150_000;
|
||||
|
||||
function signToken(
|
||||
payload = {
|
||||
data: 'legacy-session-random-data',
|
||||
iat: ISSUED_AT_SECONDS,
|
||||
exp: EXPIRES_AT_SECONDS,
|
||||
},
|
||||
algorithm = 'HS384',
|
||||
secret = SECRET,
|
||||
) {
|
||||
return jwt.sign(payload, secret, { algorithm });
|
||||
}
|
||||
|
||||
async function setup(t) {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [identityDirectoryMigration],
|
||||
logger: { info() {} },
|
||||
});
|
||||
return {
|
||||
database,
|
||||
directory: new LegacySequelizeIdentityDirectoryRepository(database),
|
||||
};
|
||||
}
|
||||
|
||||
function sessionSource(snapshot) {
|
||||
return new LegacyAuthInfoSessionSource(async () => snapshot);
|
||||
}
|
||||
|
||||
function authentication(directory, snapshot, secret = SECRET) {
|
||||
return new LegacyPanelAuthenticationService(
|
||||
directory,
|
||||
sessionSource(snapshot),
|
||||
secret,
|
||||
);
|
||||
}
|
||||
|
||||
test('migration creates a stable singleton identity without copying legacy credentials', async (t) => {
|
||||
const { database, directory } = await setup(t);
|
||||
const subjects = await database
|
||||
.getQueryInterface()
|
||||
.select(null, IDENTITY_SUBJECT_TABLE);
|
||||
const bindings = await database
|
||||
.getQueryInterface()
|
||||
.select(null, IDENTITY_AUTHENTICATION_BINDING_TABLE);
|
||||
assert.deepEqual(subjects, [
|
||||
{
|
||||
id: LEGACY_PRIMARY_USER_SUBJECT_ID,
|
||||
type: 'user',
|
||||
status: 'active',
|
||||
version: 1,
|
||||
created_at_ms: 0,
|
||||
updated_at_ms: 0,
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(bindings, [
|
||||
{
|
||||
provider: LEGACY_PANEL_IDENTITY_PROVIDER,
|
||||
provider_subject: LEGACY_PANEL_PROVIDER_SUBJECT,
|
||||
version: 1,
|
||||
state: 'active',
|
||||
subject_id: LEGACY_PRIMARY_USER_SUBJECT_ID,
|
||||
created_at_ms: 0,
|
||||
},
|
||||
]);
|
||||
const persisted = JSON.stringify({ subjects, bindings });
|
||||
assert.equal(persisted.includes('username'), false);
|
||||
assert.equal(persisted.includes('password'), false);
|
||||
assert.equal(persisted.includes('token'), false);
|
||||
assert.deepEqual(
|
||||
await directory.resolveAuthenticationSubject(
|
||||
LEGACY_PANEL_IDENTITY_PROVIDER,
|
||||
LEGACY_PANEL_PROVIDER_SUBJECT,
|
||||
),
|
||||
{ type: 'user', id: LEGACY_PRIMARY_USER_SUBJECT_ID },
|
||||
);
|
||||
|
||||
const subjectIndexes = new Set(
|
||||
(await database.getQueryInterface().showIndex(IDENTITY_SUBJECT_TABLE)).map(
|
||||
(index) => index.name,
|
||||
),
|
||||
);
|
||||
const bindingIndexes = new Set(
|
||||
(
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.showIndex(IDENTITY_AUTHENTICATION_BINDING_TABLE)
|
||||
).map((index) => index.name),
|
||||
);
|
||||
assert.ok(subjectIndexes.has(IDENTITY_SUBJECT_STATUS_INDEX));
|
||||
assert.ok(bindingIndexes.has(IDENTITY_AUTHENTICATION_BINDING_CURRENT_INDEX));
|
||||
assert.ok(bindingIndexes.has(IDENTITY_AUTHENTICATION_BINDING_SUBJECT_INDEX));
|
||||
});
|
||||
|
||||
test('authenticates a current HS384 legacy session as one stable single-factor user', async (t) => {
|
||||
const { directory } = await setup(t);
|
||||
const token = signToken();
|
||||
const service = authentication(directory, {
|
||||
token: '',
|
||||
tokens: {
|
||||
desktop: [
|
||||
{
|
||||
value: token,
|
||||
timestamp: 123,
|
||||
platform: 'desktop',
|
||||
},
|
||||
],
|
||||
},
|
||||
username: 'a-display-name-that-may-change',
|
||||
twoFactorActivated: true,
|
||||
});
|
||||
assert.deepEqual(
|
||||
await service.authenticate({ token, platform: 'desktop', nowMs: NOW_MS }),
|
||||
{
|
||||
subject: { type: 'user', id: LEGACY_PRIMARY_USER_SUBJECT_ID },
|
||||
authenticationId: `legacy_panel:${createHash('sha256')
|
||||
.update(token, 'utf8')
|
||||
.digest('hex')}`,
|
||||
authenticatedAtMs: ISSUED_AT_SECONDS * 1000,
|
||||
expiresAtMs: EXPIRES_AT_SECONDS * 1000,
|
||||
assurance: 'single_factor',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('supports only the bounded legacy primary, string and TokenInfo list formats', async () => {
|
||||
const token = signToken();
|
||||
assert.equal(await sessionSource({ token }).isActive(token, 'mobile'), true);
|
||||
assert.equal(
|
||||
await sessionSource({ tokens: { desktop: token } }).isActive(
|
||||
token,
|
||||
'desktop',
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
await sessionSource({
|
||||
tokens: { desktop: [{ value: token }] },
|
||||
}).isActive(token, 'desktop'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
await sessionSource({ tokens: { mobile: [{ value: token }] } }).isActive(
|
||||
token,
|
||||
'desktop',
|
||||
),
|
||||
false,
|
||||
);
|
||||
await assert.rejects(
|
||||
sessionSource({
|
||||
tokens: {
|
||||
desktop: Array.from(
|
||||
{ length: MAX_LEGACY_PANEL_TOKENS_PER_PLATFORM + 1 },
|
||||
() => ({ value: token }),
|
||||
),
|
||||
},
|
||||
}).isActive(token, 'desktop'),
|
||||
LegacyPanelSessionUnavailableError,
|
||||
);
|
||||
await assert.rejects(
|
||||
sessionSource({ tokens: { desktop: [{}] } }).isActive(token, 'desktop'),
|
||||
LegacyPanelSessionUnavailableError,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects logout, platform drift, expiry, wrong signature and wrong algorithm', async (t) => {
|
||||
const { directory } = await setup(t);
|
||||
const token = signToken();
|
||||
for (const [service, request] of [
|
||||
[authentication(directory, { tokens: {} }), {}],
|
||||
[
|
||||
authentication(directory, {
|
||||
tokens: { mobile: [{ value: token }] },
|
||||
}),
|
||||
{},
|
||||
],
|
||||
[
|
||||
authentication(directory, {
|
||||
tokens: { desktop: [{ value: token }] },
|
||||
}),
|
||||
{ nowMs: EXPIRES_AT_SECONDS * 1000 },
|
||||
],
|
||||
[
|
||||
authentication(
|
||||
directory,
|
||||
{ tokens: { desktop: [{ value: token }] } },
|
||||
'different-secret',
|
||||
),
|
||||
{},
|
||||
],
|
||||
[
|
||||
authentication(directory, {
|
||||
tokens: { desktop: [{ value: signToken(undefined, 'HS256') }] },
|
||||
}),
|
||||
{ token: signToken(undefined, 'HS256') },
|
||||
],
|
||||
]) {
|
||||
await assert.rejects(
|
||||
service.authenticate({
|
||||
token,
|
||||
platform: 'desktop',
|
||||
nowMs: NOW_MS,
|
||||
...request,
|
||||
}),
|
||||
LegacyPanelAuthenticationRejectedError,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects extensible JWTs and request-supplied subjects before identity lookup', async (t) => {
|
||||
const { directory } = await setup(t);
|
||||
let sessionReads = 0;
|
||||
const extraPayloadToken = signToken({
|
||||
data: 'legacy-session-random-data',
|
||||
iat: ISSUED_AT_SECONDS,
|
||||
exp: EXPIRES_AT_SECONDS,
|
||||
subject: 'attacker',
|
||||
});
|
||||
const service = new LegacyPanelAuthenticationService(
|
||||
directory,
|
||||
{
|
||||
async isActive() {
|
||||
sessionReads += 1;
|
||||
return true;
|
||||
},
|
||||
},
|
||||
SECRET,
|
||||
);
|
||||
await assert.rejects(
|
||||
service.authenticate({
|
||||
token: extraPayloadToken,
|
||||
platform: 'desktop',
|
||||
nowMs: NOW_MS,
|
||||
}),
|
||||
LegacyPanelAuthenticationRejectedError,
|
||||
);
|
||||
assert.equal(sessionReads, 0);
|
||||
const futureIssuedToken = signToken({
|
||||
data: 'legacy-session-random-data',
|
||||
iat: ISSUED_AT_SECONDS + 60,
|
||||
exp: EXPIRES_AT_SECONDS,
|
||||
});
|
||||
await assert.rejects(
|
||||
service.authenticate({
|
||||
token: futureIssuedToken,
|
||||
platform: 'desktop',
|
||||
nowMs: NOW_MS,
|
||||
}),
|
||||
LegacyPanelAuthenticationRejectedError,
|
||||
);
|
||||
assert.equal(sessionReads, 0);
|
||||
await assert.rejects(
|
||||
service.authenticate({
|
||||
token: signToken(),
|
||||
platform: 'desktop',
|
||||
nowMs: NOW_MS,
|
||||
subject: { type: 'user', id: 'attacker' },
|
||||
}),
|
||||
/request shape is invalid/,
|
||||
);
|
||||
assert.equal(sessionReads, 0);
|
||||
});
|
||||
|
||||
test('revocation and subject disablement remove legacy authentication authority', async (t) => {
|
||||
const revoked = await setup(t);
|
||||
await revoked.database
|
||||
.getQueryInterface()
|
||||
.bulkInsert(IDENTITY_AUTHENTICATION_BINDING_TABLE, [
|
||||
{
|
||||
provider: LEGACY_PANEL_IDENTITY_PROVIDER,
|
||||
provider_subject: LEGACY_PANEL_PROVIDER_SUBJECT,
|
||||
version: 2,
|
||||
state: 'revoked',
|
||||
subject_id: LEGACY_PRIMARY_USER_SUBJECT_ID,
|
||||
created_at_ms: NOW_MS,
|
||||
},
|
||||
]);
|
||||
assert.equal(
|
||||
await revoked.directory.resolveAuthenticationSubject(
|
||||
LEGACY_PANEL_IDENTITY_PROVIDER,
|
||||
LEGACY_PANEL_PROVIDER_SUBJECT,
|
||||
),
|
||||
null,
|
||||
);
|
||||
|
||||
const disabled = await setup(t);
|
||||
await disabled.database
|
||||
.getQueryInterface()
|
||||
.bulkUpdate(
|
||||
IDENTITY_SUBJECT_TABLE,
|
||||
{ status: 'disabled', version: 2, updated_at_ms: NOW_MS },
|
||||
{ id: LEGACY_PRIMARY_USER_SUBJECT_ID },
|
||||
);
|
||||
const token = signToken();
|
||||
await assert.rejects(
|
||||
authentication(disabled.directory, {
|
||||
tokens: { desktop: [{ value: token }] },
|
||||
}).authenticate({ token, platform: 'desktop', nowMs: NOW_MS }),
|
||||
LegacyPanelAuthenticationRejectedError,
|
||||
);
|
||||
});
|
||||
|
||||
test('fails closed on corrupt identity storage and session source failures', async (t) => {
|
||||
const { database, directory } = await setup(t);
|
||||
await database.query('PRAGMA ignore_check_constraints = ON');
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.bulkUpdate(
|
||||
IDENTITY_AUTHENTICATION_BINDING_TABLE,
|
||||
{ state: 'corrupt' },
|
||||
{ provider: LEGACY_PANEL_IDENTITY_PROVIDER },
|
||||
);
|
||||
await assert.rejects(
|
||||
directory.resolveAuthenticationSubject(
|
||||
LEGACY_PANEL_IDENTITY_PROVIDER,
|
||||
LEGACY_PANEL_PROVIDER_SUBJECT,
|
||||
),
|
||||
IdentityDirectoryUnavailableError,
|
||||
);
|
||||
|
||||
const orphaned = await setup(t);
|
||||
await orphaned.database.query('PRAGMA foreign_keys = OFF');
|
||||
await orphaned.database
|
||||
.getQueryInterface()
|
||||
.bulkDelete(IDENTITY_SUBJECT_TABLE, {
|
||||
id: LEGACY_PRIMARY_USER_SUBJECT_ID,
|
||||
});
|
||||
await assert.rejects(
|
||||
orphaned.directory.resolveAuthenticationSubject(
|
||||
LEGACY_PANEL_IDENTITY_PROVIDER,
|
||||
LEGACY_PANEL_PROVIDER_SUBJECT,
|
||||
),
|
||||
IdentityDirectoryUnavailableError,
|
||||
);
|
||||
|
||||
const token = signToken();
|
||||
const service = new LegacyPanelAuthenticationService(
|
||||
directory,
|
||||
{
|
||||
async isActive() {
|
||||
throw new Error(`must not leak ${token}`);
|
||||
},
|
||||
},
|
||||
SECRET,
|
||||
);
|
||||
await assert.rejects(
|
||||
service.authenticate({ token, platform: 'desktop', nowMs: NOW_MS }),
|
||||
(error) => {
|
||||
assert.ok(error instanceof LegacyPanelAuthenticationUnavailableError);
|
||||
assert.equal(error.message.includes(token), false);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects non-SQLite identity directory repositories', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
new LegacySequelizeIdentityDirectoryRepository({
|
||||
getDialect() {
|
||||
return 'postgres';
|
||||
},
|
||||
}),
|
||||
/SQLite-only/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,396 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const { Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const { runSchemaMigration } = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
runCancellationRequestMigration,
|
||||
} = require('../../back/migrations/0004-run-cancellation-request');
|
||||
const {
|
||||
runAttemptDeadlineMigration,
|
||||
} = require('../../back/migrations/0006-run-attempt-deadline');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeRunRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/runRepository');
|
||||
const {
|
||||
LegacyShadowRunCorrelator,
|
||||
} = require('../../back/runtime/application/legacyShadowRunCorrelator');
|
||||
const {
|
||||
LegacyShadowRunWriter,
|
||||
} = require('../../back/runtime/application/legacyShadowRunWriter');
|
||||
|
||||
const databases = [];
|
||||
let idSequence = 600;
|
||||
let timeSequence = 1_750_000_100_000;
|
||||
|
||||
function nextId() {
|
||||
idSequence += 1;
|
||||
return `019f70f0-0000-7000-8000-${String(idSequence).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
function nextTime() {
|
||||
timeSequence += 10;
|
||||
return timeSequence;
|
||||
}
|
||||
|
||||
async function createRepository() {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
const migrationModel = defineSchemaMigrationModel(database);
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
runCancellationRequestMigration,
|
||||
runAttemptDeadlineMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
databases.push(database);
|
||||
return new LegacySequelizeRunRepository(database);
|
||||
}
|
||||
|
||||
async function createActive(writer, overrides = {}) {
|
||||
const atMs = nextTime();
|
||||
const reference = await writer.accept({
|
||||
origin: 'manual',
|
||||
projectId: 'default',
|
||||
taskId: 'legacy-cron:15',
|
||||
taskRevision: 'sha256:revision',
|
||||
legacyCronId: 15,
|
||||
triggerType: 'manual',
|
||||
acceptedAtMs: atMs,
|
||||
...overrides,
|
||||
});
|
||||
await writer.spawned(reference, {
|
||||
atMs: atMs + 1,
|
||||
...(overrides.pid === undefined ? {} : { pid: overrides.pid }),
|
||||
...(overrides.logArtifactId === undefined
|
||||
? {}
|
||||
: { logArtifactId: overrides.logArtifactId }),
|
||||
});
|
||||
await writer.running(reference, atMs + 2);
|
||||
return { reference, atMs };
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(databases.splice(0).map((database) => database.close()));
|
||||
});
|
||||
|
||||
test('lists bounded active candidates by legacy cron and origin', async () => {
|
||||
const repository = await createRepository();
|
||||
const writer = new LegacyShadowRunWriter(repository, nextId);
|
||||
await createActive(writer, { pid: 11, logArtifactId: 'log-a' });
|
||||
await createActive(writer, { pid: 22, logArtifactId: 'log-b' });
|
||||
await createActive(writer, {
|
||||
origin: 'scheduled_node',
|
||||
pid: 33,
|
||||
logArtifactId: 'log-c',
|
||||
});
|
||||
|
||||
const manual = await repository.listActiveByLegacyCron({
|
||||
legacyCronId: 15,
|
||||
origins: ['manual'],
|
||||
});
|
||||
assert.equal(manual.candidates.length, 2);
|
||||
assert.equal(manual.truncated, false);
|
||||
assert.deepEqual(
|
||||
new Set(manual.candidates.map((candidate) => candidate.pid)),
|
||||
new Set([11, 22]),
|
||||
);
|
||||
|
||||
const bounded = await repository.listActiveByLegacyCron({
|
||||
legacyCronId: 15,
|
||||
origins: ['manual', 'scheduled_node'],
|
||||
limit: 1,
|
||||
});
|
||||
assert.equal(bounded.candidates.length, 1);
|
||||
assert.equal(bounded.truncated, true);
|
||||
await assert.rejects(
|
||||
repository.listActiveByLegacyCron({
|
||||
legacyCronId: 0,
|
||||
origins: ['manual'],
|
||||
}),
|
||||
RangeError,
|
||||
);
|
||||
});
|
||||
|
||||
test('correlates callbacks by log before pid and refuses ambiguity', async () => {
|
||||
const repository = await createRepository();
|
||||
const writer = new LegacyShadowRunWriter(repository, nextId);
|
||||
const first = await createActive(writer, {
|
||||
pid: 41,
|
||||
logArtifactId: 'log-first',
|
||||
});
|
||||
const second = await createActive(writer, {
|
||||
pid: 42,
|
||||
logArtifactId: 'log-second',
|
||||
});
|
||||
const failures = [];
|
||||
const correlator = new LegacyShadowRunCorrelator(repository, writer, {
|
||||
failure: (failure) => failures.push(failure),
|
||||
});
|
||||
|
||||
const ambiguous = await correlator.callback(
|
||||
{
|
||||
legacyCronId: 15,
|
||||
atMs: nextTime(),
|
||||
phase: 'running',
|
||||
},
|
||||
['manual'],
|
||||
);
|
||||
assert.equal(ambiguous.matched, 0);
|
||||
assert.equal(failures[0].reason, 'ambiguous');
|
||||
|
||||
const conflicting = await correlator.callback(
|
||||
{
|
||||
legacyCronId: 15,
|
||||
pid: 41,
|
||||
logArtifactId: 'log-second',
|
||||
atMs: nextTime(),
|
||||
phase: 'finished',
|
||||
exitCode: 0,
|
||||
},
|
||||
['manual'],
|
||||
);
|
||||
assert.equal(conflicting.matched, 0);
|
||||
assert.equal(failures[1].reason, 'ambiguous');
|
||||
|
||||
const finished = await correlator.callback(
|
||||
{
|
||||
legacyCronId: 15,
|
||||
pid: 999,
|
||||
logArtifactId: 'log-second',
|
||||
atMs: nextTime(),
|
||||
phase: 'finished',
|
||||
exitCode: 0,
|
||||
},
|
||||
['manual'],
|
||||
);
|
||||
assert.equal(finished.matched, 1);
|
||||
assert.equal(
|
||||
(await repository.findRunById(second.reference.runId)).status,
|
||||
'succeeded',
|
||||
);
|
||||
assert.equal(
|
||||
(await repository.findRunById(first.reference.runId)).status,
|
||||
'running',
|
||||
);
|
||||
});
|
||||
|
||||
test('cancels one exact candidate or every active candidate', async () => {
|
||||
const repository = await createRepository();
|
||||
const writer = new LegacyShadowRunWriter(repository, nextId);
|
||||
const first = await createActive(writer, { pid: 51 });
|
||||
const second = await createActive(writer, { pid: 52 });
|
||||
const failures = [];
|
||||
const correlator = new LegacyShadowRunCorrelator(repository, writer, {
|
||||
failure: (failure) => failures.push(failure),
|
||||
});
|
||||
|
||||
const one = await correlator.cancel(
|
||||
{
|
||||
legacyCronId: 15,
|
||||
pid: 51,
|
||||
atMs: nextTime(),
|
||||
scope: 'one',
|
||||
reason: 'user',
|
||||
},
|
||||
['manual'],
|
||||
);
|
||||
assert.equal(one.matched, 1);
|
||||
assert.equal(
|
||||
(await repository.findRunById(first.reference.runId)).status,
|
||||
'cancelled',
|
||||
);
|
||||
assert.equal(
|
||||
(await repository.findRunById(second.reference.runId)).status,
|
||||
'running',
|
||||
);
|
||||
|
||||
const all = await correlator.cancel(
|
||||
{
|
||||
legacyCronId: 15,
|
||||
atMs: nextTime(),
|
||||
scope: 'all',
|
||||
reason: 'policy',
|
||||
},
|
||||
['manual'],
|
||||
);
|
||||
assert.equal(all.matched, 1);
|
||||
assert.equal(
|
||||
(await repository.findRunById(second.reference.runId)).status,
|
||||
'cancelled',
|
||||
);
|
||||
assert.deepEqual(failures, []);
|
||||
});
|
||||
|
||||
test('recovers an out-of-order finish and ignores duplicate terminal callbacks', async () => {
|
||||
const repository = await createRepository();
|
||||
const writer = new LegacyShadowRunWriter(repository, nextId);
|
||||
const atMs = nextTime();
|
||||
const reference = await writer.accept({
|
||||
origin: 'manual',
|
||||
projectId: 'default',
|
||||
taskId: 'legacy-cron:16',
|
||||
taskRevision: 'sha256:revision',
|
||||
legacyCronId: 16,
|
||||
triggerType: 'manual',
|
||||
acceptedAtMs: atMs,
|
||||
});
|
||||
const failures = [];
|
||||
const correlator = new LegacyShadowRunCorrelator(repository, writer, {
|
||||
failure: (failure) => failures.push(failure),
|
||||
});
|
||||
|
||||
const first = await correlator.callback(
|
||||
{
|
||||
legacyCronId: 16,
|
||||
pid: 160,
|
||||
logArtifactId: 'log-16',
|
||||
atMs: atMs + 10,
|
||||
phase: 'finished',
|
||||
exitCode: 0,
|
||||
},
|
||||
['manual'],
|
||||
);
|
||||
assert.equal(first.matched, 1);
|
||||
assert.equal(
|
||||
(await repository.findRunById(reference.runId)).status,
|
||||
'succeeded',
|
||||
);
|
||||
const events = await repository.listEvents(reference.runId);
|
||||
assert.deepEqual(
|
||||
events.map((event) => event.type),
|
||||
[
|
||||
'run.created',
|
||||
'run.queued',
|
||||
'run.dispatching',
|
||||
'attempt.starting',
|
||||
'attempt.running',
|
||||
'run.running',
|
||||
'attempt.succeeded',
|
||||
'run.succeeded',
|
||||
],
|
||||
);
|
||||
|
||||
const duplicate = await correlator.callback(
|
||||
{
|
||||
legacyCronId: 16,
|
||||
pid: 160,
|
||||
logArtifactId: 'log-16',
|
||||
atMs: atMs + 20,
|
||||
phase: 'finished',
|
||||
exitCode: 0,
|
||||
},
|
||||
['manual'],
|
||||
);
|
||||
assert.equal(duplicate.matched, 0);
|
||||
assert.equal(failures[0].reason, 'unmatched');
|
||||
assert.equal((await repository.listEvents(reference.runId)).length, 8);
|
||||
});
|
||||
|
||||
test('keeps cancellation terminal when a late successful callback arrives', async () => {
|
||||
const repository = await createRepository();
|
||||
const writer = new LegacyShadowRunWriter(repository, nextId);
|
||||
const active = await createActive(writer, { pid: 170 });
|
||||
const failures = [];
|
||||
const correlator = new LegacyShadowRunCorrelator(repository, writer, {
|
||||
failure: (failure) => failures.push(failure),
|
||||
});
|
||||
|
||||
await correlator.cancel(
|
||||
{
|
||||
legacyCronId: 15,
|
||||
pid: 170,
|
||||
atMs: nextTime(),
|
||||
scope: 'one',
|
||||
reason: 'user',
|
||||
},
|
||||
['manual'],
|
||||
);
|
||||
const eventCount = (await repository.listEvents(active.reference.runId))
|
||||
.length;
|
||||
const late = await correlator.callback(
|
||||
{
|
||||
legacyCronId: 15,
|
||||
pid: 170,
|
||||
atMs: nextTime(),
|
||||
phase: 'finished',
|
||||
exitCode: 0,
|
||||
},
|
||||
['manual'],
|
||||
);
|
||||
|
||||
assert.equal(late.matched, 0);
|
||||
assert.equal(
|
||||
(await repository.findRunById(active.reference.runId)).status,
|
||||
'cancelled',
|
||||
);
|
||||
assert.equal(
|
||||
(await repository.listEvents(active.reference.runId)).length,
|
||||
eventCount,
|
||||
);
|
||||
assert.equal(failures[0].reason, 'unmatched');
|
||||
});
|
||||
|
||||
test('continues cancel-all correlation after an individual shadow write fails', async () => {
|
||||
const writes = [];
|
||||
const failures = [];
|
||||
const locator = {
|
||||
async listActiveByLegacyCron() {
|
||||
return {
|
||||
truncated: false,
|
||||
candidates: [
|
||||
{
|
||||
runId: 'run-a',
|
||||
attemptId: 'attempt-a',
|
||||
origin: 'manual',
|
||||
runStatus: 'running',
|
||||
attemptStatus: 'running',
|
||||
createdAtMs: 1,
|
||||
},
|
||||
{
|
||||
runId: 'run-b',
|
||||
attemptId: 'attempt-b',
|
||||
origin: 'manual',
|
||||
runStatus: 'running',
|
||||
attemptStatus: 'running',
|
||||
createdAtMs: 2,
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
};
|
||||
const writer = {
|
||||
async cancelled(reference) {
|
||||
writes.push(reference.runId);
|
||||
if (reference.runId === 'run-a') throw new Error('write failed');
|
||||
},
|
||||
};
|
||||
const correlator = new LegacyShadowRunCorrelator(locator, writer, {
|
||||
failure: (failure) => failures.push(failure),
|
||||
});
|
||||
|
||||
const result = await correlator.cancel(
|
||||
{
|
||||
legacyCronId: 15,
|
||||
atMs: 4,
|
||||
scope: 'all',
|
||||
reason: 'user',
|
||||
},
|
||||
['manual'],
|
||||
);
|
||||
assert.deepEqual(writes, ['run-a', 'run-b']);
|
||||
assert.equal(result.matched, 1);
|
||||
assert.equal(failures[0].reason, 'write_failed');
|
||||
});
|
||||
@@ -0,0 +1,292 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const { Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const { runSchemaMigration } = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
runCancellationRequestMigration,
|
||||
} = require('../../back/migrations/0004-run-cancellation-request');
|
||||
const {
|
||||
runAttemptDeadlineMigration,
|
||||
} = require('../../back/migrations/0006-run-attempt-deadline');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeRunRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/runRepository');
|
||||
const {
|
||||
LegacyShadowRunObserver,
|
||||
} = require('../../back/runtime/application/legacyShadowRunObserver');
|
||||
const {
|
||||
LegacyShadowRunWriter,
|
||||
} = require('../../back/runtime/application/legacyShadowRunWriter');
|
||||
const {
|
||||
RuntimeRolloutPolicy,
|
||||
} = require('../../back/runtime/domain/runtimeRollout');
|
||||
|
||||
const databases = [];
|
||||
const ACCEPTED_AT_MS = 1_750_000_000_000;
|
||||
let idSequence = 300;
|
||||
|
||||
function nextId() {
|
||||
idSequence += 1;
|
||||
return `019f70e0-0000-7000-8000-${String(idSequence).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
async function createRepository() {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
const migrationModel = defineSchemaMigrationModel(database);
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
runCancellationRequestMigration,
|
||||
runAttemptDeadlineMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
databases.push(database);
|
||||
return new LegacySequelizeRunRepository(database);
|
||||
}
|
||||
|
||||
function acceptedFact(overrides = {}) {
|
||||
return {
|
||||
origin: 'manual',
|
||||
projectId: 'default',
|
||||
taskId: 'legacy-cron:7',
|
||||
taskRevision: 'sha256:revision',
|
||||
taskName: 'shadow test',
|
||||
legacyCronId: 7,
|
||||
triggerType: 'manual',
|
||||
triggeredBy: 'legacy:manual',
|
||||
acceptedAtMs: ACCEPTED_AT_MS,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function policy(mode = 'shadow') {
|
||||
return new RuntimeRolloutPolicy({
|
||||
defaultMode: 'off',
|
||||
origins: { manual: mode },
|
||||
allowLegacyFallbackBeforeStart: false,
|
||||
});
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(databases.splice(0).map((database) => database.close()));
|
||||
});
|
||||
|
||||
test('persists an accepted legacy execution as one queued shadow aggregate', async () => {
|
||||
const repository = await createRepository();
|
||||
const writer = new LegacyShadowRunWriter(repository, nextId);
|
||||
const reference = await writer.accept(acceptedFact());
|
||||
const run = await repository.findRunById(reference.runId);
|
||||
const attempt = await repository.findAttemptById(reference.attemptId);
|
||||
const events = await repository.listEvents(reference.runId);
|
||||
|
||||
assert.equal(run.status, 'queued');
|
||||
assert.equal(run.version, 2);
|
||||
assert.equal(run.eventSequence, 2);
|
||||
assert.equal(run.executionOwner, 'legacy');
|
||||
assert.equal(attempt.status, 'claimed');
|
||||
assert.deepEqual(
|
||||
events.map((event) => [event.sequence, event.type]),
|
||||
[
|
||||
[1, 'run.created'],
|
||||
[2, 'run.queued'],
|
||||
],
|
||||
);
|
||||
assert.equal(
|
||||
events.every((event) => event.payload.shadow === true),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('serializes a successful observed process lifecycle without starting it', async () => {
|
||||
const repository = await createRepository();
|
||||
const references = [];
|
||||
const writer = new LegacyShadowRunWriter(repository, nextId);
|
||||
const originalAccept = writer.accept.bind(writer);
|
||||
writer.accept = async (fact) => {
|
||||
const reference = await originalAccept(fact);
|
||||
references.push(reference);
|
||||
return reference;
|
||||
};
|
||||
const failures = [];
|
||||
const observer = new LegacyShadowRunObserver(policy(), writer, {
|
||||
failure: (failure) => failures.push(failure),
|
||||
});
|
||||
const observation = observer.begin(acceptedFact());
|
||||
observation.spawned({
|
||||
atMs: ACCEPTED_AT_MS + 1,
|
||||
pid: 4242,
|
||||
executorHandle: 'legacy-local:4242',
|
||||
logArtifactId: 'legacy-log:1234567890123456789012345',
|
||||
});
|
||||
observation.running({ atMs: ACCEPTED_AT_MS + 1 });
|
||||
observation.exited({ atMs: ACCEPTED_AT_MS + 2, exitCode: 0 });
|
||||
await observation.settled();
|
||||
|
||||
assert.deepEqual(failures, []);
|
||||
const reference = references[0];
|
||||
const run = await repository.findRunById(reference.runId);
|
||||
const attempt = await repository.findAttemptById(reference.attemptId);
|
||||
const events = await repository.listEvents(reference.runId);
|
||||
|
||||
assert.equal(run.status, 'succeeded');
|
||||
assert.equal(run.version, 8);
|
||||
assert.equal(run.eventSequence, 8);
|
||||
assert.equal(attempt.status, 'succeeded');
|
||||
assert.equal(attempt.pid, 4242);
|
||||
assert.equal(attempt.executorHandle, 'legacy-local:4242');
|
||||
assert.equal(attempt.logArtifactId, 'legacy-log:1234567890123456789012345');
|
||||
assert.equal(attempt.startedAtMs, ACCEPTED_AT_MS + 1);
|
||||
assert.equal(attempt.finishedAtMs, ACCEPTED_AT_MS + 2);
|
||||
assert.deepEqual(
|
||||
events.map((event) => event.type),
|
||||
[
|
||||
'run.created',
|
||||
'run.queued',
|
||||
'run.dispatching',
|
||||
'attempt.starting',
|
||||
'attempt.running',
|
||||
'run.running',
|
||||
'attempt.succeeded',
|
||||
'run.succeeded',
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('maps non-zero exits and start errors to stable failed terminal states', async () => {
|
||||
for (const scenario of ['exit', 'start_error']) {
|
||||
const repository = await createRepository();
|
||||
const references = [];
|
||||
const writer = new LegacyShadowRunWriter(repository, nextId);
|
||||
const originalAccept = writer.accept.bind(writer);
|
||||
writer.accept = async (fact) => {
|
||||
const reference = await originalAccept(fact);
|
||||
references.push(reference);
|
||||
return reference;
|
||||
};
|
||||
const observer = new LegacyShadowRunObserver(policy(), writer, {
|
||||
failure() {},
|
||||
});
|
||||
const observation = observer.begin(
|
||||
acceptedFact({ taskId: `case:${scenario}` }),
|
||||
);
|
||||
if (scenario === 'exit') {
|
||||
observation.spawned({ atMs: ACCEPTED_AT_MS + 1, pid: 12 });
|
||||
observation.running({ atMs: ACCEPTED_AT_MS + 1 });
|
||||
observation.exited({ atMs: ACCEPTED_AT_MS + 2, exitCode: 9 });
|
||||
} else {
|
||||
observation.startFailed({
|
||||
atMs: ACCEPTED_AT_MS + 1,
|
||||
errorCode: 'LEGACY_PROCESS_ERROR',
|
||||
});
|
||||
}
|
||||
await observation.settled();
|
||||
|
||||
const reference = references[0];
|
||||
const run = await repository.findRunById(reference.runId);
|
||||
const attempt = await repository.findAttemptById(reference.attemptId);
|
||||
assert.equal(run.status, 'failed');
|
||||
assert.equal(attempt.status, 'failed');
|
||||
if (scenario === 'exit') {
|
||||
assert.equal(run.errorCode, 'LEGACY_EXIT_NON_ZERO');
|
||||
assert.equal(attempt.exitCode, 9);
|
||||
} else {
|
||||
assert.equal(run.errorCode, 'LEGACY_PROCESS_ERROR');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('is default-off, fail-open, and rejects primary ownership', async () => {
|
||||
let writes = 0;
|
||||
const failures = [];
|
||||
const writer = {
|
||||
async accept() {
|
||||
writes += 1;
|
||||
const error = new Error('database unavailable');
|
||||
error.code = 'SQLITE_BUSY';
|
||||
throw error;
|
||||
},
|
||||
};
|
||||
const off = new LegacyShadowRunObserver(policy('off'), writer, {
|
||||
failure: (failure) => failures.push(failure),
|
||||
});
|
||||
const noOp = off.begin(acceptedFact());
|
||||
noOp.spawned({ atMs: ACCEPTED_AT_MS + 1 });
|
||||
await noOp.settled();
|
||||
assert.equal(writes, 0);
|
||||
|
||||
const shadow = new LegacyShadowRunObserver(policy(), writer, {
|
||||
failure: (failure) => failures.push(failure),
|
||||
});
|
||||
const failed = shadow.begin(acceptedFact());
|
||||
failed.spawned({ atMs: ACCEPTED_AT_MS + 1 });
|
||||
failed.exited({ atMs: ACCEPTED_AT_MS + 2, exitCode: 0 });
|
||||
await failed.settled();
|
||||
assert.equal(writes, 1);
|
||||
assert.deepEqual(failures, [
|
||||
{
|
||||
origin: 'manual',
|
||||
operation: 'accept',
|
||||
errorCode: 'SQLITE_BUSY',
|
||||
},
|
||||
]);
|
||||
|
||||
const primary = new LegacyShadowRunObserver(policy('primary'), writer, {
|
||||
failure() {},
|
||||
});
|
||||
assert.throws(() => primary.begin(acceptedFact()), /primary execution/);
|
||||
});
|
||||
|
||||
test('reports an individual shadow write failure and continues later facts', async () => {
|
||||
const calls = [];
|
||||
const failures = [];
|
||||
const writer = {
|
||||
async accept() {
|
||||
calls.push('accept');
|
||||
return { runId: 'run-1', attemptId: 'attempt-1' };
|
||||
},
|
||||
async spawned() {
|
||||
calls.push('spawned');
|
||||
const error = new Error('first write lost');
|
||||
error.code = 'RUN_VERSION_CONFLICT';
|
||||
throw error;
|
||||
},
|
||||
async running() {
|
||||
calls.push('running');
|
||||
},
|
||||
async exited() {
|
||||
calls.push('exited');
|
||||
},
|
||||
};
|
||||
const observer = new LegacyShadowRunObserver(policy(), writer, {
|
||||
failure: (failure) => failures.push(failure),
|
||||
});
|
||||
const observation = observer.begin(acceptedFact());
|
||||
observation.spawned({ atMs: ACCEPTED_AT_MS + 1 });
|
||||
observation.running({ atMs: ACCEPTED_AT_MS + 2 });
|
||||
observation.exited({ atMs: ACCEPTED_AT_MS + 3, exitCode: 0 });
|
||||
await observation.settled();
|
||||
|
||||
assert.deepEqual(calls, ['accept', 'spawned', 'running', 'exited']);
|
||||
assert.deepEqual(failures, [
|
||||
{
|
||||
origin: 'manual',
|
||||
operation: 'spawned',
|
||||
errorCode: 'RUN_VERSION_CONFLICT',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
},
|
||||
]);
|
||||
});
|
||||
@@ -0,0 +1,60 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
createLegacyLogArtifactId,
|
||||
createLegacyTaskRevision,
|
||||
} = require('../../back/runtime/compatibility/legacyTaskRevision');
|
||||
|
||||
test('creates a deterministic opaque revision from execution-affecting fields', () => {
|
||||
const input = {
|
||||
command: 'node script.js --token secret-value',
|
||||
schedule: '0 * * * *',
|
||||
extraSchedules: ['30 * * * *'],
|
||||
taskBefore: 'prepare',
|
||||
taskAfter: 'cleanup',
|
||||
workDirectory: '/ql/scripts',
|
||||
logName: 'script',
|
||||
environmentRevision: 'env-v1',
|
||||
sourceRevision: 'git:abc123',
|
||||
};
|
||||
const first = createLegacyTaskRevision(input);
|
||||
const second = createLegacyTaskRevision(structuredClone(input));
|
||||
|
||||
assert.equal(first, second);
|
||||
assert.match(first, /^sha256:[a-f0-9]{64}$/);
|
||||
assert.equal(first.includes('secret-value'), false);
|
||||
assert.notEqual(
|
||||
first,
|
||||
createLegacyTaskRevision({ ...input, schedule: '1 * * * *' }),
|
||||
);
|
||||
assert.notEqual(
|
||||
first,
|
||||
createLegacyTaskRevision({ ...input, command: 'node other.js' }),
|
||||
);
|
||||
});
|
||||
|
||||
test('preserves extra schedule order as part of the task snapshot', () => {
|
||||
const left = createLegacyTaskRevision({
|
||||
command: 'task',
|
||||
extraSchedules: ['a', 'b'],
|
||||
});
|
||||
const right = createLegacyTaskRevision({
|
||||
command: 'task',
|
||||
extraSchedules: ['b', 'a'],
|
||||
});
|
||||
|
||||
assert.notEqual(left, right);
|
||||
});
|
||||
|
||||
test('creates a bounded opaque artifact id for arbitrary legacy log paths', () => {
|
||||
const longPath = `custom/${'nested/'.repeat(100)}secret-task.log`;
|
||||
const artifactId = createLegacyLogArtifactId(longPath);
|
||||
|
||||
assert.match(artifactId, /^legacy-log:[a-f0-9]{25}$/);
|
||||
assert.equal(artifactId.length <= 36, true);
|
||||
assert.equal(artifactId.includes('secret-task'), false);
|
||||
assert.equal(createLegacyLogArtifactId(longPath), artifactId);
|
||||
assert.notEqual(createLegacyLogArtifactId(`${longPath}.1`), artifactId);
|
||||
});
|
||||
@@ -0,0 +1,607 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const { Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
runSchemaMigration,
|
||||
} = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
LOCAL_ARTIFACT_RETENTION_TABLE,
|
||||
localArtifactRetentionMigration,
|
||||
} = require('../../back/migrations/0015-local-artifact-retention');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LocalArtifactByteRangeReader,
|
||||
UnsafeLocalArtifactReadTargetError,
|
||||
} = require('../../back/runtime/adapters/fs/localArtifactByteRangeReader');
|
||||
const {
|
||||
LocalArtifactTruncationFactStore,
|
||||
localArtifactTruncationFactFileName,
|
||||
} = require('../../back/runtime/adapters/fs/localArtifactTruncationFactStore');
|
||||
const {
|
||||
LegacySequelizeLocalArtifactReadMetadataRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/localArtifactReadMetadataRepository');
|
||||
const {
|
||||
LocalArtifactReadEvidenceConflictError,
|
||||
LocalArtifactReadService,
|
||||
} = require('../../back/runtime/application/localArtifactReadService');
|
||||
const {
|
||||
MAX_LOCAL_ARTIFACT_READ_BYTES,
|
||||
} = require('../../back/runtime/domain/artifactRead');
|
||||
const {
|
||||
encodeLocalArtifactTruncationFact,
|
||||
} = require('../../back/runtime/domain/localArtifactTruncation');
|
||||
|
||||
const RUN_ID = '019f7600-0000-7000-8000-000000000001';
|
||||
const ATTEMPT_ID = '019f7600-0000-7000-8000-000000000002';
|
||||
const LOG_ARTIFACT_ID = `local-${'e'.repeat(30)}`;
|
||||
const PROJECT_ID = 'project-a';
|
||||
const SUBJECT = Object.freeze({ type: 'user', id: 'user-1' });
|
||||
const RANGE = Object.freeze({ offset: 2, length: 4 });
|
||||
const METADATA = Object.freeze({
|
||||
projectId: PROJECT_ID,
|
||||
runId: RUN_ID,
|
||||
attemptId: ATTEMPT_ID,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
});
|
||||
|
||||
function request(overrides = {}) {
|
||||
return {
|
||||
subject: SUBJECT,
|
||||
projectId: PROJECT_ID,
|
||||
runId: RUN_ID,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
range: RANGE,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function temporaryArtifactRoot(t) {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-read-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
await fs.chmod(root, 0o700);
|
||||
return root;
|
||||
}
|
||||
|
||||
async function writeArtifact(root, value = '0123456789') {
|
||||
const directory = path.join(root, LOG_ARTIFACT_ID.slice(6, 8));
|
||||
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
const target = path.join(directory, `${LOG_ARTIFACT_ID}.log`);
|
||||
await fs.writeFile(target, value, { mode: 0o600 });
|
||||
return { directory, target };
|
||||
}
|
||||
|
||||
function fact(quotaReached) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
runId: RUN_ID,
|
||||
attemptId: ATTEMPT_ID,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
maximumBytes: 64 * 1024,
|
||||
quotaReached,
|
||||
observedAtMs: 1_800_000_000_000,
|
||||
};
|
||||
}
|
||||
|
||||
function service(overrides = {}) {
|
||||
return new LocalArtifactReadService(
|
||||
overrides.metadata ?? {
|
||||
async find() {
|
||||
return METADATA;
|
||||
},
|
||||
},
|
||||
overrides.authorizer ?? {
|
||||
async authorize() {
|
||||
return 'allow';
|
||||
},
|
||||
},
|
||||
overrides.bytes ?? {
|
||||
async read() {
|
||||
return {
|
||||
status: 'available',
|
||||
content: Buffer.from('2345'),
|
||||
start: 2,
|
||||
endExclusive: 6,
|
||||
totalBytes: 10,
|
||||
nextOffset: 6,
|
||||
};
|
||||
},
|
||||
},
|
||||
overrides.facts ?? {
|
||||
async read() {
|
||||
return null;
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
test('validates bounded range before metadata or policy side effects', async () => {
|
||||
const calls = [];
|
||||
const reader = service({
|
||||
metadata: {
|
||||
async find() {
|
||||
calls.push('metadata');
|
||||
return METADATA;
|
||||
},
|
||||
},
|
||||
authorizer: {
|
||||
async authorize() {
|
||||
calls.push('authorize');
|
||||
return 'allow';
|
||||
},
|
||||
},
|
||||
bytes: {
|
||||
async read() {
|
||||
calls.push('bytes');
|
||||
return { status: 'missing' };
|
||||
},
|
||||
},
|
||||
facts: {
|
||||
async read() {
|
||||
calls.push('facts');
|
||||
return null;
|
||||
},
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
reader.read(
|
||||
request({
|
||||
range: { offset: 0, length: MAX_LOCAL_ARTIFACT_READ_BYTES + 1 },
|
||||
}),
|
||||
),
|
||||
/length is invalid/,
|
||||
);
|
||||
assert.deepEqual(calls, []);
|
||||
});
|
||||
|
||||
test('never touches file or truncation evidence before artifact.read allows it', async () => {
|
||||
const calls = [];
|
||||
const reader = service({
|
||||
metadata: {
|
||||
async find(input) {
|
||||
calls.push(['metadata', input]);
|
||||
return METADATA;
|
||||
},
|
||||
},
|
||||
authorizer: {
|
||||
async authorize(input) {
|
||||
calls.push(['authorize', input]);
|
||||
return 'deny';
|
||||
},
|
||||
},
|
||||
bytes: {
|
||||
async read() {
|
||||
calls.push(['bytes']);
|
||||
throw new Error('must not read');
|
||||
},
|
||||
},
|
||||
facts: {
|
||||
async read() {
|
||||
calls.push(['facts']);
|
||||
throw new Error('must not read');
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.deepEqual(await reader.read(request()), {
|
||||
status: 'forbidden',
|
||||
effect: 'deny',
|
||||
});
|
||||
assert.equal(calls.length, 2);
|
||||
assert.equal(calls[0][0], 'metadata');
|
||||
assert.deepEqual(calls[1], [
|
||||
'authorize',
|
||||
{
|
||||
action: 'artifact.read',
|
||||
subject: SUBJECT,
|
||||
projectId: PROJECT_ID,
|
||||
runId: RUN_ID,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('returns not_found without policy or file probes for an unbound Artifact', async () => {
|
||||
const calls = [];
|
||||
const reader = service({
|
||||
metadata: {
|
||||
async find() {
|
||||
calls.push('metadata');
|
||||
return null;
|
||||
},
|
||||
},
|
||||
authorizer: {
|
||||
async authorize() {
|
||||
calls.push('authorize');
|
||||
return 'allow';
|
||||
},
|
||||
},
|
||||
bytes: {
|
||||
async read() {
|
||||
calls.push('bytes');
|
||||
return { status: 'missing' };
|
||||
},
|
||||
},
|
||||
facts: {
|
||||
async read() {
|
||||
calls.push('facts');
|
||||
return null;
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.deepEqual(await reader.read(request()), { status: 'not_found' });
|
||||
assert.deepEqual(calls, ['metadata']);
|
||||
});
|
||||
|
||||
test('reads a bounded file snapshot and preserves true, false and unknown truncation', async (t) => {
|
||||
const root = await temporaryArtifactRoot(t);
|
||||
const { directory } = await writeArtifact(root);
|
||||
const bytes = new LocalArtifactByteRangeReader(root);
|
||||
const facts = new LocalArtifactTruncationFactStore(root);
|
||||
const factTarget = path.join(
|
||||
directory,
|
||||
localArtifactTruncationFactFileName(LOG_ARTIFACT_ID),
|
||||
);
|
||||
|
||||
const reader = service({ bytes, facts });
|
||||
const unknown = await reader.read(request());
|
||||
assert.equal(unknown.status, 'available');
|
||||
assert.equal(unknown.content.toString(), '2345');
|
||||
assert.deepEqual(
|
||||
{
|
||||
start: unknown.start,
|
||||
endExclusive: unknown.endExclusive,
|
||||
totalBytes: unknown.totalBytes,
|
||||
nextOffset: unknown.nextOffset,
|
||||
truncation: unknown.truncation,
|
||||
},
|
||||
{
|
||||
start: 2,
|
||||
endExclusive: 6,
|
||||
totalBytes: 10,
|
||||
nextOffset: 6,
|
||||
truncation: { truncated: 'unknown' },
|
||||
},
|
||||
);
|
||||
|
||||
for (const quotaReached of [false, true]) {
|
||||
await fs.writeFile(
|
||||
factTarget,
|
||||
encodeLocalArtifactTruncationFact(fact(quotaReached)),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
const result = await reader.read(request());
|
||||
assert.deepEqual(result.truncation, {
|
||||
truncated: quotaReached,
|
||||
maximumBytes: 64 * 1024,
|
||||
observedAtMs: 1_800_000_000_000,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test('returns retained without touching files and resolves an ENOENT retirement race', async () => {
|
||||
const retention = Object.freeze({
|
||||
disposition: 'deleted',
|
||||
finishedAtMs: 100,
|
||||
eligibleAtMs: 200,
|
||||
bytesReclaimed: 10,
|
||||
recordedAtMs: 300,
|
||||
});
|
||||
let fileReads = 0;
|
||||
let factReads = 0;
|
||||
const alreadyRetained = service({
|
||||
metadata: {
|
||||
async find() {
|
||||
return { ...METADATA, retention };
|
||||
},
|
||||
},
|
||||
bytes: {
|
||||
async read() {
|
||||
fileReads += 1;
|
||||
return { status: 'missing' };
|
||||
},
|
||||
},
|
||||
facts: {
|
||||
async read() {
|
||||
factReads += 1;
|
||||
return null;
|
||||
},
|
||||
},
|
||||
});
|
||||
const retained = await alreadyRetained.read(request());
|
||||
assert.equal(retained.status, 'retained');
|
||||
assert.deepEqual(retained.retention, retention);
|
||||
assert.deepEqual(retained.truncation, { truncated: 'unknown' });
|
||||
assert.equal(fileReads, 0);
|
||||
assert.equal(factReads, 0);
|
||||
|
||||
let lookups = 0;
|
||||
const raced = service({
|
||||
metadata: {
|
||||
async find() {
|
||||
lookups += 1;
|
||||
return lookups === 1 ? METADATA : { ...METADATA, retention };
|
||||
},
|
||||
},
|
||||
bytes: {
|
||||
async read() {
|
||||
fileReads += 1;
|
||||
return { status: 'missing' };
|
||||
},
|
||||
},
|
||||
facts: {
|
||||
async read() {
|
||||
factReads += 1;
|
||||
return null;
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.equal((await raced.read(request())).status, 'retained');
|
||||
assert.equal(lookups, 2);
|
||||
assert.equal(fileReads, 1);
|
||||
assert.equal(factReads, 0);
|
||||
});
|
||||
|
||||
test('distinguishes unexplained missing content and rejects drifted fact identity', async () => {
|
||||
const missing = service({
|
||||
bytes: {
|
||||
async read() {
|
||||
return { status: 'missing' };
|
||||
},
|
||||
},
|
||||
facts: {
|
||||
async read() {
|
||||
return fact(true);
|
||||
},
|
||||
},
|
||||
});
|
||||
const result = await missing.read(request());
|
||||
assert.equal(result.status, 'missing');
|
||||
assert.deepEqual(result.truncation, {
|
||||
truncated: true,
|
||||
maximumBytes: 64 * 1024,
|
||||
observedAtMs: 1_800_000_000_000,
|
||||
});
|
||||
|
||||
const drifted = service({
|
||||
facts: {
|
||||
async read() {
|
||||
return {
|
||||
...fact(false),
|
||||
attemptId: '019f7600-0000-7000-8000-000000000099',
|
||||
};
|
||||
},
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
drifted.read(request()),
|
||||
LocalArtifactReadEvidenceConflictError,
|
||||
);
|
||||
});
|
||||
|
||||
test('file reader refuses symlink files and shard escapes', async (t) => {
|
||||
const root = await temporaryArtifactRoot(t);
|
||||
const directory = path.join(root, LOG_ARTIFACT_ID.slice(6, 8));
|
||||
await fs.mkdir(directory, { mode: 0o700 });
|
||||
const outside = path.join(root, 'outside.log');
|
||||
await fs.writeFile(outside, 'outside');
|
||||
const target = path.join(directory, `${LOG_ARTIFACT_ID}.log`);
|
||||
await fs.symlink(outside, target);
|
||||
const reader = new LocalArtifactByteRangeReader(root);
|
||||
await assert.rejects(
|
||||
reader.read(LOG_ARTIFACT_ID, RANGE),
|
||||
UnsafeLocalArtifactReadTargetError,
|
||||
);
|
||||
await fs.unlink(target);
|
||||
await fs.rmdir(directory);
|
||||
await fs.symlink(path.dirname(outside), directory);
|
||||
await assert.rejects(
|
||||
reader.read(LOG_ARTIFACT_ID, RANGE),
|
||||
UnsafeLocalArtifactReadTargetError,
|
||||
);
|
||||
});
|
||||
|
||||
async function setupDatabase(t) {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [runSchemaMigration, localArtifactRetentionMigration],
|
||||
logger: { info() {} },
|
||||
});
|
||||
return database;
|
||||
}
|
||||
|
||||
async function seedMetadata(database, overrides = {}) {
|
||||
const values = {
|
||||
projectId: PROJECT_ID,
|
||||
runId: RUN_ID,
|
||||
attemptId: ATTEMPT_ID,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
executionOwner: 'runtime',
|
||||
executorType: 'local_process',
|
||||
...overrides,
|
||||
};
|
||||
const query = database.getQueryInterface();
|
||||
await query.bulkInsert(RUN_TABLE, [
|
||||
{
|
||||
id: values.runId,
|
||||
project_id: values.projectId,
|
||||
task_id: 'task-1',
|
||||
task_revision: 'revision-1',
|
||||
trigger_type: 'manual',
|
||||
execution_origin: 'manual',
|
||||
execution_owner: values.executionOwner,
|
||||
status: 'succeeded',
|
||||
version: 1,
|
||||
event_sequence: 1,
|
||||
priority: 0,
|
||||
created_at_ms: 1,
|
||||
finished_at_ms: 100,
|
||||
},
|
||||
]);
|
||||
await query.bulkInsert(RUN_ATTEMPT_TABLE, [
|
||||
{
|
||||
id: values.attemptId,
|
||||
run_id: values.runId,
|
||||
attempt: 1,
|
||||
status: 'succeeded',
|
||||
executor_type: values.executorType,
|
||||
log_artifact_id: values.logArtifactId,
|
||||
callback_sequence: 0,
|
||||
created_at_ms: 1,
|
||||
finished_at_ms: 100,
|
||||
},
|
||||
]);
|
||||
return values;
|
||||
}
|
||||
|
||||
test('SQLite metadata binds project, runtime owner, local executor and tombstone', async (t) => {
|
||||
const database = await setupDatabase(t);
|
||||
const values = await seedMetadata(database);
|
||||
const repository = new LegacySequelizeLocalArtifactReadMetadataRepository(
|
||||
database,
|
||||
);
|
||||
assert.equal(
|
||||
await repository.find({
|
||||
projectId: 'other-project',
|
||||
runId: RUN_ID,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
}),
|
||||
null,
|
||||
);
|
||||
assert.deepEqual(
|
||||
await repository.find({
|
||||
projectId: PROJECT_ID,
|
||||
runId: RUN_ID,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
}),
|
||||
METADATA,
|
||||
);
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.bulkInsert(LOCAL_ARTIFACT_RETENTION_TABLE, [
|
||||
{
|
||||
attempt_id: values.attemptId,
|
||||
log_artifact_id: values.logArtifactId,
|
||||
finished_at_ms: 100,
|
||||
eligible_at_ms: 200,
|
||||
disposition: 'deleted',
|
||||
bytes_reclaimed: 10,
|
||||
recorded_at_ms: 300,
|
||||
},
|
||||
]);
|
||||
const retained = await repository.find({
|
||||
projectId: PROJECT_ID,
|
||||
runId: RUN_ID,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
});
|
||||
assert.deepEqual(retained.retention, {
|
||||
disposition: 'deleted',
|
||||
finishedAtMs: 100,
|
||||
eligibleAtMs: 200,
|
||||
bytesReclaimed: 10,
|
||||
recordedAtMs: 300,
|
||||
});
|
||||
});
|
||||
|
||||
test('SQLite metadata excludes legacy owner and non-local executor', async (t) => {
|
||||
const legacyDatabase = await setupDatabase(t);
|
||||
await seedMetadata(legacyDatabase, { executionOwner: 'legacy' });
|
||||
const legacy = new LegacySequelizeLocalArtifactReadMetadataRepository(
|
||||
legacyDatabase,
|
||||
);
|
||||
assert.equal(
|
||||
await legacy.find({
|
||||
projectId: PROJECT_ID,
|
||||
runId: RUN_ID,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
}),
|
||||
null,
|
||||
);
|
||||
|
||||
const remoteDatabase = await setupDatabase(t);
|
||||
await seedMetadata(remoteDatabase, { executorType: 'remote_worker' });
|
||||
const remote = new LegacySequelizeLocalArtifactReadMetadataRepository(
|
||||
remoteDatabase,
|
||||
);
|
||||
assert.equal(
|
||||
await remote.find({
|
||||
projectId: PROJECT_ID,
|
||||
runId: RUN_ID,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
}),
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
test('SQLite metadata rejects a tombstone bound to a different Artifact', async (t) => {
|
||||
const database = await setupDatabase(t);
|
||||
const values = await seedMetadata(database);
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.bulkInsert(LOCAL_ARTIFACT_RETENTION_TABLE, [
|
||||
{
|
||||
attempt_id: values.attemptId,
|
||||
log_artifact_id: `local-${'f'.repeat(30)}`,
|
||||
finished_at_ms: 100,
|
||||
eligible_at_ms: 200,
|
||||
disposition: 'already_absent',
|
||||
bytes_reclaimed: 0,
|
||||
recorded_at_ms: 300,
|
||||
},
|
||||
]);
|
||||
const repository = new LegacySequelizeLocalArtifactReadMetadataRepository(
|
||||
database,
|
||||
);
|
||||
await assert.rejects(
|
||||
repository.find({
|
||||
projectId: PROJECT_ID,
|
||||
runId: RUN_ID,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
}),
|
||||
/metadata is corrupt or ambiguous/,
|
||||
);
|
||||
});
|
||||
|
||||
test('SQLite metadata rejects a tombstone with drifted Attempt completion time', async (t) => {
|
||||
const database = await setupDatabase(t);
|
||||
const values = await seedMetadata(database);
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.bulkInsert(LOCAL_ARTIFACT_RETENTION_TABLE, [
|
||||
{
|
||||
attempt_id: values.attemptId,
|
||||
log_artifact_id: values.logArtifactId,
|
||||
finished_at_ms: 101,
|
||||
eligible_at_ms: 200,
|
||||
disposition: 'deleted',
|
||||
bytes_reclaimed: 10,
|
||||
recorded_at_ms: 300,
|
||||
},
|
||||
]);
|
||||
const repository = new LegacySequelizeLocalArtifactReadMetadataRepository(
|
||||
database,
|
||||
);
|
||||
await assert.rejects(
|
||||
repository.find({
|
||||
projectId: PROJECT_ID,
|
||||
runId: RUN_ID,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
}),
|
||||
/metadata is corrupt or ambiguous/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,566 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const { QueryTypes, Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
runSchemaMigration,
|
||||
} = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
COMPLETION_RECEIPT_JOURNAL_TABLE,
|
||||
completionReceiptJournalMigration,
|
||||
} = require('../../back/migrations/0007-completion-receipt-journal');
|
||||
const {
|
||||
LOCAL_ARTIFACT_RETENTION_TABLE,
|
||||
localArtifactRetentionMigration,
|
||||
} = require('../../back/migrations/0015-local-artifact-retention');
|
||||
const {
|
||||
LOCAL_ARTIFACT_MAINTENANCE_CURSOR_TABLE,
|
||||
localArtifactMaintenanceCursorMigration,
|
||||
} = require('../../back/migrations/0016-local-artifact-maintenance-cursor');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LocalArtifactFileRetirementStore,
|
||||
UnsafeLocalArtifactRetirementError,
|
||||
} = require('../../back/runtime/adapters/fs/localArtifactFileRetirementStore');
|
||||
const {
|
||||
LegacySequelizeLocalArtifactRetentionRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/localArtifactRetentionRepository');
|
||||
const {
|
||||
LegacySequelizeLocalArtifactRetentionCheckpointStore,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/localArtifactRetentionCheckpointStore');
|
||||
const {
|
||||
LocalArtifactRetentionService,
|
||||
} = require('../../back/runtime/application/localArtifactRetentionService');
|
||||
const {
|
||||
localExecutionArtifactId,
|
||||
} = require('../../back/runtime/domain/localExecutionArtifact');
|
||||
const {
|
||||
encodeLocalArtifactTruncationFact,
|
||||
} = require('../../back/runtime/domain/localArtifactTruncation');
|
||||
|
||||
const DAY_MS = 24 * 60 * 60_000;
|
||||
const OBSERVED_AT_MS = 1_800_000_000_000;
|
||||
|
||||
function identity(index) {
|
||||
const suffix = String(index).padStart(12, '0');
|
||||
return {
|
||||
runId: `019f7400-0000-7000-8000-${suffix}`,
|
||||
attemptId: `019f7401-0000-7000-8000-${suffix}`,
|
||||
};
|
||||
}
|
||||
|
||||
function artifactId(index) {
|
||||
const ids = identity(index);
|
||||
return localExecutionArtifactId({
|
||||
...ids,
|
||||
projectId: 'default',
|
||||
taskId: `task-${index}`,
|
||||
taskRevision: 'revision-1',
|
||||
executorType: 'local_process',
|
||||
priority: 0,
|
||||
queuedAtMs: 1,
|
||||
attemptCreatedAtMs: 1,
|
||||
});
|
||||
}
|
||||
|
||||
async function setup(t) {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-retention-'));
|
||||
const artifacts = path.join(root, 'artifacts');
|
||||
await fs.mkdir(artifacts, { recursive: true, mode: 0o700 });
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
completionReceiptJournalMigration,
|
||||
localArtifactRetentionMigration,
|
||||
localArtifactMaintenanceCursorMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
return {
|
||||
root,
|
||||
artifacts,
|
||||
database,
|
||||
repository: new LegacySequelizeLocalArtifactRetentionRepository(database),
|
||||
checkpoints: new LegacySequelizeLocalArtifactRetentionCheckpointStore(
|
||||
database,
|
||||
),
|
||||
files: new LocalArtifactFileRetirementStore(artifacts),
|
||||
};
|
||||
}
|
||||
|
||||
async function seedAttempt(
|
||||
context,
|
||||
index,
|
||||
{
|
||||
finishedAtMs = OBSERVED_AT_MS - 10 * DAY_MS,
|
||||
runStatus = 'succeeded',
|
||||
attemptStatus = 'succeeded',
|
||||
executionOwner = 'runtime',
|
||||
executorType = 'local_process',
|
||||
logArtifactId = artifactId(index),
|
||||
} = {},
|
||||
) {
|
||||
const ids = identity(index);
|
||||
const query = context.database.getQueryInterface();
|
||||
await query.bulkInsert(RUN_TABLE, [
|
||||
{
|
||||
id: ids.runId,
|
||||
project_id: 'default',
|
||||
task_id: `task-${index}`,
|
||||
task_revision: 'revision-1',
|
||||
trigger_type: 'manual',
|
||||
execution_origin: 'manual',
|
||||
execution_owner: executionOwner,
|
||||
status: runStatus,
|
||||
version: 1,
|
||||
event_sequence: 1,
|
||||
priority: 0,
|
||||
created_at_ms: finishedAtMs - 1_000,
|
||||
finished_at_ms: runStatus === 'running' ? null : finishedAtMs,
|
||||
},
|
||||
]);
|
||||
await query.bulkInsert(RUN_ATTEMPT_TABLE, [
|
||||
{
|
||||
id: ids.attemptId,
|
||||
run_id: ids.runId,
|
||||
attempt: 1,
|
||||
status: attemptStatus,
|
||||
executor_type: executorType,
|
||||
log_artifact_id: logArtifactId,
|
||||
callback_sequence: 0,
|
||||
created_at_ms: finishedAtMs - 1_000,
|
||||
finished_at_ms: attemptStatus === 'running' ? null : finishedAtMs,
|
||||
},
|
||||
]);
|
||||
return { ...ids, logArtifactId, finishedAtMs };
|
||||
}
|
||||
|
||||
async function writeArtifact(context, logArtifactId, value = 'artifact-value') {
|
||||
const directory = path.join(context.artifacts, logArtifactId.slice(6, 8));
|
||||
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
const target = path.join(directory, `${logArtifactId}.log`);
|
||||
await fs.writeFile(target, value, { mode: 0o600 });
|
||||
return { directory, target, bytes: Buffer.byteLength(value) };
|
||||
}
|
||||
|
||||
function retentionService(context, overrides = {}) {
|
||||
return new LocalArtifactRetentionService(
|
||||
overrides.repository ?? context.repository,
|
||||
overrides.files ?? context.files,
|
||||
overrides.capacity ?? {
|
||||
async inspect() {
|
||||
return {
|
||||
availableBytes: BigInt(1024 * 1024),
|
||||
totalBytes: BigInt(2 * 1024 * 1024),
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
normalRetentionMs: overrides.normalRetentionMs ?? 7 * DAY_MS,
|
||||
pressureRetentionMs: overrides.pressureRetentionMs ?? DAY_MS,
|
||||
minimumFreeBytes: overrides.minimumFreeBytes ?? 64 * 1024,
|
||||
pageSize: overrides.pageSize ?? 16,
|
||||
maximumDeletions: overrides.maximumDeletions ?? 8,
|
||||
clock: { now: () => OBSERVED_AT_MS },
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
test('selects only settled runtime-owned terminal local Artifacts', async (t) => {
|
||||
const context = await setup(t);
|
||||
const eligible = await seedAttempt(context, 1);
|
||||
await seedAttempt(context, 2, { runStatus: 'running' });
|
||||
await seedAttempt(context, 3, { attemptStatus: 'running' });
|
||||
await seedAttempt(context, 4, { executionOwner: 'legacy' });
|
||||
await seedAttempt(context, 5, { executorType: 'docker' });
|
||||
await seedAttempt(context, 6, { logArtifactId: 'legacy-log-not-local' });
|
||||
const receiptBlocked = await seedAttempt(context, 7);
|
||||
await seedAttempt(context, 8, {
|
||||
runStatus: 'lost',
|
||||
attemptStatus: 'lost',
|
||||
});
|
||||
await context.database
|
||||
.getQueryInterface()
|
||||
.bulkInsert(COMPLETION_RECEIPT_JOURNAL_TABLE, [
|
||||
{
|
||||
attempt_id: receiptBlocked.attemptId,
|
||||
run_id: receiptBlocked.runId,
|
||||
state: 'pending',
|
||||
registered_at_ms: 1,
|
||||
updated_at_ms: 1,
|
||||
},
|
||||
]);
|
||||
|
||||
const page = await context.repository.list({
|
||||
cutoffMs: OBSERVED_AT_MS - 7 * DAY_MS,
|
||||
limit: 16,
|
||||
});
|
||||
assert.deepEqual(page.candidates, [
|
||||
{
|
||||
attemptId: eligible.attemptId,
|
||||
logArtifactId: eligible.logArtifactId,
|
||||
finishedAtMs: eligible.finishedAtMs,
|
||||
},
|
||||
]);
|
||||
await context.database
|
||||
.getQueryInterface()
|
||||
.bulkDelete(COMPLETION_RECEIPT_JOURNAL_TABLE, {
|
||||
attempt_id: receiptBlocked.attemptId,
|
||||
});
|
||||
assert.equal(
|
||||
(await context.repository.list({ cutoffMs: OBSERVED_AT_MS, limit: 16 }))
|
||||
.candidates.length,
|
||||
2,
|
||||
);
|
||||
});
|
||||
|
||||
test('uses pressure retention, deletes durably, and records an immutable tombstone', async (t) => {
|
||||
const context = await setup(t);
|
||||
const candidate = await seedAttempt(context, 1, {
|
||||
finishedAtMs: OBSERVED_AT_MS - 2 * DAY_MS,
|
||||
});
|
||||
const artifact = await writeArtifact(context, candidate.logArtifactId);
|
||||
const normal = await retentionService(context).sweep();
|
||||
assert.equal(normal.pressure, false);
|
||||
assert.equal(normal.candidatesScanned, 0);
|
||||
assert.equal((await fs.stat(artifact.target)).size, artifact.bytes);
|
||||
|
||||
const pressure = retentionService(context, {
|
||||
capacity: {
|
||||
async inspect() {
|
||||
return { availableBytes: BigInt(1), totalBytes: BigInt(1024) };
|
||||
},
|
||||
},
|
||||
});
|
||||
const result = await pressure.sweep();
|
||||
assert.deepEqual(
|
||||
{
|
||||
status: result.status,
|
||||
pressure: result.pressure,
|
||||
retentionMs: result.retentionMs,
|
||||
recordsWritten: result.recordsWritten,
|
||||
bytesReclaimed: result.bytesReclaimed,
|
||||
},
|
||||
{
|
||||
status: 'complete',
|
||||
pressure: true,
|
||||
retentionMs: DAY_MS,
|
||||
recordsWritten: 1,
|
||||
bytesReclaimed: artifact.bytes,
|
||||
},
|
||||
);
|
||||
await assert.rejects(fs.lstat(artifact.target), /ENOENT/);
|
||||
const tombstone = await context.database
|
||||
.getQueryInterface()
|
||||
.select(null, LOCAL_ARTIFACT_RETENTION_TABLE, {
|
||||
where: { attempt_id: candidate.attemptId },
|
||||
plain: true,
|
||||
});
|
||||
assert.equal(tombstone.disposition, 'deleted');
|
||||
assert.equal(Number(tombstone.bytes_reclaimed), artifact.bytes);
|
||||
assert.equal((await pressure.sweep()).candidatesScanned, 0);
|
||||
});
|
||||
|
||||
test('recovers a crash between file deletion and tombstone persistence', async (t) => {
|
||||
const context = await setup(t);
|
||||
const candidate = await seedAttempt(context, 1);
|
||||
const artifact = await writeArtifact(context, candidate.logArtifactId);
|
||||
let recordCalls = 0;
|
||||
const crashing = retentionService(context, {
|
||||
repository: {
|
||||
list: (options) => context.repository.list(options),
|
||||
async record() {
|
||||
recordCalls += 1;
|
||||
throw new Error('simulated database outage');
|
||||
},
|
||||
},
|
||||
});
|
||||
const failed = await crashing.sweep();
|
||||
assert.equal(failed.entries[0].outcome, 'record_failed');
|
||||
assert.equal(failed.entries[0].bytesReclaimed, artifact.bytes);
|
||||
assert.equal(recordCalls, 1);
|
||||
await assert.rejects(fs.lstat(artifact.target), /ENOENT/);
|
||||
|
||||
const recovered = await retentionService(context).sweep();
|
||||
assert.equal(recovered.entries[0].outcome, 'already_absent');
|
||||
assert.equal(recovered.recordsWritten, 1);
|
||||
assert.equal(
|
||||
await context.database
|
||||
.getQueryInterface()
|
||||
.rawSelect(
|
||||
LOCAL_ARTIFACT_RETENTION_TABLE,
|
||||
{ where: { attempt_id: candidate.attemptId } },
|
||||
['disposition'],
|
||||
),
|
||||
'already_absent',
|
||||
);
|
||||
});
|
||||
|
||||
test('enforces a deletion budget and resumes from a stable cursor', async (t) => {
|
||||
const context = await setup(t);
|
||||
for (let index = 1; index <= 3; index += 1) {
|
||||
const candidate = await seedAttempt(context, index);
|
||||
await writeArtifact(context, candidate.logArtifactId, `artifact-${index}`);
|
||||
}
|
||||
const service = retentionService(context, {
|
||||
pageSize: 3,
|
||||
maximumDeletions: 2,
|
||||
});
|
||||
const first = await service.sweep();
|
||||
assert.equal(first.status, 'deletion_budget_exhausted');
|
||||
assert.equal(first.recordsWritten, 2);
|
||||
assert.deepEqual(first.nextCursor, {
|
||||
finishedAtMs: OBSERVED_AT_MS - 10 * DAY_MS,
|
||||
attemptId: identity(2).attemptId,
|
||||
});
|
||||
const second = await service.sweep(first.nextCursor);
|
||||
assert.equal(second.status, 'complete');
|
||||
assert.equal(second.recordsWritten, 1);
|
||||
const [count] = await context.database.query(
|
||||
`SELECT COUNT(*) AS count FROM "${LOCAL_ARTIFACT_RETENTION_TABLE}"`,
|
||||
{ type: QueryTypes.SELECT },
|
||||
);
|
||||
assert.equal(count.count, 3);
|
||||
});
|
||||
|
||||
test('removes a stale quota FIFO but refuses symlink or non-file targets', async (t) => {
|
||||
const context = await setup(t);
|
||||
const logArtifactId = artifactId(1);
|
||||
const artifact = await writeArtifact(context, logArtifactId);
|
||||
const fifo = path.join(artifact.directory, `.${logArtifactId}.log.fifo`);
|
||||
await new Promise((resolve, reject) => {
|
||||
require('node:child_process').execFile(
|
||||
'mkfifo',
|
||||
['-m', '600', fifo],
|
||||
(error) => (error ? reject(error) : resolve()),
|
||||
);
|
||||
});
|
||||
const truncation = path.join(
|
||||
artifact.directory,
|
||||
`.${logArtifactId}.log.truncated.json`,
|
||||
);
|
||||
const truncationTemporary = path.join(
|
||||
artifact.directory,
|
||||
`.${logArtifactId}.log.truncated.tmp`,
|
||||
);
|
||||
await fs.writeFile(
|
||||
truncation,
|
||||
encodeLocalArtifactTruncationFact({
|
||||
schemaVersion: 1,
|
||||
...identity(1),
|
||||
logArtifactId,
|
||||
maximumBytes: 64 * 1024,
|
||||
quotaReached: true,
|
||||
observedAtMs: OBSERVED_AT_MS,
|
||||
}),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
await fs.writeFile(truncationTemporary, 'partial', { mode: 0o600 });
|
||||
assert.equal(
|
||||
(await context.files.retire(logArtifactId)).disposition,
|
||||
'deleted',
|
||||
);
|
||||
await assert.rejects(fs.lstat(fifo), /ENOENT/);
|
||||
await assert.rejects(fs.lstat(truncation), /ENOENT/);
|
||||
await assert.rejects(fs.lstat(truncationTemporary), /ENOENT/);
|
||||
|
||||
const outside = path.join(context.root, 'outside');
|
||||
await fs.writeFile(outside, 'outside');
|
||||
await fs.writeFile(artifact.target, 'replacement');
|
||||
await fs.unlink(artifact.target);
|
||||
await fs.symlink(outside, artifact.target);
|
||||
await assert.rejects(
|
||||
context.files.retire(logArtifactId),
|
||||
UnsafeLocalArtifactRetirementError,
|
||||
);
|
||||
assert.equal(await fs.readFile(outside, 'utf8'), 'outside');
|
||||
});
|
||||
|
||||
test('treats a missing Artifact shard as already absent', async (t) => {
|
||||
const context = await setup(t);
|
||||
assert.deepEqual(await context.files.retire(artifactId(1)), {
|
||||
disposition: 'already_absent',
|
||||
bytesReclaimed: 0,
|
||||
});
|
||||
});
|
||||
|
||||
test('rejects invalid capacity and pages before touching Artifact files', async (t) => {
|
||||
const context = await setup(t);
|
||||
const candidate = await seedAttempt(context, 1);
|
||||
const artifact = await writeArtifact(context, candidate.logArtifactId);
|
||||
let retireCalls = 0;
|
||||
const files = {
|
||||
async retire() {
|
||||
retireCalls += 1;
|
||||
return { disposition: 'deleted', bytesReclaimed: artifact.bytes };
|
||||
},
|
||||
};
|
||||
await assert.rejects(
|
||||
retentionService(context, {
|
||||
files,
|
||||
capacity: {
|
||||
async inspect() {
|
||||
return { availableBytes: BigInt(2), totalBytes: BigInt(1) };
|
||||
},
|
||||
},
|
||||
}).sweep(),
|
||||
/capacity snapshot is invalid/,
|
||||
);
|
||||
assert.equal(retireCalls, 0);
|
||||
|
||||
const invalidPage = retentionService(context, {
|
||||
files,
|
||||
repository: {
|
||||
async list() {
|
||||
return {
|
||||
candidates: [candidate],
|
||||
truncated: true,
|
||||
};
|
||||
},
|
||||
async record() {
|
||||
throw new Error('record must remain unreachable');
|
||||
},
|
||||
},
|
||||
});
|
||||
await assert.rejects(invalidPage.sweep(), /resume cursor is inconsistent/);
|
||||
assert.equal(retireCalls, 0);
|
||||
assert.equal((await fs.stat(artifact.target)).size, artifact.bytes);
|
||||
});
|
||||
|
||||
test('persists and fences the retention resume cursor without idle rewrites', async (t) => {
|
||||
const context = await setup(t);
|
||||
const cursor = {
|
||||
finishedAtMs: OBSERVED_AT_MS - DAY_MS,
|
||||
attemptId: identity(1).attemptId,
|
||||
};
|
||||
assert.deepEqual(await context.checkpoints.load(), { version: 0 });
|
||||
assert.equal(
|
||||
await context.checkpoints.compareAndSet({
|
||||
expectedVersion: 0,
|
||||
cursor,
|
||||
updatedAtMs: OBSERVED_AT_MS,
|
||||
}),
|
||||
true,
|
||||
);
|
||||
assert.deepEqual(await context.checkpoints.load(), {
|
||||
version: 1,
|
||||
cursor,
|
||||
});
|
||||
assert.equal(
|
||||
await context.checkpoints.compareAndSet({
|
||||
expectedVersion: 0,
|
||||
updatedAtMs: OBSERVED_AT_MS,
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
await context.checkpoints.compareAndSet({
|
||||
expectedVersion: 1,
|
||||
updatedAtMs: OBSERVED_AT_MS + 1,
|
||||
}),
|
||||
true,
|
||||
);
|
||||
assert.deepEqual(await context.checkpoints.load(), { version: 2 });
|
||||
assert.equal(
|
||||
await context.database
|
||||
.getQueryInterface()
|
||||
.rawSelect(
|
||||
LOCAL_ARTIFACT_MAINTENANCE_CURSOR_TABLE,
|
||||
{ where: { scope: 'retention' } },
|
||||
['version'],
|
||||
),
|
||||
2,
|
||||
);
|
||||
});
|
||||
|
||||
test('serializes concurrent initial cursor claims', async (t) => {
|
||||
const context = await setup(t);
|
||||
const competing = new LegacySequelizeLocalArtifactRetentionCheckpointStore(
|
||||
context.database,
|
||||
);
|
||||
const cursor = {
|
||||
finishedAtMs: OBSERVED_AT_MS - DAY_MS,
|
||||
attemptId: identity(1).attemptId,
|
||||
};
|
||||
const results = await Promise.all([
|
||||
context.checkpoints.compareAndSet({
|
||||
expectedVersion: 0,
|
||||
cursor,
|
||||
updatedAtMs: OBSERVED_AT_MS,
|
||||
}),
|
||||
competing.compareAndSet({
|
||||
expectedVersion: 0,
|
||||
cursor,
|
||||
updatedAtMs: OBSERVED_AT_MS,
|
||||
}),
|
||||
]);
|
||||
assert.deepEqual(results.sort(), [false, true]);
|
||||
assert.deepEqual(await context.checkpoints.load(), { version: 1, cursor });
|
||||
});
|
||||
|
||||
test('refuses a symlink truncation fact before deleting its Artifact', async (t) => {
|
||||
const context = await setup(t);
|
||||
const logArtifactId = artifactId(1);
|
||||
const artifact = await writeArtifact(context, logArtifactId);
|
||||
const outside = path.join(context.root, 'outside-truncation');
|
||||
await fs.writeFile(outside, 'outside');
|
||||
const truncation = path.join(
|
||||
artifact.directory,
|
||||
`.${logArtifactId}.log.truncated.json`,
|
||||
);
|
||||
await fs.symlink(outside, truncation);
|
||||
await assert.rejects(
|
||||
context.files.retire(logArtifactId),
|
||||
UnsafeLocalArtifactRetirementError,
|
||||
);
|
||||
assert.equal(await fs.readFile(outside, 'utf8'), 'outside');
|
||||
assert.equal((await fs.stat(artifact.target)).size, artifact.bytes);
|
||||
});
|
||||
|
||||
test('preserves the truncation fact when log deletion fails', async (t) => {
|
||||
const context = await setup(t);
|
||||
const logArtifactId = artifactId(1);
|
||||
const artifact = await writeArtifact(context, logArtifactId);
|
||||
const truncation = path.join(
|
||||
artifact.directory,
|
||||
`.${logArtifactId}.log.truncated.json`,
|
||||
);
|
||||
await fs.writeFile(
|
||||
truncation,
|
||||
encodeLocalArtifactTruncationFact({
|
||||
schemaVersion: 1,
|
||||
...identity(1),
|
||||
logArtifactId,
|
||||
maximumBytes: 64 * 1024,
|
||||
quotaReached: true,
|
||||
observedAtMs: OBSERVED_AT_MS,
|
||||
}),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
await fs.chmod(artifact.directory, 0o500);
|
||||
try {
|
||||
await assert.rejects(context.files.retire(logArtifactId));
|
||||
} finally {
|
||||
await fs.chmod(artifact.directory, 0o700);
|
||||
}
|
||||
assert.equal((await fs.stat(artifact.target)).size, artifact.bytes);
|
||||
assert.ok((await fs.stat(truncation)).isFile());
|
||||
});
|
||||
@@ -0,0 +1,275 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
LocalArtifactRetentionLifecycle,
|
||||
} = require('../../back/runtime/application/localArtifactRetentionLifecycle');
|
||||
|
||||
const CURSOR = {
|
||||
finishedAtMs: 1_800_000_000_000,
|
||||
attemptId: '019f7500-0000-7000-8000-000000000001',
|
||||
};
|
||||
|
||||
function deferred() {
|
||||
let resolve;
|
||||
const promise = new Promise((resolvePromise) => {
|
||||
resolve = resolvePromise;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
function fakeScheduler() {
|
||||
let id = 0;
|
||||
const pending = new Map();
|
||||
const cleared = [];
|
||||
return {
|
||||
pending,
|
||||
cleared,
|
||||
scheduler: {
|
||||
setTimeout(callback, delayMs) {
|
||||
const timer = {
|
||||
id: ++id,
|
||||
delayMs,
|
||||
unrefCalls: 0,
|
||||
unref() {
|
||||
this.unrefCalls += 1;
|
||||
},
|
||||
};
|
||||
pending.set(timer.id, { timer, callback });
|
||||
return timer;
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
cleared.push(timer.id);
|
||||
pending.delete(timer.id);
|
||||
},
|
||||
},
|
||||
fireNext() {
|
||||
const next = pending.values().next().value;
|
||||
assert.ok(next, 'expected a scheduled timer');
|
||||
pending.delete(next.timer.id);
|
||||
next.callback();
|
||||
return next.timer;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function memoryCheckpoints({ fenced = false } = {}) {
|
||||
let checkpoint = { version: 0 };
|
||||
const writes = [];
|
||||
return {
|
||||
writes,
|
||||
store: {
|
||||
async load() {
|
||||
return checkpoint;
|
||||
},
|
||||
async compareAndSet(value) {
|
||||
writes.push(value);
|
||||
if (fenced || value.expectedVersion !== checkpoint.version)
|
||||
return false;
|
||||
checkpoint = {
|
||||
version: checkpoint.version + 1,
|
||||
...(value.cursor ? { cursor: { ...value.cursor } } : {}),
|
||||
};
|
||||
return true;
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function sweepResult(overrides = {}) {
|
||||
return {
|
||||
status: 'complete',
|
||||
pressure: false,
|
||||
observedAtMs: 1_800_000_001_000,
|
||||
retentionMs: 7 * 24 * 60 * 60_000,
|
||||
availableBytes: BigInt(1024),
|
||||
totalBytes: BigInt(2048),
|
||||
candidatesScanned: 0,
|
||||
deletionsAttempted: 0,
|
||||
recordsWritten: 0,
|
||||
failedCandidates: 0,
|
||||
bytesReclaimed: 0,
|
||||
entries: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function flush() {
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
}
|
||||
|
||||
test('persists only changed cursors and emits bounded aggregate metrics', async () => {
|
||||
const timers = fakeScheduler();
|
||||
const checkpoints = memoryCheckpoints();
|
||||
const cursors = [];
|
||||
const summaries = [];
|
||||
let calls = 0;
|
||||
const lifecycle = new LocalArtifactRetentionLifecycle(
|
||||
{
|
||||
async sweep(cursor) {
|
||||
cursors.push(cursor);
|
||||
calls += 1;
|
||||
if (calls === 1) {
|
||||
return sweepResult({
|
||||
status: 'page_complete',
|
||||
candidatesScanned: 8,
|
||||
deletionsAttempted: 8,
|
||||
recordsWritten: 8,
|
||||
bytesReclaimed: 4096,
|
||||
nextCursor: CURSOR,
|
||||
entries: [
|
||||
{
|
||||
attemptId: CURSOR.attemptId,
|
||||
logArtifactId: `local-${'a'.repeat(30)}`,
|
||||
outcome: 'deleted',
|
||||
bytesReclaimed: 4096,
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
return sweepResult();
|
||||
},
|
||||
},
|
||||
checkpoints.store,
|
||||
{
|
||||
intervalMs: 5_000,
|
||||
initialDelayMs: 100,
|
||||
scheduler: timers.scheduler,
|
||||
onCycle(summary) {
|
||||
summaries.push(summary);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(lifecycle.start(), true);
|
||||
const initial = timers.fireNext();
|
||||
assert.equal(initial.delayMs, 100);
|
||||
assert.equal(initial.unrefCalls, 1);
|
||||
await flush();
|
||||
assert.equal(summaries[0].cursorAction, 'advanced');
|
||||
assert.equal(summaries[0].recordsWritten, 8);
|
||||
assert.equal('entries' in summaries[0], false);
|
||||
assert.equal(JSON.stringify(summaries[0]).includes(CURSOR.attemptId), false);
|
||||
assert.equal(summaries[0].availableBytes, '1024');
|
||||
|
||||
timers.fireNext();
|
||||
await flush();
|
||||
assert.equal(summaries[1].cursorAction, 'cleared');
|
||||
timers.fireNext();
|
||||
await flush();
|
||||
assert.equal(summaries[2].cursorAction, 'unchanged');
|
||||
assert.deepEqual(cursors, [undefined, CURSOR, undefined]);
|
||||
assert.equal(checkpoints.writes.length, 2);
|
||||
assert.equal(await lifecycle.stop(), 'drained');
|
||||
});
|
||||
|
||||
test('reports cursor fencing and observer failures without stopping cadence', async () => {
|
||||
const timers = fakeScheduler();
|
||||
const checkpoints = memoryCheckpoints({ fenced: true });
|
||||
const errors = [];
|
||||
const lifecycle = new LocalArtifactRetentionLifecycle(
|
||||
{
|
||||
async sweep() {
|
||||
return sweepResult({ status: 'page_complete', nextCursor: CURSOR });
|
||||
},
|
||||
},
|
||||
checkpoints.store,
|
||||
{
|
||||
intervalMs: 5_000,
|
||||
scheduler: timers.scheduler,
|
||||
onCycle(summary) {
|
||||
assert.equal(summary.cursorAction, 'fenced');
|
||||
throw new Error('metrics sink failed');
|
||||
},
|
||||
onError(error) {
|
||||
errors.push(error.message);
|
||||
},
|
||||
},
|
||||
);
|
||||
lifecycle.start();
|
||||
timers.fireNext();
|
||||
await flush();
|
||||
assert.deepEqual(errors, ['metrics sink failed']);
|
||||
assert.equal(timers.pending.size, 1);
|
||||
assert.equal(await lifecycle.stop(), 'drained');
|
||||
});
|
||||
|
||||
test('never overlaps a slow sweep and bounds shutdown', async () => {
|
||||
const timers = fakeScheduler();
|
||||
const running = deferred();
|
||||
const checkpoints = memoryCheckpoints();
|
||||
let calls = 0;
|
||||
const lifecycle = new LocalArtifactRetentionLifecycle(
|
||||
{
|
||||
async sweep() {
|
||||
calls += 1;
|
||||
return running.promise;
|
||||
},
|
||||
},
|
||||
checkpoints.store,
|
||||
{
|
||||
intervalMs: 5_000,
|
||||
stopTimeoutMs: 5,
|
||||
scheduler: timers.scheduler,
|
||||
},
|
||||
);
|
||||
lifecycle.start();
|
||||
timers.fireNext();
|
||||
await flush();
|
||||
assert.equal(calls, 1);
|
||||
assert.equal(timers.pending.size, 0);
|
||||
assert.equal(await lifecycle.stop(), 'timed_out');
|
||||
assert.equal(lifecycle.start(), false);
|
||||
running.resolve(sweepResult());
|
||||
await flush();
|
||||
assert.equal(lifecycle.start(), true);
|
||||
assert.equal(await lifecycle.stop(), 'drained');
|
||||
});
|
||||
|
||||
test('rejects hot loops, invalid incomplete results, and unbounded stops', async () => {
|
||||
const timers = fakeScheduler();
|
||||
const checkpoints = memoryCheckpoints();
|
||||
const service = {
|
||||
async sweep() {
|
||||
return sweepResult();
|
||||
},
|
||||
};
|
||||
assert.throws(
|
||||
() =>
|
||||
new LocalArtifactRetentionLifecycle(service, checkpoints.store, {
|
||||
intervalMs: 999,
|
||||
}),
|
||||
RangeError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
new LocalArtifactRetentionLifecycle(service, checkpoints.store, {
|
||||
intervalMs: 1_000,
|
||||
stopTimeoutMs: 60_001,
|
||||
}),
|
||||
RangeError,
|
||||
);
|
||||
|
||||
const errors = [];
|
||||
const invalid = new LocalArtifactRetentionLifecycle(
|
||||
{
|
||||
async sweep() {
|
||||
return sweepResult({ status: 'page_complete' });
|
||||
},
|
||||
},
|
||||
checkpoints.store,
|
||||
{
|
||||
intervalMs: 1_000,
|
||||
scheduler: timers.scheduler,
|
||||
onError(error) {
|
||||
errors.push(error.message);
|
||||
},
|
||||
},
|
||||
);
|
||||
invalid.start();
|
||||
timers.fireNext();
|
||||
await flush();
|
||||
assert.match(errors[0], /requires a resume cursor/);
|
||||
await invalid.stop();
|
||||
});
|
||||
@@ -0,0 +1,111 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
LocalArtifactTruncationFactStore,
|
||||
UnsafeLocalArtifactTruncationFactError,
|
||||
localArtifactTruncationFactFileName,
|
||||
} = require('../../back/runtime/adapters/fs/localArtifactTruncationFactStore');
|
||||
const {
|
||||
decodeLocalArtifactTruncationFact,
|
||||
encodeLocalArtifactTruncationFact,
|
||||
} = require('../../back/runtime/domain/localArtifactTruncation');
|
||||
|
||||
const LOG_ARTIFACT_ID = `local-${'c'.repeat(30)}`;
|
||||
const OTHER_ARTIFACT_ID = `local-${'d'.repeat(30)}`;
|
||||
const FACT = {
|
||||
schemaVersion: 1,
|
||||
runId: '019f7500-0000-7000-8000-000000000001',
|
||||
attemptId: '019f7500-0000-7000-8000-000000000002',
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
maximumBytes: 64 * 1024,
|
||||
quotaReached: true,
|
||||
observedAtMs: 1_800_000_000_000,
|
||||
};
|
||||
|
||||
async function temporaryRoot(t) {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-truncation-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
await fs.chmod(root, 0o700);
|
||||
return root;
|
||||
}
|
||||
|
||||
test('round-trips one exact canonical truncation fact', () => {
|
||||
const encoded = encodeLocalArtifactTruncationFact(FACT);
|
||||
assert.deepEqual(decodeLocalArtifactTruncationFact(encoded), FACT);
|
||||
assert.throws(
|
||||
() =>
|
||||
decodeLocalArtifactTruncationFact(
|
||||
JSON.stringify({ ...FACT, unknown: true }),
|
||||
),
|
||||
/shape is invalid/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
decodeLocalArtifactTruncationFact(
|
||||
JSON.stringify({ attemptId: FACT.attemptId, ...FACT }),
|
||||
),
|
||||
/not canonical/,
|
||||
);
|
||||
assert.throws(
|
||||
() => encodeLocalArtifactTruncationFact({ ...FACT, maximumBytes: 1 }),
|
||||
/maximumBytes is invalid/,
|
||||
);
|
||||
});
|
||||
|
||||
test('reads only the exact private Artifact fact and handles absence', async (t) => {
|
||||
const root = await temporaryRoot(t);
|
||||
const store = new LocalArtifactTruncationFactStore(root);
|
||||
assert.equal(await store.read(LOG_ARTIFACT_ID), null);
|
||||
const directory = path.join(root, LOG_ARTIFACT_ID.slice(6, 8));
|
||||
await fs.mkdir(directory, { mode: 0o700 });
|
||||
const target = path.join(
|
||||
directory,
|
||||
localArtifactTruncationFactFileName(LOG_ARTIFACT_ID),
|
||||
);
|
||||
await fs.writeFile(target, encodeLocalArtifactTruncationFact(FACT), {
|
||||
mode: 0o600,
|
||||
});
|
||||
assert.deepEqual(await store.read(LOG_ARTIFACT_ID), FACT);
|
||||
|
||||
await fs.writeFile(
|
||||
target,
|
||||
encodeLocalArtifactTruncationFact({
|
||||
...FACT,
|
||||
logArtifactId: OTHER_ARTIFACT_ID,
|
||||
}),
|
||||
);
|
||||
await assert.rejects(
|
||||
store.read(LOG_ARTIFACT_ID),
|
||||
UnsafeLocalArtifactTruncationFactError,
|
||||
);
|
||||
});
|
||||
|
||||
test('refuses symlink files and shard escapes', async (t) => {
|
||||
const root = await temporaryRoot(t);
|
||||
const store = new LocalArtifactTruncationFactStore(root);
|
||||
const outside = path.join(root, 'outside.json');
|
||||
await fs.writeFile(outside, encodeLocalArtifactTruncationFact(FACT));
|
||||
const directory = path.join(root, LOG_ARTIFACT_ID.slice(6, 8));
|
||||
await fs.mkdir(directory, { mode: 0o700 });
|
||||
const target = path.join(
|
||||
directory,
|
||||
localArtifactTruncationFactFileName(LOG_ARTIFACT_ID),
|
||||
);
|
||||
await fs.symlink(outside, target);
|
||||
await assert.rejects(
|
||||
store.read(LOG_ARTIFACT_ID),
|
||||
UnsafeLocalArtifactTruncationFactError,
|
||||
);
|
||||
await fs.unlink(target);
|
||||
await fs.rmdir(directory);
|
||||
await fs.symlink(path.dirname(outside), directory);
|
||||
await assert.rejects(
|
||||
store.read(LOG_ARTIFACT_ID),
|
||||
UnsafeLocalArtifactTruncationFactError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,254 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const { Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const {
|
||||
taskExecutionRevisionMigration,
|
||||
} = require('../../back/migrations/0012-task-execution-revisions');
|
||||
const {
|
||||
LOCAL_EXECUTION_CONTEXT_RECIPE_TABLE,
|
||||
localExecutionContextRecipeMigration,
|
||||
} = require('../../back/migrations/0013-local-execution-context-recipes');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeLocalExecutionContextRecipeRepository,
|
||||
LocalExecutionContextRecipeCorruptError,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/localExecutionContextRecipeRepository');
|
||||
const {
|
||||
LegacySequelizeTaskExecutionRevisionRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/taskExecutionRevisionRepository');
|
||||
const {
|
||||
LocalTaskExecutionRevisionPublisher,
|
||||
} = require('../../back/runtime/application/localTaskExecutionRevisionPublisher');
|
||||
const {
|
||||
RecipeLocalExecutionContextMaterializer,
|
||||
} = require('../../back/runtime/application/recipeLocalExecutionContextMaterializer');
|
||||
const {
|
||||
createLocalExecutionContextRecipe,
|
||||
localExecutionContextRecipeDigest,
|
||||
} = require('../../back/runtime/domain/localExecutionContextRecipe');
|
||||
|
||||
function contextRecipe() {
|
||||
return createLocalExecutionContextRecipe([
|
||||
{ name: 'MODE', kind: 'public', value: 'edge' },
|
||||
{ name: 'TOKEN', kind: 'secret', secretRef: 'secret://token' },
|
||||
]);
|
||||
}
|
||||
|
||||
function revision(recipe = contextRecipe()) {
|
||||
return {
|
||||
projectId: 'default',
|
||||
taskId: 'task-recipe',
|
||||
taskRevision: 'revision-1',
|
||||
executorType: 'local_process',
|
||||
execution: {
|
||||
command: { kind: 'argv', file: '/bin/true', args: [] },
|
||||
environmentPolicy: 'isolated',
|
||||
terminationGraceMs: 100,
|
||||
},
|
||||
contextRef: recipe.contextRef,
|
||||
};
|
||||
}
|
||||
|
||||
function candidate() {
|
||||
return {
|
||||
runId: 'run-recipe',
|
||||
attemptId: 'attempt-recipe',
|
||||
projectId: 'default',
|
||||
taskId: 'task-recipe',
|
||||
taskRevision: 'revision-1',
|
||||
executorType: 'local_process',
|
||||
priority: 0,
|
||||
queuedAtMs: 1_760_000_000_000,
|
||||
attemptCreatedAtMs: 1_760_000_000_000,
|
||||
};
|
||||
}
|
||||
|
||||
async function createRepositories(t) {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [
|
||||
taskExecutionRevisionMigration,
|
||||
localExecutionContextRecipeMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
return {
|
||||
database,
|
||||
recipes: new LegacySequelizeLocalExecutionContextRecipeRepository(database),
|
||||
revisions: new LegacySequelizeTaskExecutionRevisionRepository(database),
|
||||
};
|
||||
}
|
||||
|
||||
test('uses stable content-address and digest vectors without persisting Secret values', () => {
|
||||
const recipe = contextRecipe();
|
||||
assert.equal(
|
||||
recipe.contextRef,
|
||||
'localctx:sha256:d70176732d42e2701d95adce0371b765226a3d5085e17d14816615c933fc5c7f',
|
||||
);
|
||||
assert.equal(
|
||||
localExecutionContextRecipeDigest(recipe),
|
||||
'a1acbef2b7b882a0e6f204bd79f303a10fa6043753251ebb222adb31d42b2597',
|
||||
);
|
||||
assert.equal(
|
||||
createLocalExecutionContextRecipe([...recipe.environment].reverse())
|
||||
.contextRef,
|
||||
recipe.contextRef,
|
||||
);
|
||||
assert.equal(JSON.stringify(recipe).includes('in-memory-secret'), false);
|
||||
});
|
||||
|
||||
test('concurrent immutable recipe publication converges and resolves exactly', async (t) => {
|
||||
const { recipes } = await createRepositories(t);
|
||||
const recipe = contextRecipe();
|
||||
const results = await Promise.all(
|
||||
Array.from({ length: 12 }, (_, index) =>
|
||||
recipes.insert(recipe, 100 + index),
|
||||
),
|
||||
);
|
||||
assert.equal(results.filter((result) => result === 'inserted').length, 1);
|
||||
assert.equal(results.filter((result) => result === 'idempotent').length, 11);
|
||||
|
||||
const stored = await recipes.resolve(recipe.contextRef);
|
||||
assert.deepEqual(stored.environment, recipe.environment);
|
||||
assert.ok(Object.isFrozen(stored));
|
||||
assert.ok(Object.isFrozen(stored.environment));
|
||||
assert.equal(await recipes.resolve(`${recipe.contextRef}-missing`), null);
|
||||
await assert.rejects(
|
||||
recipes.insert({ ...recipe, contextRef: 'localctx:sha256:wrong' }, 200),
|
||||
/not content-addressed/,
|
||||
);
|
||||
});
|
||||
|
||||
test('fails closed on persisted recipe corruption and cluster dialect use', async (t) => {
|
||||
const { database, recipes } = await createRepositories(t);
|
||||
const recipe = contextRecipe();
|
||||
await recipes.insert(recipe, 100);
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.bulkUpdate(
|
||||
LOCAL_EXECUTION_CONTEXT_RECIPE_TABLE,
|
||||
{ content_digest: '0'.repeat(64) },
|
||||
{ context_ref: recipe.contextRef },
|
||||
);
|
||||
await assert.rejects(
|
||||
recipes.resolve(recipe.contextRef),
|
||||
LocalExecutionContextRecipeCorruptError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
new LegacySequelizeLocalExecutionContextRecipeRepository({
|
||||
getDialect: () => 'postgres',
|
||||
}),
|
||||
/PostgreSQL adapter/,
|
||||
);
|
||||
});
|
||||
|
||||
test('publishes recipe before revision and materializes it with ephemeral Secrets', async (t) => {
|
||||
const { recipes, revisions } = await createRepositories(t);
|
||||
const recipe = contextRecipe();
|
||||
const publisher = new LocalTaskExecutionRevisionPublisher(recipes, revisions);
|
||||
assert.deepEqual(
|
||||
await publisher.publish({
|
||||
revision: revision(recipe),
|
||||
contextRecipe: recipe,
|
||||
createdAtMs: 100,
|
||||
}),
|
||||
{ contextRecipe: 'inserted', revision: 'inserted' },
|
||||
);
|
||||
assert.deepEqual(
|
||||
await publisher.publish({
|
||||
revision: revision(recipe),
|
||||
contextRecipe: recipe,
|
||||
createdAtMs: 101,
|
||||
}),
|
||||
{ contextRecipe: 'idempotent', revision: 'idempotent' },
|
||||
);
|
||||
assert.equal(
|
||||
(await revisions.resolve(revision(recipe))).contextRef,
|
||||
recipe.contextRef,
|
||||
);
|
||||
|
||||
const materializer = new RecipeLocalExecutionContextMaterializer(
|
||||
recipes,
|
||||
{
|
||||
async prepare() {
|
||||
return {
|
||||
logArtifactId: `local-${'c'.repeat(30)}`,
|
||||
output: { async write() {} },
|
||||
dispose() {},
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
async resolve() {
|
||||
return ['in-memory-secret'];
|
||||
},
|
||||
},
|
||||
);
|
||||
const context = await materializer.prepare({
|
||||
candidate: candidate(),
|
||||
contextRef: recipe.contextRef,
|
||||
});
|
||||
assert.deepEqual(
|
||||
{ ...context.context.environment },
|
||||
{
|
||||
MODE: 'edge',
|
||||
TOKEN: 'in-memory-secret',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('a missing or failed recipe publication can never create a dangling revision', async () => {
|
||||
const calls = [];
|
||||
const publisher = new LocalTaskExecutionRevisionPublisher(
|
||||
{
|
||||
async resolve() {
|
||||
return null;
|
||||
},
|
||||
async insert() {
|
||||
calls.push('recipe');
|
||||
throw new Error('recipe write failed');
|
||||
},
|
||||
},
|
||||
{
|
||||
async resolve() {
|
||||
return null;
|
||||
},
|
||||
async insert() {
|
||||
calls.push('revision');
|
||||
return 'inserted';
|
||||
},
|
||||
},
|
||||
);
|
||||
const recipe = contextRecipe();
|
||||
await assert.rejects(
|
||||
publisher.publish({
|
||||
revision: revision(recipe),
|
||||
contextRecipe: recipe,
|
||||
createdAtMs: 100,
|
||||
}),
|
||||
/recipe write failed/,
|
||||
);
|
||||
assert.deepEqual(calls, ['recipe']);
|
||||
await assert.rejects(
|
||||
publisher.publish({
|
||||
revision: { ...revision(recipe), contextRef: 'localctx:sha256:other' },
|
||||
contextRecipe: recipe,
|
||||
createdAtMs: 100,
|
||||
}),
|
||||
/does not match/,
|
||||
);
|
||||
assert.deepEqual(calls, ['recipe']);
|
||||
});
|
||||
@@ -0,0 +1,225 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
LocalFileExecutionArtifactAllocator,
|
||||
} = require('../../back/runtime/adapters/fs/localFileExecutionArtifactAllocator');
|
||||
const {
|
||||
durableLocalProcessOutput,
|
||||
} = require('../../back/runtime/adapters/local-process/durableLocalProcessOutput');
|
||||
const {
|
||||
localExecutionArtifactId,
|
||||
} = require('../../back/runtime/domain/localExecutionArtifact');
|
||||
|
||||
const CAPACITY_POLICY = Object.freeze({
|
||||
maximumAttemptBytes: 64 * 1024,
|
||||
minimumFreeBytes: 0,
|
||||
});
|
||||
|
||||
function candidate(overrides = {}) {
|
||||
return {
|
||||
runId: 'run-artifact',
|
||||
attemptId: 'attempt-artifact',
|
||||
projectId: 'default',
|
||||
taskId: 'task-artifact',
|
||||
taskRevision: 'revision-1',
|
||||
executorType: 'local_process',
|
||||
priority: 0,
|
||||
queuedAtMs: 1_760_000_000_000,
|
||||
attemptCreatedAtMs: 1_760_000_000_000,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function temporaryRoots(t) {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-artifact-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
return {
|
||||
root,
|
||||
artifacts: path.join(root, 'artifacts'),
|
||||
receipts: path.join(root, 'receipts'),
|
||||
};
|
||||
}
|
||||
|
||||
function allocator(roots, policy = CAPACITY_POLICY, capacity) {
|
||||
return new LocalFileExecutionArtifactAllocator(
|
||||
roots.artifacts,
|
||||
roots.receipts,
|
||||
policy,
|
||||
capacity,
|
||||
);
|
||||
}
|
||||
|
||||
test('allocates a private deterministic Artifact and serializes accepted writes', async (t) => {
|
||||
const roots = await temporaryRoots(t);
|
||||
const artifacts = allocator(roots);
|
||||
const prepared = await artifacts.prepare(candidate());
|
||||
assert.equal(prepared.logArtifactId, localExecutionArtifactId(candidate()));
|
||||
assert.equal(prepared.logArtifactId.length, 36);
|
||||
const capability = durableLocalProcessOutput(prepared.output);
|
||||
assert.ok(capability);
|
||||
assert.equal(capability.maximumBytes, CAPACITY_POLICY.maximumAttemptBytes);
|
||||
assert.equal(capability.logArtifactId, prepared.logArtifactId);
|
||||
assert.equal(JSON.stringify(prepared.output).includes(roots.root), false);
|
||||
|
||||
await Promise.all([
|
||||
prepared.output.write({
|
||||
stream: 'stdout',
|
||||
chunk: Buffer.from('first\n'),
|
||||
observedAtMs: 1,
|
||||
}),
|
||||
prepared.output.write({
|
||||
stream: 'stderr',
|
||||
chunk: Buffer.from('second\n'),
|
||||
observedAtMs: 2,
|
||||
}),
|
||||
]);
|
||||
await prepared.dispose();
|
||||
assert.equal(
|
||||
await fs.readFile(capability.outputFilePath, 'utf8'),
|
||||
'first\nsecond\n',
|
||||
);
|
||||
assert.equal((await fs.stat(capability.outputFilePath)).mode & 0o777, 0o600);
|
||||
assert.equal(
|
||||
(await fs.stat(path.dirname(capability.outputFilePath))).mode & 0o777,
|
||||
0o700,
|
||||
);
|
||||
await assert.rejects(
|
||||
prepared.output.write({
|
||||
stream: 'stdout',
|
||||
chunk: Buffer.from('late'),
|
||||
observedAtMs: 3,
|
||||
}),
|
||||
/closed/,
|
||||
);
|
||||
|
||||
const replay = await artifacts.prepare(candidate());
|
||||
assert.equal(replay.logArtifactId, prepared.logArtifactId);
|
||||
await replay.output.write({
|
||||
stream: 'stdout',
|
||||
chunk: Buffer.from('replay\n'),
|
||||
observedAtMs: 4,
|
||||
});
|
||||
await replay.dispose();
|
||||
assert.equal(
|
||||
await fs.readFile(capability.outputFilePath, 'utf8'),
|
||||
'first\nsecond\nreplay\n',
|
||||
);
|
||||
});
|
||||
|
||||
test('uses a different opaque Artifact for each Attempt', async (t) => {
|
||||
const roots = await temporaryRoots(t);
|
||||
const artifacts = allocator(roots);
|
||||
const first = await artifacts.prepare(candidate());
|
||||
const second = await artifacts.prepare(candidate({ attemptId: 'attempt-2' }));
|
||||
assert.notEqual(first.logArtifactId, second.logArtifactId);
|
||||
await Promise.all([first.dispose(), second.dispose()]);
|
||||
});
|
||||
|
||||
test('drains writes accepted before asynchronous disposal closes the file', async (t) => {
|
||||
const roots = await temporaryRoots(t);
|
||||
const artifacts = allocator(roots);
|
||||
const prepared = await artifacts.prepare(candidate());
|
||||
const capability = durableLocalProcessOutput(prepared.output);
|
||||
const accepted = prepared.output.write({
|
||||
stream: 'stdout',
|
||||
chunk: Buffer.from('accepted-before-close'),
|
||||
observedAtMs: 1,
|
||||
});
|
||||
await prepared.dispose();
|
||||
await accepted;
|
||||
assert.equal(
|
||||
await fs.readFile(capability.outputFilePath, 'utf8'),
|
||||
'accepted-before-close',
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects relative roots and refuses a symlink Artifact target', async (t) => {
|
||||
assert.throws(
|
||||
() =>
|
||||
new LocalFileExecutionArtifactAllocator(
|
||||
'relative',
|
||||
'/tmp/receipts',
|
||||
CAPACITY_POLICY,
|
||||
),
|
||||
/must be absolute/,
|
||||
);
|
||||
const roots = await temporaryRoots(t);
|
||||
const reference = candidate();
|
||||
const artifactId = localExecutionArtifactId(reference);
|
||||
const shard = artifactId.slice(6, 8);
|
||||
const directory = path.join(roots.artifacts, shard);
|
||||
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
const outside = path.join(roots.root, 'outside.log');
|
||||
await fs.writeFile(outside, 'outside');
|
||||
await fs.symlink(outside, path.join(directory, `${artifactId}.log`));
|
||||
const artifacts = allocator(roots);
|
||||
await assert.rejects(artifacts.prepare(reference));
|
||||
assert.equal(await fs.readFile(outside, 'utf8'), 'outside');
|
||||
});
|
||||
|
||||
test('hard-caps ordinary output and never writes a partial byte past quota', async (t) => {
|
||||
const roots = await temporaryRoots(t);
|
||||
const artifacts = allocator(roots);
|
||||
const prepared = await artifacts.prepare(candidate());
|
||||
const capability = durableLocalProcessOutput(prepared.output);
|
||||
const oversized = Buffer.alloc(
|
||||
CAPACITY_POLICY.maximumAttemptBytes + 37,
|
||||
0x61,
|
||||
);
|
||||
await assert.rejects(
|
||||
prepared.output.write({
|
||||
stream: 'stdout',
|
||||
chunk: oversized,
|
||||
observedAtMs: 1,
|
||||
}),
|
||||
/byte quota/,
|
||||
);
|
||||
await assert.rejects(
|
||||
prepared.output.write({
|
||||
stream: 'stderr',
|
||||
chunk: Buffer.from('must-not-append'),
|
||||
observedAtMs: 2,
|
||||
}),
|
||||
/byte quota/,
|
||||
);
|
||||
await prepared.dispose();
|
||||
const stored = await fs.readFile(capability.outputFilePath);
|
||||
assert.equal(stored.length, CAPACITY_POLICY.maximumAttemptBytes);
|
||||
assert.equal(stored.equals(oversized.subarray(0, stored.length)), true);
|
||||
});
|
||||
|
||||
test('reserves free space before opening an Attempt Artifact', async (t) => {
|
||||
const roots = await temporaryRoots(t);
|
||||
let inspected = 0;
|
||||
const artifacts = allocator(
|
||||
roots,
|
||||
{ maximumAttemptBytes: 64 * 1024, minimumFreeBytes: 128 * 1024 },
|
||||
{
|
||||
async inspect(root) {
|
||||
inspected += 1;
|
||||
assert.equal(root, roots.artifacts);
|
||||
return {
|
||||
availableBytes: BigInt(128 * 1024),
|
||||
totalBytes: BigInt(1024 * 1024),
|
||||
};
|
||||
},
|
||||
},
|
||||
);
|
||||
await assert.rejects(
|
||||
artifacts.prepare(candidate()),
|
||||
/capacity is unavailable/,
|
||||
);
|
||||
assert.equal(inspected, 1);
|
||||
const artifactId = localExecutionArtifactId(candidate());
|
||||
const target = path.join(
|
||||
roots.artifacts,
|
||||
artifactId.slice(6, 8),
|
||||
`${artifactId}.log`,
|
||||
);
|
||||
await assert.rejects(fs.lstat(target), /ENOENT/);
|
||||
});
|
||||
@@ -0,0 +1,457 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const { setTimeout: delay } = require('node:timers/promises');
|
||||
const {
|
||||
LocalProcessExecutor,
|
||||
} = require('../../back/runtime/adapters/local-process/localProcessExecutor');
|
||||
const {
|
||||
PosixProcessTerminator,
|
||||
} = require('../../back/runtime/adapters/local-process/processTerminator');
|
||||
const {
|
||||
ExecutorCapabilityUnavailableError,
|
||||
ExecutorHandleNotFoundError,
|
||||
ExecutorStartError,
|
||||
InvalidExecutionSpecError,
|
||||
} = require('../../back/runtime/domain/executorErrors');
|
||||
|
||||
let idSequence = 300;
|
||||
|
||||
function nextId() {
|
||||
idSequence += 1;
|
||||
return `019f70e0-0000-7000-8000-${String(idSequence).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
function createSpec(overrides = {}) {
|
||||
return {
|
||||
runId: nextId(),
|
||||
attemptId: nextId(),
|
||||
projectId: 'default',
|
||||
taskId: 'executor-contract-test',
|
||||
taskRevision: 'revision-1',
|
||||
command: {
|
||||
kind: 'argv',
|
||||
file: process.execPath,
|
||||
args: ['-e', "process.stdout.write('ok')"],
|
||||
},
|
||||
environmentPolicy: 'isolated',
|
||||
terminationGraceMs: 100,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createOutputCollector(write) {
|
||||
const chunks = { stdout: [], stderr: [] };
|
||||
return {
|
||||
chunks,
|
||||
context: {
|
||||
environment: {},
|
||||
output: {
|
||||
async write(output) {
|
||||
chunks[output.stream].push(Buffer.from(output.chunk));
|
||||
await write?.(output);
|
||||
},
|
||||
},
|
||||
},
|
||||
text(stream) {
|
||||
return Buffer.concat(chunks[stream]).toString('utf8');
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test(
|
||||
'executes argv commands and drains ordered stdout/stderr before completion',
|
||||
{ timeout: 5_000 },
|
||||
async () => {
|
||||
const executor = new LocalProcessExecutor({ createHandleId: nextId });
|
||||
const output = createOutputCollector(async () => delay(2));
|
||||
const handle = await executor.start(
|
||||
createSpec({
|
||||
command: {
|
||||
kind: 'argv',
|
||||
file: process.execPath,
|
||||
args: [
|
||||
'-e',
|
||||
"process.stdout.write('stdout-value'); process.stderr.write('stderr-value')",
|
||||
],
|
||||
},
|
||||
}),
|
||||
output.context,
|
||||
);
|
||||
|
||||
assert.ok(handle.pid > 0);
|
||||
assert.equal(handle.executorType, 'local_process');
|
||||
const result = await handle.completion;
|
||||
assert.equal(result.outcome, 'succeeded');
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(output.text('stdout'), 'stdout-value');
|
||||
assert.equal(output.text('stderr'), 'stderr-value');
|
||||
assert.deepEqual(await executor.inspect(handle), {
|
||||
status: 'exited',
|
||||
result,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'supports explicit shell compatibility commands and stable non-zero results',
|
||||
{ timeout: 5_000 },
|
||||
async () => {
|
||||
const executor = new LocalProcessExecutor({ createHandleId: nextId });
|
||||
const output = createOutputCollector();
|
||||
const handle = await executor.start(
|
||||
createSpec({
|
||||
command: {
|
||||
kind: 'shell',
|
||||
command: "printf 'shell-stdout'; printf 'shell-stderr' >&2; exit 7",
|
||||
shell: '/bin/bash',
|
||||
},
|
||||
}),
|
||||
output.context,
|
||||
);
|
||||
|
||||
const result = await handle.completion;
|
||||
assert.equal(result.outcome, 'failed');
|
||||
assert.equal(result.exitCode, 7);
|
||||
assert.equal(result.errorCode, 'PROCESS_EXIT_NON_ZERO');
|
||||
assert.equal(output.text('stdout'), 'shell-stdout');
|
||||
assert.equal(output.text('stderr'), 'shell-stderr');
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'honors isolated environment and an authorized working directory',
|
||||
{ timeout: 5_000 },
|
||||
async () => {
|
||||
const previous = process.env.QL3_PARENT_ONLY_VALUE;
|
||||
process.env.QL3_PARENT_ONLY_VALUE = 'must-not-leak';
|
||||
try {
|
||||
const executor = new LocalProcessExecutor({ createHandleId: nextId });
|
||||
const output = createOutputCollector();
|
||||
output.context.environment = { QL3_SUPPLIED_VALUE: 'visible' };
|
||||
const handle = await executor.start(
|
||||
createSpec({
|
||||
workingDirectory: process.cwd(),
|
||||
command: {
|
||||
kind: 'argv',
|
||||
file: process.execPath,
|
||||
args: [
|
||||
'-e',
|
||||
'process.stdout.write(JSON.stringify({ cwd: process.cwd(), supplied: process.env.QL3_SUPPLIED_VALUE, inherited: process.env.QL3_PARENT_ONLY_VALUE }))',
|
||||
],
|
||||
},
|
||||
}),
|
||||
output.context,
|
||||
);
|
||||
|
||||
assert.equal((await handle.completion).outcome, 'succeeded');
|
||||
assert.deepEqual(JSON.parse(output.text('stdout')), {
|
||||
cwd: process.cwd(),
|
||||
supplied: 'visible',
|
||||
});
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.QL3_PARENT_ONLY_VALUE;
|
||||
else process.env.QL3_PARENT_ONLY_VALUE = previous;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'applies output backpressure without buffering the complete process output',
|
||||
{ timeout: 10_000 },
|
||||
async () => {
|
||||
let activeWrites = 0;
|
||||
let maximumActiveWrites = 0;
|
||||
const executor = new LocalProcessExecutor({ createHandleId: nextId });
|
||||
const output = createOutputCollector(async () => {
|
||||
activeWrites += 1;
|
||||
maximumActiveWrites = Math.max(maximumActiveWrites, activeWrites);
|
||||
await delay(1);
|
||||
activeWrites -= 1;
|
||||
});
|
||||
const handle = await executor.start(
|
||||
createSpec({
|
||||
command: {
|
||||
kind: 'argv',
|
||||
file: process.execPath,
|
||||
args: [
|
||||
'-e',
|
||||
"for (let i = 0; i < 1000; i += 1) process.stdout.write('x'.repeat(1024))",
|
||||
],
|
||||
},
|
||||
}),
|
||||
output.context,
|
||||
);
|
||||
|
||||
assert.equal((await handle.completion).outcome, 'succeeded');
|
||||
assert.equal(output.text('stdout').length, 1_024_000);
|
||||
assert.equal(maximumActiveWrites, 1);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'continues draining output and reports a bounded diagnostic when the sink fails',
|
||||
{ timeout: 5_000 },
|
||||
async () => {
|
||||
const executor = new LocalProcessExecutor({ createHandleId: nextId });
|
||||
let writes = 0;
|
||||
const context = {
|
||||
environment: {},
|
||||
output: {
|
||||
async write() {
|
||||
writes += 1;
|
||||
throw new Error('sink contains potentially sensitive details');
|
||||
},
|
||||
},
|
||||
};
|
||||
const handle = await executor.start(
|
||||
createSpec({
|
||||
command: {
|
||||
kind: 'argv',
|
||||
file: process.execPath,
|
||||
args: [
|
||||
'-e',
|
||||
"for (let i = 0; i < 200; i += 1) process.stdout.write('x'.repeat(1024))",
|
||||
],
|
||||
},
|
||||
}),
|
||||
context,
|
||||
);
|
||||
|
||||
const result = await handle.completion;
|
||||
assert.equal(result.outcome, 'succeeded');
|
||||
assert.equal(writes, 1);
|
||||
assert.deepEqual(result.diagnostics, [
|
||||
{
|
||||
code: 'OUTPUT_SINK_FAILED',
|
||||
summary: 'Execution output sink failed; output may be incomplete',
|
||||
},
|
||||
]);
|
||||
assert.doesNotMatch(JSON.stringify(result), /potentially sensitive/);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'cancels a process group and keeps repeated stop calls idempotent',
|
||||
{ timeout: 5_000 },
|
||||
async () => {
|
||||
const executor = new LocalProcessExecutor({ createHandleId: nextId });
|
||||
const output = createOutputCollector();
|
||||
const handle = await executor.start(
|
||||
createSpec({
|
||||
command: {
|
||||
kind: 'argv',
|
||||
file: process.execPath,
|
||||
args: ['-e', 'setInterval(() => undefined, 1000)'],
|
||||
},
|
||||
}),
|
||||
output.context,
|
||||
);
|
||||
|
||||
const stopResult = await executor.stop(handle, {
|
||||
kind: 'user',
|
||||
requestedAtMs: Date.now(),
|
||||
});
|
||||
const result = await handle.completion;
|
||||
assert.deepEqual(stopResult, {
|
||||
status: 'termination_requested',
|
||||
termSignalSent: true,
|
||||
killSignalSent: false,
|
||||
});
|
||||
assert.equal(result.outcome, 'cancelled');
|
||||
assert.equal(result.signal, 'SIGTERM');
|
||||
assert.deepEqual(
|
||||
await executor.stop(handle, {
|
||||
kind: 'user',
|
||||
requestedAtMs: Date.now(),
|
||||
}),
|
||||
{
|
||||
status: 'already_exited',
|
||||
termSignalSent: false,
|
||||
killSignalSent: false,
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'escalates ignored timeout termination to SIGKILL and reports timed_out',
|
||||
{ timeout: 5_000 },
|
||||
async () => {
|
||||
const executor = new LocalProcessExecutor({ createHandleId: nextId });
|
||||
const output = createOutputCollector();
|
||||
const handle = await executor.start(
|
||||
createSpec({
|
||||
// The complete test suite runs files concurrently; leave enough time
|
||||
// for the child Node process to install its SIGTERM handler first.
|
||||
timeoutMs: 1_000,
|
||||
terminationGraceMs: 30,
|
||||
command: {
|
||||
kind: 'argv',
|
||||
file: process.execPath,
|
||||
args: [
|
||||
'-e',
|
||||
"process.on('SIGTERM', () => undefined); setInterval(() => undefined, 1000)",
|
||||
],
|
||||
},
|
||||
}),
|
||||
output.context,
|
||||
);
|
||||
|
||||
const result = await handle.completion;
|
||||
assert.equal(result.outcome, 'timed_out');
|
||||
assert.equal(result.signal, 'SIGKILL');
|
||||
assert.equal(result.errorCode, 'EXECUTION_TIMED_OUT');
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'observes AbortSignal cancellation without leaking listeners into completion',
|
||||
{ timeout: 5_000 },
|
||||
async () => {
|
||||
const controller = new AbortController();
|
||||
const executor = new LocalProcessExecutor({ createHandleId: nextId });
|
||||
const output = createOutputCollector();
|
||||
output.context.signal = controller.signal;
|
||||
const handle = await executor.start(
|
||||
createSpec({
|
||||
command: {
|
||||
kind: 'argv',
|
||||
file: process.execPath,
|
||||
args: ['-e', 'setInterval(() => undefined, 1000)'],
|
||||
},
|
||||
}),
|
||||
output.context,
|
||||
);
|
||||
|
||||
controller.abort();
|
||||
assert.equal((await handle.completion).outcome, 'cancelled');
|
||||
},
|
||||
);
|
||||
|
||||
test('rejects unsupported required limits and invalid specs before spawn', async () => {
|
||||
const executor = new LocalProcessExecutor({ createHandleId: nextId });
|
||||
const output = createOutputCollector();
|
||||
|
||||
await assert.rejects(
|
||||
executor.start(
|
||||
createSpec({
|
||||
resourcePolicy: {
|
||||
memoryBytes: { value: 64 * 1024 * 1024, enforcement: 'required' },
|
||||
},
|
||||
}),
|
||||
output.context,
|
||||
),
|
||||
ExecutorCapabilityUnavailableError,
|
||||
);
|
||||
await assert.rejects(
|
||||
executor.start(
|
||||
createSpec({ workingDirectory: 'relative/path' }),
|
||||
output.context,
|
||||
),
|
||||
InvalidExecutionSpecError,
|
||||
);
|
||||
|
||||
const bestEffort = await executor.start(
|
||||
createSpec({
|
||||
resourcePolicy: {
|
||||
memoryBytes: {
|
||||
value: 64 * 1024 * 1024,
|
||||
enforcement: 'best_effort',
|
||||
},
|
||||
networkIsolation: 'best_effort',
|
||||
},
|
||||
}),
|
||||
output.context,
|
||||
);
|
||||
assert.deepEqual((await bestEffort.completion).diagnostics, [
|
||||
{
|
||||
code: 'RESOURCE_POLICY_BEST_EFFORT_UNAVAILABLE',
|
||||
summary:
|
||||
'Best-effort capabilities were unavailable: memoryLimit, networkIsolation',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('maps spawn failures and rejects handles owned by another executor', async () => {
|
||||
const executor = new LocalProcessExecutor({ createHandleId: nextId });
|
||||
const output = createOutputCollector();
|
||||
await assert.rejects(
|
||||
executor.start(
|
||||
createSpec({
|
||||
command: {
|
||||
kind: 'argv',
|
||||
file: '/path/that/does/not/exist/ql3-command',
|
||||
args: [],
|
||||
},
|
||||
}),
|
||||
output.context,
|
||||
),
|
||||
ExecutorStartError,
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
executor.inspect({
|
||||
id: nextId(),
|
||||
executorType: 'local_process',
|
||||
runId: nextId(),
|
||||
attemptId: nextId(),
|
||||
startedAtMs: Date.now(),
|
||||
completion: Promise.resolve({
|
||||
outcome: 'lost',
|
||||
startedAtMs: Date.now(),
|
||||
finishedAtMs: Date.now(),
|
||||
}),
|
||||
}),
|
||||
ExecutorHandleNotFoundError,
|
||||
);
|
||||
});
|
||||
|
||||
test('process terminator stops after TERM or escalates after the grace window', async () => {
|
||||
let resolveClosed;
|
||||
const closed = new Promise((resolve) => {
|
||||
resolveClosed = resolve;
|
||||
});
|
||||
const signals = [];
|
||||
const graceful = new PosixProcessTerminator((pid, signal) => {
|
||||
signals.push([pid, signal]);
|
||||
if (signal === 'SIGTERM') resolveClosed();
|
||||
});
|
||||
assert.deepEqual(
|
||||
await graceful.terminate({
|
||||
pid: 42,
|
||||
processGroup: true,
|
||||
graceMs: 50,
|
||||
closed,
|
||||
}),
|
||||
{
|
||||
alreadyExited: false,
|
||||
termSignalSent: true,
|
||||
killSignalSent: false,
|
||||
},
|
||||
);
|
||||
assert.deepEqual(signals, [[-42, 'SIGTERM']]);
|
||||
|
||||
const escalatedSignals = [];
|
||||
const escalated = new PosixProcessTerminator((pid, signal) => {
|
||||
escalatedSignals.push([pid, signal]);
|
||||
});
|
||||
assert.deepEqual(
|
||||
await escalated.terminate({
|
||||
pid: 43,
|
||||
processGroup: false,
|
||||
graceMs: 1,
|
||||
closed: new Promise(() => undefined),
|
||||
}),
|
||||
{
|
||||
alreadyExited: false,
|
||||
termSignalSent: true,
|
||||
killSignalSent: true,
|
||||
},
|
||||
);
|
||||
assert.deepEqual(escalatedSignals, [
|
||||
[43, 'SIGTERM'],
|
||||
[43, 'SIGKILL'],
|
||||
]);
|
||||
});
|
||||
@@ -0,0 +1,217 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
createLocalProcessDurableHandle,
|
||||
LinuxProcProcessIdentityProvider,
|
||||
LocalProcessPersistedExecutionInspector,
|
||||
MAX_LOCAL_PROCESS_DURABLE_HANDLE_BYTES,
|
||||
parseLocalProcessDurableHandle,
|
||||
} = require('../../back/runtime/adapters/local-process/localProcessIdentity');
|
||||
const {
|
||||
LocalProcessExecutor,
|
||||
} = require('../../back/runtime/adapters/local-process/localProcessExecutor');
|
||||
|
||||
let idSequence = 900;
|
||||
|
||||
function nextId() {
|
||||
idSequence += 1;
|
||||
return `019f70f0-0000-7000-8000-${String(idSequence).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
function identity(overrides = {}) {
|
||||
return {
|
||||
platform: 'linux',
|
||||
bootId: '11111111-2222-3333-4444-555555555555',
|
||||
pid: 4321,
|
||||
processGroupId: 4321,
|
||||
startTimeTicks: '987654321',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function procStat({
|
||||
pid = 4321,
|
||||
processGroupId = 4321,
|
||||
startTimeTicks = '987654321',
|
||||
state = 'S',
|
||||
} = {}) {
|
||||
const fields = Array(20).fill('0');
|
||||
fields[0] = state;
|
||||
fields[1] = '1';
|
||||
fields[2] = String(processGroupId);
|
||||
fields[19] = startTimeTicks;
|
||||
return `${pid} (node worker with spaces) ${fields.join(' ')}`;
|
||||
}
|
||||
|
||||
function missingFile() {
|
||||
const error = new Error('not found');
|
||||
error.code = 'ENOENT';
|
||||
return error;
|
||||
}
|
||||
|
||||
test('round-trips a bounded opaque Linux process identity', () => {
|
||||
const handleId = nextId();
|
||||
const durableHandle = createLocalProcessDurableHandle(handleId, identity());
|
||||
|
||||
assert.ok(
|
||||
Buffer.byteLength(durableHandle) <= MAX_LOCAL_PROCESS_DURABLE_HANDLE_BYTES,
|
||||
);
|
||||
assert.deepEqual(parseLocalProcessDurableHandle(durableHandle), {
|
||||
handleId,
|
||||
identity: identity(),
|
||||
});
|
||||
assert.doesNotMatch(durableHandle, /node worker|command|environment/);
|
||||
});
|
||||
|
||||
test('rejects malformed, oversized, and unsafe durable handles', () => {
|
||||
assert.equal(parseLocalProcessDurableHandle('legacy-uuid-only'), null);
|
||||
assert.equal(parseLocalProcessDurableHandle('ql3lp1.not+base64url'), null);
|
||||
assert.equal(
|
||||
parseLocalProcessDurableHandle(
|
||||
`ql3lp1.${'a'.repeat(MAX_LOCAL_PROCESS_DURABLE_HANDLE_BYTES)}`,
|
||||
),
|
||||
null,
|
||||
);
|
||||
assert.throws(() =>
|
||||
createLocalProcessDurableHandle('invalid\0handle', identity()),
|
||||
);
|
||||
assert.throws(() =>
|
||||
createLocalProcessDurableHandle(nextId(), identity({ pid: 0 })),
|
||||
);
|
||||
});
|
||||
|
||||
test('captures and verifies boot, start-time, and process-group identity', async () => {
|
||||
const files = new Map([
|
||||
[
|
||||
'/proc/sys/kernel/random/boot_id',
|
||||
'11111111-2222-3333-4444-555555555555\n',
|
||||
],
|
||||
['/proc/4321/stat', procStat()],
|
||||
]);
|
||||
const provider = new LinuxProcProcessIdentityProvider({
|
||||
platform: 'linux',
|
||||
async readTextFile(path) {
|
||||
const value = files.get(path);
|
||||
if (value === undefined) throw missingFile();
|
||||
return value;
|
||||
},
|
||||
});
|
||||
|
||||
const captured = await provider.capture(4321);
|
||||
assert.deepEqual(captured, identity());
|
||||
assert.deepEqual(await provider.inspect(captured), { status: 'running' });
|
||||
|
||||
files.set('/proc/4321/stat', procStat({ startTimeTicks: '987654322' }));
|
||||
assert.deepEqual(await provider.inspect(captured), {
|
||||
status: 'identity_mismatch',
|
||||
});
|
||||
|
||||
files.set('/proc/4321/stat', procStat({ state: 'Z' }));
|
||||
assert.deepEqual(await provider.inspect(captured), { status: 'exited' });
|
||||
|
||||
files.delete('/proc/4321/stat');
|
||||
assert.deepEqual(await provider.inspect(captured), { status: 'exited' });
|
||||
});
|
||||
|
||||
test('never accepts an identity from another boot or unsupported platform', async () => {
|
||||
const provider = new LinuxProcProcessIdentityProvider({
|
||||
platform: 'linux',
|
||||
async readTextFile(path) {
|
||||
if (path === '/proc/sys/kernel/random/boot_id') {
|
||||
return 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee';
|
||||
}
|
||||
return procStat();
|
||||
},
|
||||
});
|
||||
assert.deepEqual(await provider.inspect(identity()), {
|
||||
status: 'identity_mismatch',
|
||||
});
|
||||
|
||||
const unsupported = new LinuxProcProcessIdentityProvider({
|
||||
platform: 'darwin',
|
||||
async readTextFile() {
|
||||
throw new Error('must not read /proc');
|
||||
},
|
||||
});
|
||||
assert.equal(await unsupported.capture(4321), null);
|
||||
assert.deepEqual(await unsupported.inspect(identity()), {
|
||||
status: 'unsupported',
|
||||
});
|
||||
});
|
||||
|
||||
test('classifies invalid persisted values before consulting the OS', async () => {
|
||||
let inspections = 0;
|
||||
const inspector = new LocalProcessPersistedExecutionInspector({
|
||||
async capture() {
|
||||
return null;
|
||||
},
|
||||
async inspect() {
|
||||
inspections += 1;
|
||||
return { status: 'running' };
|
||||
},
|
||||
});
|
||||
|
||||
assert.deepEqual(await inspector.inspect('not-a-durable-handle'), {
|
||||
status: 'invalid',
|
||||
});
|
||||
assert.equal(inspections, 0);
|
||||
|
||||
const durableHandle = createLocalProcessDurableHandle(nextId(), identity());
|
||||
assert.deepEqual(await inspector.inspect(durableHandle), {
|
||||
status: 'running',
|
||||
identityPid: 4321,
|
||||
});
|
||||
assert.equal(inspections, 1);
|
||||
});
|
||||
|
||||
test(
|
||||
'LocalProcessExecutor exposes a durable identity without replacing its live handle',
|
||||
{ timeout: 5_000 },
|
||||
async () => {
|
||||
const observed = identity();
|
||||
const executor = new LocalProcessExecutor({
|
||||
createHandleId: nextId,
|
||||
identityProvider: {
|
||||
async capture(pid) {
|
||||
return { ...observed, pid, processGroupId: pid };
|
||||
},
|
||||
async inspect() {
|
||||
return { status: 'running' };
|
||||
},
|
||||
},
|
||||
});
|
||||
const handle = await executor.start(
|
||||
{
|
||||
runId: nextId(),
|
||||
attemptId: nextId(),
|
||||
projectId: 'default',
|
||||
taskId: 'durable-handle-test',
|
||||
taskRevision: 'revision-1',
|
||||
command: {
|
||||
kind: 'argv',
|
||||
file: process.execPath,
|
||||
args: ['-e', 'setInterval(() => undefined, 1000)'],
|
||||
},
|
||||
environmentPolicy: 'isolated',
|
||||
terminationGraceMs: 100,
|
||||
},
|
||||
{
|
||||
environment: {},
|
||||
output: { async write() {} },
|
||||
},
|
||||
);
|
||||
|
||||
const parsed = parseLocalProcessDurableHandle(handle.durableHandle);
|
||||
assert.equal(parsed.handleId, handle.id);
|
||||
assert.equal(parsed.identity.pid, handle.pid);
|
||||
assert.equal(parsed.identity.processGroupId, handle.pid);
|
||||
|
||||
await executor.stop(handle, {
|
||||
kind: 'user',
|
||||
requestedAtMs: Date.now(),
|
||||
});
|
||||
assert.equal((await handle.completion).outcome, 'cancelled');
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,310 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
LocalRunDispatcher,
|
||||
} = require('../../back/runtime/application/localRunDispatcher');
|
||||
const {
|
||||
PrimaryClaimedRunRejectedError,
|
||||
} = require('../../back/runtime/application/primaryRunOrchestrator');
|
||||
|
||||
const NOW = 1_760_000_000_000;
|
||||
|
||||
function deferred() {
|
||||
let resolve;
|
||||
let reject;
|
||||
const promise = new Promise((resolvePromise, rejectPromise) => {
|
||||
resolve = resolvePromise;
|
||||
reject = rejectPromise;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
function candidate(id, overrides = {}) {
|
||||
return {
|
||||
runId: `run-${id}`,
|
||||
attemptId: `attempt-${id}`,
|
||||
projectId: 'default',
|
||||
taskId: `task-${id}`,
|
||||
taskRevision: 'revision-1',
|
||||
executorType: 'local_process',
|
||||
priority: 0,
|
||||
queuedAtMs: NOW,
|
||||
attemptCreatedAtMs: NOW,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function specFor(reference, overrides = {}) {
|
||||
return {
|
||||
runId: reference.runId,
|
||||
attemptId: reference.attemptId,
|
||||
projectId: reference.projectId,
|
||||
taskId: reference.taskId,
|
||||
taskRevision: reference.taskRevision,
|
||||
command: { kind: 'argv', file: '/bin/true', args: ['original'] },
|
||||
environmentPolicy: 'isolated',
|
||||
terminationGraceMs: 100,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function candidateSource(pages, calls = []) {
|
||||
let page = 0;
|
||||
return {
|
||||
async listCandidates(options) {
|
||||
calls.push(options);
|
||||
return pages[page++] ?? [];
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function activeFor(command, completion = Promise.resolve({})) {
|
||||
return {
|
||||
run: { id: command.runId },
|
||||
attempt: { id: command.attemptId },
|
||||
handle: {},
|
||||
completion,
|
||||
async cancel() {},
|
||||
};
|
||||
}
|
||||
|
||||
test('does no plan or activation work when no local candidate exists', async () => {
|
||||
let planCalls = 0;
|
||||
let activationCalls = 0;
|
||||
const dispatcher = new LocalRunDispatcher(
|
||||
candidateSource([[]]),
|
||||
{
|
||||
async prepare() {
|
||||
planCalls += 1;
|
||||
return null;
|
||||
},
|
||||
},
|
||||
{
|
||||
async activateClaimed() {
|
||||
activationCalls += 1;
|
||||
},
|
||||
},
|
||||
{ executorType: 'local_process', clock: { now: () => NOW } },
|
||||
);
|
||||
|
||||
const result = await dispatcher.dispatchOnce();
|
||||
assert.deepEqual([result.status, result.reason], ['idle', 'no_candidates']);
|
||||
assert.deepEqual([planCalls, activationCalls], [0, 0]);
|
||||
});
|
||||
|
||||
test('bounded-pages to a matching executor and activates one cloned pinned plan', async () => {
|
||||
const remote = candidate('remote', { executorType: 'remote_worker' });
|
||||
const local = candidate('local', {
|
||||
queuedAtMs: NOW + 1,
|
||||
attemptCreatedAtMs: NOW + 1,
|
||||
});
|
||||
const pages = [];
|
||||
const sourceSpec = specFor(local, { timeoutMs: 5_000 });
|
||||
const context = { environment: {}, output: { async write() {} } };
|
||||
const completion = deferred();
|
||||
const activations = [];
|
||||
let disposed = 0;
|
||||
const dispatcher = new LocalRunDispatcher(
|
||||
candidateSource([[remote], [local]], pages),
|
||||
{
|
||||
async prepare(reference) {
|
||||
assert.equal(Object.isFrozen(reference), true);
|
||||
assert.equal(reference.attemptId, local.attemptId);
|
||||
return {
|
||||
executionSpec: sourceSpec,
|
||||
context,
|
||||
dispose() {
|
||||
disposed += 1;
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
async activateClaimed(command) {
|
||||
activations.push({ command, spec: command.createSpec() });
|
||||
return activeFor(command, completion.promise);
|
||||
},
|
||||
},
|
||||
{
|
||||
executorType: 'local_process',
|
||||
pageSize: 1,
|
||||
maxPages: 2,
|
||||
clock: { now: () => NOW },
|
||||
},
|
||||
);
|
||||
|
||||
const result = await dispatcher.dispatchOnce();
|
||||
assert.equal(result.status, 'activated');
|
||||
assert.deepEqual(result.stats, {
|
||||
pages: 2,
|
||||
candidatesScanned: 2,
|
||||
executorMismatches: 1,
|
||||
plansUnavailable: 0,
|
||||
activationRaces: 0,
|
||||
});
|
||||
assert.equal(result.truncated, true);
|
||||
assert.equal(pages[1].after.attemptId, remote.attemptId);
|
||||
assert.equal(activations[0].command.timeoutMs, 5_000);
|
||||
assert.equal(activations[0].command.context, context);
|
||||
sourceSpec.command.args[0] = 'mutated';
|
||||
assert.deepEqual(activations[0].spec.command.args, ['original']);
|
||||
assert.equal(disposed, 0);
|
||||
completion.resolve({});
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.equal(disposed, 1);
|
||||
});
|
||||
|
||||
test('continues after a stale candidate and activates the next Attempt', async () => {
|
||||
const first = candidate('first');
|
||||
const second = candidate('second', {
|
||||
queuedAtMs: NOW + 1,
|
||||
attemptCreatedAtMs: NOW + 1,
|
||||
});
|
||||
const attempted = [];
|
||||
const dispatcher = new LocalRunDispatcher(
|
||||
candidateSource([[first, second]]),
|
||||
{
|
||||
async prepare(reference) {
|
||||
return {
|
||||
executionSpec: specFor(reference),
|
||||
context: { environment: {}, output: { async write() {} } },
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
async activateClaimed(command) {
|
||||
attempted.push(command.attemptId);
|
||||
if (command.attemptId === first.attemptId) {
|
||||
throw new PrimaryClaimedRunRejectedError('not_queued');
|
||||
}
|
||||
return activeFor(command);
|
||||
},
|
||||
},
|
||||
{
|
||||
executorType: 'local_process',
|
||||
pageSize: 2,
|
||||
clock: { now: () => NOW },
|
||||
},
|
||||
);
|
||||
|
||||
const result = await dispatcher.dispatchOnce();
|
||||
assert.equal(result.status, 'activated');
|
||||
assert.deepEqual(attempted, [first.attemptId, second.attemptId]);
|
||||
assert.equal(result.stats.activationRaces, 1);
|
||||
});
|
||||
|
||||
test('fails closed on plan drift before activation and disposes its context', async () => {
|
||||
const reference = candidate('drift');
|
||||
let activationCalls = 0;
|
||||
let disposeCalls = 0;
|
||||
const dispatcher = new LocalRunDispatcher(
|
||||
candidateSource([[reference]]),
|
||||
{
|
||||
async prepare() {
|
||||
return {
|
||||
executionSpec: specFor(reference, { taskRevision: 'drifted' }),
|
||||
context: { environment: {}, output: { async write() {} } },
|
||||
dispose() {
|
||||
disposeCalls += 1;
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
async activateClaimed() {
|
||||
activationCalls += 1;
|
||||
},
|
||||
},
|
||||
{ executorType: 'local_process', clock: { now: () => NOW } },
|
||||
);
|
||||
|
||||
await assert.rejects(dispatcher.dispatchOnce(), /identity does not match/);
|
||||
assert.deepEqual([activationCalls, disposeCalls], [0, 1]);
|
||||
});
|
||||
|
||||
test('reports missing plans and rejects unordered or unbounded pages', async () => {
|
||||
const reference = candidate('missing');
|
||||
const missing = new LocalRunDispatcher(
|
||||
candidateSource([[reference]]),
|
||||
{
|
||||
async prepare() {
|
||||
return null;
|
||||
},
|
||||
},
|
||||
{ async activateClaimed() {} },
|
||||
{ executorType: 'local_process', clock: { now: () => NOW } },
|
||||
);
|
||||
const result = await missing.dispatchOnce();
|
||||
assert.deepEqual(
|
||||
[result.status, result.reason],
|
||||
['idle', 'plans_unavailable'],
|
||||
);
|
||||
|
||||
const duplicate = new LocalRunDispatcher(
|
||||
candidateSource([[reference, reference]]),
|
||||
{
|
||||
async prepare() {
|
||||
return null;
|
||||
},
|
||||
},
|
||||
{ async activateClaimed() {} },
|
||||
{
|
||||
executorType: 'local_process',
|
||||
pageSize: 2,
|
||||
clock: { now: () => NOW },
|
||||
},
|
||||
);
|
||||
await assert.rejects(duplicate.dispatchOnce(), /not strictly ordered/);
|
||||
assert.throws(
|
||||
() =>
|
||||
new LocalRunDispatcher(
|
||||
candidateSource([[]]),
|
||||
{
|
||||
async prepare() {
|
||||
return null;
|
||||
},
|
||||
},
|
||||
{ async activateClaimed() {} },
|
||||
{
|
||||
executorType: 'local_process',
|
||||
maxPages: 17,
|
||||
clock: { now: () => NOW },
|
||||
},
|
||||
),
|
||||
RangeError,
|
||||
);
|
||||
});
|
||||
|
||||
test('passes the Artifact identity to activation and awaits failed-plan cleanup', async () => {
|
||||
const reference = candidate('artifact');
|
||||
const order = [];
|
||||
const dispatcher = new LocalRunDispatcher(
|
||||
candidateSource([[reference]]),
|
||||
{
|
||||
async prepare() {
|
||||
return {
|
||||
executionSpec: specFor(reference),
|
||||
context: { environment: {}, output: { async write() {} } },
|
||||
logArtifactId: `local-${'a'.repeat(30)}`,
|
||||
async dispose() {
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
order.push('disposed');
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
async activateClaimed(command) {
|
||||
order.push(command.logArtifactId);
|
||||
throw new PrimaryClaimedRunRejectedError('not_queued');
|
||||
},
|
||||
},
|
||||
{ executorType: 'local_process', clock: { now: () => NOW } },
|
||||
);
|
||||
|
||||
const result = await dispatcher.dispatchOnce();
|
||||
assert.equal(result.status, 'idle');
|
||||
assert.deepEqual(order, [`local-${'a'.repeat(30)}`, 'disposed']);
|
||||
});
|
||||
@@ -0,0 +1,440 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const { QueryTypes, Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const {
|
||||
LOCAL_SECRET_ENVELOPE_TABLE,
|
||||
localSecretEnvelopeMigration,
|
||||
} = require('../../back/migrations/0014-local-secret-envelopes');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
decryptLocalSecretEnvelopeToBuffer,
|
||||
encryptLocalSecretEnvelope,
|
||||
} = require('../../back/runtime/adapters/crypto/aes256GcmLocalSecret');
|
||||
const {
|
||||
LocalSecretKeyringFileProvider,
|
||||
} = require('../../back/runtime/adapters/fs/localSecretKeyringFileProvider');
|
||||
const {
|
||||
LegacySequelizeLocalSecretEnvelopeRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/localSecretEnvelopeRepository');
|
||||
const {
|
||||
EncryptedLocalSecretService,
|
||||
LocalSecretMutationConflictError,
|
||||
LocalSecretVersionConflictError,
|
||||
} = require('../../back/runtime/application/encryptedLocalSecretService');
|
||||
const {
|
||||
LOCAL_SECRET_ALGORITHM,
|
||||
LocalSecretUnavailableError,
|
||||
createLocalSecretRef,
|
||||
parseLocalSecretRef,
|
||||
} = require('../../back/runtime/domain/localSecret');
|
||||
|
||||
const KEY = Buffer.alloc(32, 0x11);
|
||||
|
||||
function keyProvider(key = KEY) {
|
||||
return {
|
||||
async active() {
|
||||
return { keyId: 'edge-key-1', key: Uint8Array.from(key) };
|
||||
},
|
||||
async resolve(keyId) {
|
||||
return keyId === 'edge-key-1'
|
||||
? { keyId, key: Uint8Array.from(key) }
|
||||
: null;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function candidate(projectId = 'default') {
|
||||
return {
|
||||
runId: 'run-secret',
|
||||
attemptId: 'attempt-secret',
|
||||
projectId,
|
||||
taskId: 'task-secret',
|
||||
taskRevision: 'revision-secret',
|
||||
executorType: 'local_process',
|
||||
priority: 0,
|
||||
queuedAtMs: 1_760_000_000_000,
|
||||
attemptCreatedAtMs: 1_760_000_000_000,
|
||||
};
|
||||
}
|
||||
|
||||
async function createStore(t, storage = ':memory:') {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [localSecretEnvelopeMigration],
|
||||
logger: { info() {} },
|
||||
});
|
||||
let nonce = 0;
|
||||
const repository = new LegacySequelizeLocalSecretEnvelopeRepository(database);
|
||||
const service = new EncryptedLocalSecretService(
|
||||
repository,
|
||||
keyProvider(),
|
||||
() => Buffer.alloc(12, nonce++),
|
||||
);
|
||||
return { database, repository, service };
|
||||
}
|
||||
|
||||
test('uses a stable AES-256-GCM envelope vector bound to metadata AAD', () => {
|
||||
const envelope = encryptLocalSecretEnvelope(
|
||||
{
|
||||
projectId: 'default',
|
||||
name: 'TOKEN',
|
||||
version: 1,
|
||||
mutationId: 'mutation-1',
|
||||
keyId: 'edge-key-1',
|
||||
algorithm: LOCAL_SECRET_ALGORITHM,
|
||||
createdAtMs: 100,
|
||||
},
|
||||
'fixed-secret',
|
||||
KEY,
|
||||
() => Buffer.alloc(12, 0x22),
|
||||
);
|
||||
assert.deepEqual(
|
||||
{
|
||||
nonce: envelope.nonce,
|
||||
ciphertext: envelope.ciphertext,
|
||||
authTag: envelope.authTag,
|
||||
},
|
||||
{
|
||||
nonce: 'IiIiIiIiIiIiIiIi',
|
||||
ciphertext: 'cZ5_LKTi7DqGTbtI',
|
||||
authTag: 'tsA93yEy5RJCD5Q6wngfgA',
|
||||
},
|
||||
);
|
||||
const plaintext = decryptLocalSecretEnvelopeToBuffer(envelope, KEY);
|
||||
assert.equal(plaintext.toString('utf8'), 'fixed-secret');
|
||||
plaintext.fill(0);
|
||||
assert.throws(
|
||||
() =>
|
||||
decryptLocalSecretEnvelopeToBuffer(
|
||||
{ ...envelope, projectId: 'another-project' },
|
||||
KEY,
|
||||
),
|
||||
LocalSecretUnavailableError,
|
||||
);
|
||||
});
|
||||
|
||||
test('SecretRef is canonical, project-bound and optionally pins an integer version', () => {
|
||||
const current = createLocalSecretRef({ projectId: 'default', name: 'TOKEN' });
|
||||
const exact = createLocalSecretRef({
|
||||
projectId: 'default',
|
||||
name: 'TOKEN',
|
||||
version: 2,
|
||||
});
|
||||
assert.deepEqual(parseLocalSecretRef(current), {
|
||||
projectId: 'default',
|
||||
name: 'TOKEN',
|
||||
});
|
||||
assert.deepEqual(parseLocalSecretRef(exact), {
|
||||
projectId: 'default',
|
||||
name: 'TOKEN',
|
||||
version: 2,
|
||||
});
|
||||
const unknownField = Buffer.from(
|
||||
JSON.stringify({ projectId: 'default', name: 'TOKEN', extra: true }),
|
||||
).toString('base64url');
|
||||
assert.throws(() => parseLocalSecretRef(`qlsecret:v1:${unknownField}`));
|
||||
assert.throws(() =>
|
||||
createLocalSecretRef({ projectId: 'default', name: 'TOKEN', version: 0 }),
|
||||
);
|
||||
});
|
||||
|
||||
test('creates and rotates append-only versions without storing plaintext', async (t) => {
|
||||
const { database, service } = await createStore(t);
|
||||
const firstPlaintext = 'first-plaintext-never-persisted';
|
||||
const secondPlaintext = 'second-plaintext-never-persisted';
|
||||
const first = await service.put({
|
||||
projectId: 'default',
|
||||
name: 'TOKEN',
|
||||
plaintext: firstPlaintext,
|
||||
mutationId: 'create-token',
|
||||
expectedCurrentVersion: 0,
|
||||
createdAtMs: 100,
|
||||
});
|
||||
const second = await service.put({
|
||||
projectId: 'default',
|
||||
name: 'TOKEN',
|
||||
plaintext: secondPlaintext,
|
||||
mutationId: 'rotate-token',
|
||||
expectedCurrentVersion: 1,
|
||||
createdAtMs: 200,
|
||||
});
|
||||
assert.deepEqual(
|
||||
{ status: first.status, version: first.version },
|
||||
{ status: 'inserted', version: 1 },
|
||||
);
|
||||
assert.deepEqual(
|
||||
{ status: second.status, version: second.version },
|
||||
{ status: 'inserted', version: 2 },
|
||||
);
|
||||
|
||||
const resolved = await service.resolve({
|
||||
candidate: candidate(),
|
||||
secretRefs: [
|
||||
createLocalSecretRef({ projectId: 'default', name: 'TOKEN' }),
|
||||
first.secretRef,
|
||||
second.secretRef,
|
||||
],
|
||||
});
|
||||
assert.deepEqual(resolved, [
|
||||
secondPlaintext,
|
||||
firstPlaintext,
|
||||
secondPlaintext,
|
||||
]);
|
||||
|
||||
const rows = await database.query(
|
||||
`SELECT * FROM "${LOCAL_SECRET_ENVELOPE_TABLE}" ORDER BY version`,
|
||||
{ type: QueryTypes.SELECT },
|
||||
);
|
||||
assert.equal(rows.length, 2);
|
||||
for (const row of rows) {
|
||||
assert.ok(Buffer.isBuffer(row.ciphertext));
|
||||
assert.equal(row.ciphertext.includes(Buffer.from(firstPlaintext)), false);
|
||||
assert.equal(row.ciphertext.includes(Buffer.from(secondPlaintext)), false);
|
||||
assert.equal(JSON.stringify(row).includes(firstPlaintext), false);
|
||||
assert.equal(JSON.stringify(row).includes(secondPlaintext), false);
|
||||
}
|
||||
});
|
||||
|
||||
test('replays mutations idempotently and fences stale rotations', async (t) => {
|
||||
const { service } = await createStore(t);
|
||||
const command = {
|
||||
projectId: 'default',
|
||||
name: 'TOKEN',
|
||||
plaintext: 'same-value',
|
||||
mutationId: 'create-token',
|
||||
expectedCurrentVersion: 0,
|
||||
createdAtMs: 100,
|
||||
};
|
||||
assert.equal((await service.put(command)).status, 'inserted');
|
||||
assert.equal(
|
||||
(await service.put({ ...command, createdAtMs: 999 })).status,
|
||||
'existing',
|
||||
);
|
||||
await assert.rejects(
|
||||
service.put({ ...command, plaintext: 'different-value' }),
|
||||
LocalSecretMutationConflictError,
|
||||
);
|
||||
await assert.rejects(
|
||||
service.put({
|
||||
...command,
|
||||
mutationId: 'stale-create',
|
||||
plaintext: 'stale-value',
|
||||
}),
|
||||
LocalSecretVersionConflictError,
|
||||
);
|
||||
});
|
||||
|
||||
test('batch resolution preserves position, supports missing, and checks Project first', async (t) => {
|
||||
const { repository, service } = await createStore(t);
|
||||
await service.put({
|
||||
projectId: 'default',
|
||||
name: 'A',
|
||||
plaintext: 'value-a',
|
||||
mutationId: 'create-a',
|
||||
expectedCurrentVersion: 0,
|
||||
createdAtMs: 100,
|
||||
});
|
||||
await service.put({
|
||||
projectId: 'default',
|
||||
name: 'B',
|
||||
plaintext: 'value-b',
|
||||
mutationId: 'create-b',
|
||||
expectedCurrentVersion: 0,
|
||||
createdAtMs: 100,
|
||||
});
|
||||
const refs = [
|
||||
{ projectId: 'default', name: 'B' },
|
||||
{ projectId: 'default', name: 'missing' },
|
||||
{ projectId: 'default', name: 'A', version: 1 },
|
||||
];
|
||||
const envelopes = await repository.resolveMany(refs);
|
||||
assert.deepEqual(
|
||||
envelopes.map((item) => item && item.name),
|
||||
['B', null, 'A'],
|
||||
);
|
||||
assert.equal(
|
||||
await service.resolve({
|
||||
candidate: candidate(),
|
||||
secretRefs: refs.map(createLocalSecretRef),
|
||||
}),
|
||||
null,
|
||||
);
|
||||
|
||||
let databaseCalls = 0;
|
||||
const isolated = new EncryptedLocalSecretService(
|
||||
{
|
||||
async append() {
|
||||
throw new Error('not used');
|
||||
},
|
||||
async findByMutation() {
|
||||
throw new Error('not used');
|
||||
},
|
||||
async resolveMany() {
|
||||
databaseCalls += 1;
|
||||
return [];
|
||||
},
|
||||
},
|
||||
keyProvider(),
|
||||
);
|
||||
await assert.rejects(
|
||||
isolated.resolve({
|
||||
candidate: candidate('default'),
|
||||
secretRefs: [
|
||||
createLocalSecretRef({ projectId: 'another', name: 'TOKEN' }),
|
||||
],
|
||||
}),
|
||||
LocalSecretUnavailableError,
|
||||
);
|
||||
assert.equal(databaseCalls, 0);
|
||||
});
|
||||
|
||||
test('corrupt ciphertext and wrong keys fail with a generic non-secret error', async (t) => {
|
||||
const { database, repository, service } = await createStore(t);
|
||||
const plaintext = 'must-not-appear-in-errors';
|
||||
await service.put({
|
||||
projectId: 'default',
|
||||
name: 'TOKEN',
|
||||
plaintext,
|
||||
mutationId: 'create-token',
|
||||
expectedCurrentVersion: 0,
|
||||
createdAtMs: 100,
|
||||
});
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.bulkUpdate(
|
||||
LOCAL_SECRET_ENVELOPE_TABLE,
|
||||
{ auth_tag: Buffer.alloc(16) },
|
||||
{ project_id: 'default', secret_name: 'TOKEN', version: 1 },
|
||||
);
|
||||
const request = {
|
||||
candidate: candidate(),
|
||||
secretRefs: [createLocalSecretRef({ projectId: 'default', name: 'TOKEN' })],
|
||||
};
|
||||
await assert.rejects(service.resolve(request), (error) => {
|
||||
assert.equal(error.constructor, LocalSecretUnavailableError);
|
||||
assert.equal(error.message, 'Local Secret is unavailable');
|
||||
assert.equal(error.message.includes(plaintext), false);
|
||||
assert.equal(error.message.includes('TOKEN'), false);
|
||||
return true;
|
||||
});
|
||||
|
||||
const wrongKeyService = new EncryptedLocalSecretService(
|
||||
repository,
|
||||
keyProvider(Buffer.alloc(32, 0x33)),
|
||||
);
|
||||
await assert.rejects(
|
||||
wrongKeyService.resolve(request),
|
||||
LocalSecretUnavailableError,
|
||||
);
|
||||
});
|
||||
|
||||
test('private keyring reloads rotation and rejects broad modes and symlinks', async (t) => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-secret-keyring-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
const keyringPath = path.join(root, 'keyring.json');
|
||||
const manifest = (activeKeyId, keys) =>
|
||||
JSON.stringify({ version: 1, activeKeyId, keys });
|
||||
const first = Buffer.alloc(32, 0x11).toString('base64url');
|
||||
const second = Buffer.alloc(32, 0x22).toString('base64url');
|
||||
await fs.writeFile(keyringPath, manifest('key-1', { 'key-1': first }), {
|
||||
mode: 0o600,
|
||||
});
|
||||
const provider = new LocalSecretKeyringFileProvider(keyringPath);
|
||||
assert.equal((await provider.active()).keyId, 'key-1');
|
||||
await fs.writeFile(
|
||||
keyringPath,
|
||||
manifest('key-2', { 'key-1': first, 'key-2': second }),
|
||||
);
|
||||
await fs.chmod(keyringPath, 0o600);
|
||||
assert.equal((await provider.active()).keyId, 'key-2');
|
||||
assert.equal(
|
||||
Buffer.from((await provider.resolve('key-1')).key).equals(KEY),
|
||||
true,
|
||||
);
|
||||
|
||||
await fs.chmod(keyringPath, 0o644);
|
||||
await assert.rejects(provider.active(), LocalSecretUnavailableError);
|
||||
await fs.chmod(keyringPath, 0o600);
|
||||
const symlinkPath = path.join(root, 'keyring-link.json');
|
||||
await fs.symlink(keyringPath, symlinkPath);
|
||||
await assert.rejects(
|
||||
new LocalSecretKeyringFileProvider(symlinkPath).active(),
|
||||
LocalSecretUnavailableError,
|
||||
);
|
||||
await fs.writeFile(
|
||||
keyringPath,
|
||||
JSON.stringify({ version: 1, activeKeyId: 'key-1', keys: {}, extra: true }),
|
||||
);
|
||||
await fs.chmod(keyringPath, 0o600);
|
||||
await assert.rejects(provider.active(), LocalSecretUnavailableError);
|
||||
});
|
||||
|
||||
test('concurrent rotations serialize so exactly one expected version wins', async (t) => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-secret-db-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
const storage = path.join(root, 'database.sqlite');
|
||||
const firstStore = await createStore(t, storage);
|
||||
await firstStore.service.put({
|
||||
projectId: 'default',
|
||||
name: 'TOKEN',
|
||||
plaintext: 'version-one',
|
||||
mutationId: 'create-token',
|
||||
expectedCurrentVersion: 0,
|
||||
createdAtMs: 100,
|
||||
});
|
||||
const secondDatabase = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => secondDatabase.close());
|
||||
const secondService = new EncryptedLocalSecretService(
|
||||
new LegacySequelizeLocalSecretEnvelopeRepository(secondDatabase),
|
||||
keyProvider(),
|
||||
);
|
||||
const rotations = await Promise.allSettled([
|
||||
firstStore.service.put({
|
||||
projectId: 'default',
|
||||
name: 'TOKEN',
|
||||
plaintext: 'rotation-a',
|
||||
mutationId: 'rotate-a',
|
||||
expectedCurrentVersion: 1,
|
||||
createdAtMs: 200,
|
||||
}),
|
||||
secondService.put({
|
||||
projectId: 'default',
|
||||
name: 'TOKEN',
|
||||
plaintext: 'rotation-b',
|
||||
mutationId: 'rotate-b',
|
||||
expectedCurrentVersion: 1,
|
||||
createdAtMs: 201,
|
||||
}),
|
||||
]);
|
||||
assert.equal(
|
||||
rotations.filter((item) => item.status === 'fulfilled').length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
rotations.filter((item) => item.status === 'rejected').length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
rotations.find((item) => item.status === 'rejected').reason.constructor,
|
||||
LocalSecretVersionConflictError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,81 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const {
|
||||
LegacyManualPrimaryLogFiles,
|
||||
} = require('../../back/runtime/adapters/legacy/defaultManualPrimaryRuntime');
|
||||
|
||||
const temporaryDirectories = [];
|
||||
|
||||
async function temporaryRoot() {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-manual-log-'));
|
||||
temporaryDirectories.push(root);
|
||||
return root;
|
||||
}
|
||||
|
||||
function input(logName) {
|
||||
return {
|
||||
cron: {
|
||||
id: 31,
|
||||
command: 'demo.js',
|
||||
extraSchedules: [],
|
||||
logName,
|
||||
},
|
||||
acceptedAtMs: 1_750_000_000_000,
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporaryDirectories
|
||||
.splice(0)
|
||||
.map((directory) => fs.rm(directory, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
test('writes manual Primary output below the configured log root', async () => {
|
||||
const root = await temporaryRoot();
|
||||
const logs = new LegacyManualPrimaryLogFiles(root);
|
||||
const prepared = await logs.prepare(input('nested/task-31'));
|
||||
|
||||
assert.match(prepared.logPath, /^nested\/task-31\/.+\.log$/);
|
||||
await prepared.output.write({
|
||||
stream: 'stdout',
|
||||
chunk: Buffer.from('hello primary\n'),
|
||||
observedAtMs: 1_750_000_000_001,
|
||||
});
|
||||
await prepared.close();
|
||||
|
||||
const absolutePath = path.resolve(root, ...prepared.logPath.split('/'));
|
||||
assert.equal(await fs.readFile(absolutePath, 'utf8'), 'hello primary\n');
|
||||
});
|
||||
|
||||
test('resolves the receipt journal only after live completion cleanup', async () => {
|
||||
const root = await temporaryRoot();
|
||||
const receiptRoot = await temporaryRoot();
|
||||
const resolved = [];
|
||||
const logs = new LegacyManualPrimaryLogFiles(root, receiptRoot, {
|
||||
async resolve(attemptId) {
|
||||
resolved.push(attemptId);
|
||||
return true;
|
||||
},
|
||||
});
|
||||
const prepared = await logs.prepare(input('nested/task-31'));
|
||||
const attemptId = '019f7900-0000-7000-8000-000000000099';
|
||||
|
||||
await prepared.completionCommitted(attemptId);
|
||||
assert.deepEqual(resolved, [attemptId]);
|
||||
await prepared.close();
|
||||
});
|
||||
|
||||
test('rejects relative and absolute paths outside the configured log root', async () => {
|
||||
const root = await temporaryRoot();
|
||||
const logs = new LegacyManualPrimaryLogFiles(root);
|
||||
|
||||
await assert.rejects(logs.prepare(input('../outside')));
|
||||
await assert.rejects(logs.prepare(input(path.resolve(root, '../outside'))));
|
||||
});
|
||||
@@ -0,0 +1,392 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const { Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const { runSchemaMigration } = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
runCancellationRequestMigration,
|
||||
} = require('../../back/migrations/0004-run-cancellation-request');
|
||||
const {
|
||||
runAttemptDeadlineMigration,
|
||||
} = require('../../back/migrations/0006-run-attempt-deadline');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeRunRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/runRepository');
|
||||
const {
|
||||
ManualPrimaryOwnershipError,
|
||||
ManualPrimaryRuntime,
|
||||
} = require('../../back/runtime/application/manualPrimaryRuntime');
|
||||
const {
|
||||
installManualPrimaryExecutionRouter,
|
||||
selectManualPrimaryExecutionRouter,
|
||||
stopManualPrimaryAttempt,
|
||||
stopManualPrimaryCron,
|
||||
} = require('../../back/runtime/compatibility/manualPrimaryExecutionBridge');
|
||||
const {
|
||||
parseLegacyLogOutputRef,
|
||||
} = require('../../back/runtime/compatibility/legacyLogOutputRef');
|
||||
const {
|
||||
RuntimeRolloutPolicy,
|
||||
} = require('../../back/runtime/domain/runtimeRollout');
|
||||
|
||||
const databases = [];
|
||||
const restorers = [];
|
||||
const BASE_TIME = 1_750_000_000_000;
|
||||
let idSequence = 1_100;
|
||||
|
||||
function nextId() {
|
||||
idSequence += 1;
|
||||
return '019f7120-0000-7000-8000-' + String(idSequence).padStart(12, '0');
|
||||
}
|
||||
|
||||
function deferred() {
|
||||
let resolve;
|
||||
let reject;
|
||||
const promise = new Promise((resolvePromise, rejectPromise) => {
|
||||
resolve = resolvePromise;
|
||||
reject = rejectPromise;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
async function createRepository() {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
const migrationModel = defineSchemaMigrationModel(database);
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
runCancellationRequestMigration,
|
||||
runAttemptDeadlineMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
databases.push(database);
|
||||
return new LegacySequelizeRunRepository(database);
|
||||
}
|
||||
|
||||
function rollout(mode) {
|
||||
return new RuntimeRolloutPolicy({
|
||||
defaultMode: 'off',
|
||||
origins: { manual: mode },
|
||||
allowLegacyFallbackBeforeStart: false,
|
||||
});
|
||||
}
|
||||
|
||||
class FakeExecutor {
|
||||
type = 'local_process';
|
||||
starts = [];
|
||||
stops = [];
|
||||
completion = deferred();
|
||||
|
||||
capabilities() {
|
||||
return {
|
||||
timeout: true,
|
||||
processGroupTermination: true,
|
||||
workingDirectory: true,
|
||||
isolatedEnvironment: true,
|
||||
memoryLimit: 'none',
|
||||
cpuLimit: 'none',
|
||||
filesystemIsolation: 'none',
|
||||
networkIsolation: 'none',
|
||||
};
|
||||
}
|
||||
|
||||
async start(spec, context) {
|
||||
this.starts.push({ spec, context });
|
||||
if (context.signal && context.signal.aborted) {
|
||||
throw new Error('aborted before fake spawn');
|
||||
}
|
||||
return {
|
||||
id: nextId(),
|
||||
executorType: this.type,
|
||||
runId: spec.runId,
|
||||
attemptId: spec.attemptId,
|
||||
startedAtMs: BASE_TIME + 10,
|
||||
pid: 7301,
|
||||
completion: this.completion.promise,
|
||||
};
|
||||
}
|
||||
|
||||
async stop(handle, reason) {
|
||||
this.stops.push({ handle, reason });
|
||||
return {
|
||||
status: 'termination_requested',
|
||||
termSignalSent: true,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
|
||||
async inspect() {
|
||||
return { status: 'running' };
|
||||
}
|
||||
}
|
||||
|
||||
class FakeLogs {
|
||||
prepareCalls = 0;
|
||||
closeCalls = 0;
|
||||
committedAttempts = [];
|
||||
writes = [];
|
||||
|
||||
async prepare(input) {
|
||||
this.prepareCalls += 1;
|
||||
return {
|
||||
logPath: 'task-' + input.cron.id + '/manual.log',
|
||||
output: {
|
||||
write: async (output) => {
|
||||
this.writes.push(output);
|
||||
},
|
||||
},
|
||||
completionCommitted: async (attemptId) => {
|
||||
this.committedAttempts.push(attemptId);
|
||||
},
|
||||
close: async () => {
|
||||
this.closeCalls += 1;
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function input(cronId = 21) {
|
||||
return {
|
||||
cron: {
|
||||
id: cronId,
|
||||
name: 'manual primary',
|
||||
command: 'demo.js arg',
|
||||
schedule: '0 * * * *',
|
||||
extraSchedules: ['30 * * * *'],
|
||||
taskBefore: 'echo before',
|
||||
taskAfter: 'echo after',
|
||||
workDirectory: '/tmp',
|
||||
logName: 'task-' + cronId,
|
||||
},
|
||||
acceptedAtMs: BASE_TIME,
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
while (restorers.length > 0) restorers.pop()();
|
||||
await Promise.all(databases.splice(0).map((database) => database.close()));
|
||||
});
|
||||
|
||||
test('off mode selects Legacy without preparing logs or spawning', async () => {
|
||||
const repository = await createRepository();
|
||||
const executor = new FakeExecutor();
|
||||
const logs = new FakeLogs();
|
||||
const runtime = new ManualPrimaryRuntime(
|
||||
repository,
|
||||
executor,
|
||||
rollout('off'),
|
||||
logs,
|
||||
{ orchestrator: { createId: nextId } },
|
||||
);
|
||||
|
||||
assert.equal(runtime.ownsNewRuns(), false);
|
||||
await assert.rejects(runtime.start(input()), ManualPrimaryOwnershipError);
|
||||
assert.equal(logs.prepareCalls, 0);
|
||||
assert.equal(executor.starts.length, 0);
|
||||
});
|
||||
|
||||
test('Primary mode builds one durable Run and one legacy-compatible spec', async () => {
|
||||
const repository = await createRepository();
|
||||
const executor = new FakeExecutor();
|
||||
const logs = new FakeLogs();
|
||||
const runtime = new ManualPrimaryRuntime(
|
||||
repository,
|
||||
executor,
|
||||
rollout('primary'),
|
||||
logs,
|
||||
{
|
||||
clock: { now: () => BASE_TIME + 20 },
|
||||
orchestrator: { createId: nextId },
|
||||
},
|
||||
);
|
||||
|
||||
const active = await runtime.start(input());
|
||||
assert.equal(executor.starts.length, 1);
|
||||
assert.equal(active.pid, 7301);
|
||||
assert.equal(active.logPath, 'task-21/manual.log');
|
||||
const started = executor.starts[0];
|
||||
assert.equal(started.spec.command.kind, 'shell');
|
||||
assert.match(
|
||||
started.spec.command.command,
|
||||
/real_log_path='task-21\/manual\.log'/,
|
||||
);
|
||||
assert.match(started.spec.command.command, /task demo\.js arg/);
|
||||
assert.equal(started.spec.environmentPolicy, 'inherit');
|
||||
assert.equal(started.context.signal.aborted, false);
|
||||
|
||||
const run = await repository.findRunById(active.runId);
|
||||
assert.equal(run.executionOwner, 'runtime');
|
||||
assert.equal(run.executionOrigin, 'manual');
|
||||
assert.equal(parseLegacyLogOutputRef(run.outputRef), 'task-21/manual.log');
|
||||
|
||||
executor.completion.resolve({
|
||||
outcome: 'succeeded',
|
||||
startedAtMs: BASE_TIME + 10,
|
||||
finishedAtMs: BASE_TIME + 30,
|
||||
exitCode: 0,
|
||||
});
|
||||
const completed = await active.completion;
|
||||
assert.equal(completed.outcome, 'succeeded');
|
||||
assert.equal(completed.exitCode, 0);
|
||||
assert.deepEqual(logs.committedAttempts, [active.attemptId]);
|
||||
assert.equal(logs.closeCalls, 1);
|
||||
});
|
||||
|
||||
test('routes stop by cron and attempt through the owning Executor', async () => {
|
||||
const repository = await createRepository();
|
||||
const executor = new FakeExecutor();
|
||||
const runtime = new ManualPrimaryRuntime(
|
||||
repository,
|
||||
executor,
|
||||
rollout('primary'),
|
||||
new FakeLogs(),
|
||||
{ orchestrator: { createId: nextId } },
|
||||
);
|
||||
const active = await runtime.start(input(22));
|
||||
|
||||
assert.deepEqual(await runtime.stopCron(999, BASE_TIME + 40), {
|
||||
matched: 0,
|
||||
failed: 0,
|
||||
});
|
||||
assert.deepEqual(
|
||||
await runtime.stopAttempt(active.attemptId, BASE_TIME + 41),
|
||||
{ matched: 1, failed: 0 },
|
||||
);
|
||||
assert.equal(executor.stops.length, 1);
|
||||
assert.equal(executor.stops[0].reason.kind, 'user');
|
||||
|
||||
executor.completion.resolve({
|
||||
outcome: 'cancelled',
|
||||
startedAtMs: BASE_TIME + 10,
|
||||
finishedAtMs: BASE_TIME + 50,
|
||||
exitCode: 143,
|
||||
});
|
||||
assert.equal((await active.completion).outcome, 'cancelled');
|
||||
});
|
||||
|
||||
test('a stop racing log preparation aborts before Executor side effects', async () => {
|
||||
const repository = await createRepository();
|
||||
const executor = new FakeExecutor();
|
||||
const preparing = deferred();
|
||||
const release = deferred();
|
||||
const logs = {
|
||||
async prepare() {
|
||||
preparing.resolve();
|
||||
await release.promise;
|
||||
return {
|
||||
logPath: 'task-24/racing.log',
|
||||
output: { async write() {} },
|
||||
async close() {},
|
||||
};
|
||||
},
|
||||
};
|
||||
const runtime = new ManualPrimaryRuntime(
|
||||
repository,
|
||||
executor,
|
||||
rollout('primary'),
|
||||
logs,
|
||||
{ orchestrator: { createId: nextId } },
|
||||
);
|
||||
|
||||
const starting = runtime.start(input(24));
|
||||
await preparing.promise;
|
||||
assert.deepEqual(await runtime.stopCron(24, BASE_TIME + 1), {
|
||||
matched: 1,
|
||||
failed: 0,
|
||||
});
|
||||
release.resolve();
|
||||
await assert.rejects(starting);
|
||||
assert.equal(executor.starts.length, 1);
|
||||
assert.equal(executor.starts[0].context.signal.aborted, true);
|
||||
});
|
||||
|
||||
test('bridge is default-off and keeps stop routing available for its owner', async () => {
|
||||
assert.equal(selectManualPrimaryExecutionRouter(), undefined);
|
||||
assert.deepEqual(await stopManualPrimaryCron(1, BASE_TIME), {
|
||||
matched: 0,
|
||||
failed: 0,
|
||||
});
|
||||
|
||||
const calls = [];
|
||||
const router = {
|
||||
ownsNewRuns: () => true,
|
||||
async start() {
|
||||
throw new Error('not used');
|
||||
},
|
||||
async stopCron(cronId, requestedAtMs) {
|
||||
calls.push(['cron', cronId, requestedAtMs]);
|
||||
return { matched: 1, failed: 0 };
|
||||
},
|
||||
async stopAttempt(attemptId, requestedAtMs) {
|
||||
calls.push(['attempt', attemptId, requestedAtMs]);
|
||||
return { matched: 1, failed: 0 };
|
||||
},
|
||||
};
|
||||
restorers.push(installManualPrimaryExecutionRouter(router));
|
||||
assert.equal(selectManualPrimaryExecutionRouter(), router);
|
||||
assert.deepEqual(await stopManualPrimaryCron(23, BASE_TIME + 1), {
|
||||
matched: 1,
|
||||
failed: 0,
|
||||
});
|
||||
assert.deepEqual(
|
||||
await stopManualPrimaryAttempt('attempt-23', BASE_TIME + 2),
|
||||
{
|
||||
matched: 1,
|
||||
failed: 0,
|
||||
},
|
||||
);
|
||||
assert.deepEqual(calls, [
|
||||
['cron', 23, BASE_TIME + 1],
|
||||
['attempt', 'attempt-23', BASE_TIME + 2],
|
||||
]);
|
||||
});
|
||||
|
||||
test('rolling new triggers to off preserves the previous in-flight stop owner', async () => {
|
||||
const oldOwner = {
|
||||
ownsNewRuns: () => true,
|
||||
async start() {
|
||||
throw new Error('not used');
|
||||
},
|
||||
async stopCron(cronId) {
|
||||
return {
|
||||
matched: cronId === 25 ? 1 : 0,
|
||||
failed: 0,
|
||||
};
|
||||
},
|
||||
async stopAttempt() {
|
||||
return { matched: 0, failed: 0 };
|
||||
},
|
||||
};
|
||||
const offRouter = {
|
||||
ownsNewRuns: () => false,
|
||||
async start() {
|
||||
throw new Error('not used');
|
||||
},
|
||||
async stopCron() {
|
||||
return { matched: 0, failed: 0 };
|
||||
},
|
||||
async stopAttempt() {
|
||||
return { matched: 0, failed: 0 };
|
||||
},
|
||||
};
|
||||
restorers.push(installManualPrimaryExecutionRouter(oldOwner));
|
||||
restorers.push(installManualPrimaryExecutionRouter(offRouter));
|
||||
|
||||
assert.equal(selectManualPrimaryExecutionRouter(), undefined);
|
||||
assert.deepEqual(await stopManualPrimaryCron(25, BASE_TIME + 3), {
|
||||
matched: 1,
|
||||
failed: 0,
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,326 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
activateManualPrimaryRuntime,
|
||||
} = require('../../back/runtime/application/manualPrimaryRuntimeActivation');
|
||||
const {
|
||||
RuntimeRolloutPolicy,
|
||||
} = require('../../back/runtime/domain/runtimeRollout');
|
||||
|
||||
const NOW = 1_750_000_000_000;
|
||||
|
||||
function loadResult(status, mode = 'off') {
|
||||
return {
|
||||
status,
|
||||
policy: new RuntimeRolloutPolicy({
|
||||
defaultMode: 'off',
|
||||
origins: mode === 'off' ? {} : { manual: mode },
|
||||
allowLegacyFallbackBeforeStart: false,
|
||||
}),
|
||||
audit: {
|
||||
event: 'runtime.rollout_config_evaluated',
|
||||
evaluatedAtMs: NOW,
|
||||
sourcePath: '/data/config/qinglong3-rollout.json',
|
||||
status,
|
||||
revision: 'canary-1',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function cleanRecovery(overrides = {}) {
|
||||
return {
|
||||
pages: 1,
|
||||
scanned: 0,
|
||||
verifiedRunning: 0,
|
||||
recoveredRunning: 0,
|
||||
completedFromReceipt: 0,
|
||||
quarantinedReceipts: 0,
|
||||
publishGraceWaits: 0,
|
||||
markedLost: 0,
|
||||
skipped: 0,
|
||||
ambiguous: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function router() {
|
||||
return {
|
||||
ownsNewRuns: () => true,
|
||||
async start() {
|
||||
throw new Error('not used');
|
||||
},
|
||||
async stopCron() {
|
||||
return { matched: 0, failed: 0 };
|
||||
},
|
||||
async stopAttempt() {
|
||||
return { matched: 0, failed: 0 };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function completionLifecycle(calls) {
|
||||
return {
|
||||
startCompletion() {
|
||||
calls.push('start-completion');
|
||||
return true;
|
||||
},
|
||||
async stopCompletion() {
|
||||
calls.push('stop-completion');
|
||||
return 'drained';
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('activation remains inert unless an accepted manifest selects Primary', async () => {
|
||||
for (const [status, mode] of [
|
||||
['missing', 'off'],
|
||||
['disabled', 'off'],
|
||||
['rejected', 'off'],
|
||||
['accepted', 'shadow'],
|
||||
]) {
|
||||
const calls = [];
|
||||
const result = await activateManualPrimaryRuntime({
|
||||
load: async () => loadResult(status, mode),
|
||||
create() {
|
||||
calls.push('create');
|
||||
throw new Error('must remain inert');
|
||||
},
|
||||
install() {
|
||||
calls.push('install');
|
||||
return () => undefined;
|
||||
},
|
||||
audit(record) {
|
||||
calls.push(record.activation);
|
||||
},
|
||||
});
|
||||
assert.equal(result.active, false);
|
||||
assert.equal(await result.stop(), 'drained');
|
||||
assert.deepEqual(calls, ['not_activated']);
|
||||
}
|
||||
});
|
||||
|
||||
test('activation reconciles before starting lifecycle and installing ownership', async () => {
|
||||
const calls = [];
|
||||
const result = await activateManualPrimaryRuntime({
|
||||
load: async () => loadResult('accepted', 'primary'),
|
||||
create() {
|
||||
calls.push('create');
|
||||
return {
|
||||
router: router(),
|
||||
...completionLifecycle(calls),
|
||||
async reconcile() {
|
||||
calls.push('reconcile');
|
||||
return cleanRecovery({
|
||||
scanned: 2,
|
||||
recoveredRunning: 1,
|
||||
markedLost: 1,
|
||||
});
|
||||
},
|
||||
startTimeout() {
|
||||
calls.push('start-timeout');
|
||||
return true;
|
||||
},
|
||||
async stopTimeout() {
|
||||
calls.push('stop-timeout');
|
||||
return 'drained';
|
||||
},
|
||||
startCancellation() {
|
||||
calls.push('start-cancellation');
|
||||
return true;
|
||||
},
|
||||
async stopCancellation() {
|
||||
calls.push('stop-cancellation');
|
||||
return 'drained';
|
||||
},
|
||||
};
|
||||
},
|
||||
install() {
|
||||
calls.push('install');
|
||||
return () => calls.push('dispose');
|
||||
},
|
||||
audit(record) {
|
||||
calls.push(`audit:${record.activation}`);
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.active, true);
|
||||
assert.deepEqual(calls, [
|
||||
'audit:selected',
|
||||
'create',
|
||||
'reconcile',
|
||||
'audit:reconciled',
|
||||
'start-completion',
|
||||
'start-timeout',
|
||||
'start-cancellation',
|
||||
'install',
|
||||
'audit:activated',
|
||||
]);
|
||||
assert.equal(await result.stop(), 'drained');
|
||||
assert.equal(await result.stop(), 'drained');
|
||||
assert.deepEqual(calls.slice(-5), [
|
||||
'dispose',
|
||||
'stop-timeout',
|
||||
'stop-cancellation',
|
||||
'stop-completion',
|
||||
'audit:stopped',
|
||||
]);
|
||||
});
|
||||
|
||||
test('activation rejects unresolved recovery before starting or installing', async () => {
|
||||
const calls = [];
|
||||
await assert.rejects(
|
||||
activateManualPrimaryRuntime({
|
||||
load: async () => loadResult('accepted', 'primary'),
|
||||
create() {
|
||||
return {
|
||||
router: router(),
|
||||
...completionLifecycle(calls),
|
||||
async reconcile() {
|
||||
return cleanRecovery({ ambiguous: 1 });
|
||||
},
|
||||
startTimeout() {
|
||||
calls.push('start-timeout');
|
||||
return true;
|
||||
},
|
||||
async stopTimeout() {
|
||||
calls.push('stop-timeout');
|
||||
return 'drained';
|
||||
},
|
||||
startCancellation() {
|
||||
calls.push('start');
|
||||
return true;
|
||||
},
|
||||
async stopCancellation() {
|
||||
calls.push('stop');
|
||||
return 'drained';
|
||||
},
|
||||
};
|
||||
},
|
||||
install() {
|
||||
calls.push('install');
|
||||
return () => calls.push('dispose');
|
||||
},
|
||||
audit(record) {
|
||||
calls.push(`audit:${record.activation}`);
|
||||
},
|
||||
}),
|
||||
/did not converge safely/,
|
||||
);
|
||||
assert.deepEqual(calls, ['audit:selected', 'audit:failed']);
|
||||
});
|
||||
|
||||
test('activation rolls back router and lifecycle when final audit fails', async () => {
|
||||
const calls = [];
|
||||
await assert.rejects(
|
||||
activateManualPrimaryRuntime({
|
||||
load: async () => loadResult('accepted', 'primary'),
|
||||
create() {
|
||||
return {
|
||||
router: router(),
|
||||
...completionLifecycle(calls),
|
||||
async reconcile() {
|
||||
return cleanRecovery();
|
||||
},
|
||||
startTimeout() {
|
||||
calls.push('start-timeout');
|
||||
return true;
|
||||
},
|
||||
async stopTimeout() {
|
||||
calls.push('stop-timeout');
|
||||
return 'drained';
|
||||
},
|
||||
startCancellation() {
|
||||
calls.push('start');
|
||||
return true;
|
||||
},
|
||||
async stopCancellation() {
|
||||
calls.push('stop');
|
||||
return 'drained';
|
||||
},
|
||||
};
|
||||
},
|
||||
install() {
|
||||
calls.push('install');
|
||||
return () => calls.push('dispose');
|
||||
},
|
||||
audit(record) {
|
||||
calls.push(`audit:${record.activation}`);
|
||||
if (record.activation === 'activated') {
|
||||
throw new Error('audit unavailable');
|
||||
}
|
||||
},
|
||||
}),
|
||||
/audit unavailable/,
|
||||
);
|
||||
assert.deepEqual(calls, [
|
||||
'audit:selected',
|
||||
'audit:reconciled',
|
||||
'start-completion',
|
||||
'start-timeout',
|
||||
'start',
|
||||
'install',
|
||||
'audit:activated',
|
||||
'dispose',
|
||||
'stop-timeout',
|
||||
'stop',
|
||||
'stop-completion',
|
||||
'audit:failed',
|
||||
]);
|
||||
});
|
||||
|
||||
test('activation stops timeout production when cancellation lifecycle cannot start', async () => {
|
||||
const calls = [];
|
||||
await assert.rejects(
|
||||
activateManualPrimaryRuntime({
|
||||
load: async () => loadResult('accepted', 'primary'),
|
||||
create() {
|
||||
return {
|
||||
router: router(),
|
||||
...completionLifecycle(calls),
|
||||
async reconcile() {
|
||||
return cleanRecovery();
|
||||
},
|
||||
startTimeout() {
|
||||
calls.push('start-timeout');
|
||||
return true;
|
||||
},
|
||||
async stopTimeout() {
|
||||
calls.push('stop-timeout');
|
||||
return 'drained';
|
||||
},
|
||||
startCancellation() {
|
||||
calls.push('start-cancellation');
|
||||
return false;
|
||||
},
|
||||
async stopCancellation() {
|
||||
calls.push('stop-cancellation');
|
||||
return 'drained';
|
||||
},
|
||||
};
|
||||
},
|
||||
install() {
|
||||
calls.push('install');
|
||||
return () => calls.push('dispose');
|
||||
},
|
||||
audit(record) {
|
||||
calls.push(`audit:${record.activation}`);
|
||||
},
|
||||
}),
|
||||
/cancellation lifecycle did not start/,
|
||||
);
|
||||
assert.deepEqual(calls, [
|
||||
'audit:selected',
|
||||
'audit:reconciled',
|
||||
'start-completion',
|
||||
'start-timeout',
|
||||
'start-cancellation',
|
||||
'stop-timeout',
|
||||
'stop-completion',
|
||||
'audit:failed',
|
||||
]);
|
||||
});
|
||||
@@ -0,0 +1,332 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
InvalidMigrationStreamError,
|
||||
MigrationStreamAheadOfCodeError,
|
||||
MigrationStreamChecksumMismatchError,
|
||||
MigrationStreamHistoryCorruptionError,
|
||||
runMigrationStream,
|
||||
} = require('../../back/migrations/core/migrationStream');
|
||||
|
||||
const CHECKSUM_A = 'a'.repeat(64);
|
||||
const CHECKSUM_B = 'b'.repeat(64);
|
||||
|
||||
function stream(migrations) {
|
||||
return {
|
||||
id: 'postgresql-main',
|
||||
dialect: 'postgresql',
|
||||
migrationIdScheme: 'postgres-prefixed',
|
||||
checksumScheme: 'sha256',
|
||||
migrations,
|
||||
};
|
||||
}
|
||||
|
||||
function memoryStore(initial = []) {
|
||||
const records = new Map(
|
||||
initial.map((record) => [record.migrationId, record]),
|
||||
);
|
||||
let ensured = 0;
|
||||
return {
|
||||
records,
|
||||
get ensured() {
|
||||
return ensured;
|
||||
},
|
||||
store: {
|
||||
async ensureHistory() {
|
||||
ensured += 1;
|
||||
},
|
||||
async listAll() {
|
||||
return [...records.values()].map((record) => ({ ...record }));
|
||||
},
|
||||
async findById(id) {
|
||||
const record = records.get(id);
|
||||
return record ? { ...record } : null;
|
||||
},
|
||||
async transaction(work) {
|
||||
const staged = new Map(
|
||||
[...records].map(([id, record]) => [id, { ...record }]),
|
||||
);
|
||||
const result = await work({
|
||||
context: { statements: [] },
|
||||
async findById(id) {
|
||||
const record = staged.get(id);
|
||||
return record ? { ...record } : null;
|
||||
},
|
||||
async insert(record) {
|
||||
if (staged.has(record.migrationId)) {
|
||||
throw new Error('duplicate migration history');
|
||||
}
|
||||
staged.set(record.migrationId, { ...record });
|
||||
},
|
||||
});
|
||||
records.clear();
|
||||
for (const [id, record] of staged) records.set(id, record);
|
||||
return result;
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('runs one prefixed migration atomically and replays without running up', async () => {
|
||||
const state = memoryStore();
|
||||
const statements = [];
|
||||
const logs = [];
|
||||
let calls = 0;
|
||||
const definition = stream([
|
||||
{
|
||||
id: 'pg-0001-schema-history',
|
||||
checksum: CHECKSUM_A,
|
||||
async up(context) {
|
||||
calls += 1;
|
||||
context.statements.push('create schema metadata');
|
||||
statements.push(...context.statements);
|
||||
},
|
||||
},
|
||||
]);
|
||||
await runMigrationStream({
|
||||
stream: definition,
|
||||
store: state.store,
|
||||
clock: () => 100,
|
||||
logger: {
|
||||
info(message) {
|
||||
logs.push(message);
|
||||
},
|
||||
},
|
||||
});
|
||||
await runMigrationStream({
|
||||
stream: definition,
|
||||
store: state.store,
|
||||
clock: () => 200,
|
||||
});
|
||||
assert.equal(calls, 1);
|
||||
assert.deepEqual(statements, ['create schema metadata']);
|
||||
assert.equal(state.ensured, 2);
|
||||
assert.deepEqual(
|
||||
[...state.records.values()],
|
||||
[
|
||||
{
|
||||
streamId: 'postgresql-main',
|
||||
dialect: 'postgresql',
|
||||
migrationId: 'pg-0001-schema-history',
|
||||
checksum: CHECKSUM_A,
|
||||
appliedAtMs: 100,
|
||||
},
|
||||
],
|
||||
);
|
||||
assert.deepEqual(logs, [
|
||||
'[migration:postgresql-main] Applied pg-0001-schema-history',
|
||||
]);
|
||||
});
|
||||
|
||||
test('rejects checksum drift and corrupt dialect history before migration work', async () => {
|
||||
const migrationId = 'pg-0001-schema-history';
|
||||
const mismatch = memoryStore([
|
||||
{
|
||||
streamId: 'postgresql-main',
|
||||
dialect: 'postgresql',
|
||||
migrationId,
|
||||
checksum: CHECKSUM_A,
|
||||
appliedAtMs: 1,
|
||||
},
|
||||
]);
|
||||
await assert.rejects(
|
||||
runMigrationStream({
|
||||
stream: stream([
|
||||
{ id: migrationId, checksum: CHECKSUM_B, async up() {} },
|
||||
]),
|
||||
store: mismatch.store,
|
||||
}),
|
||||
(error) => {
|
||||
assert.ok(error instanceof MigrationStreamChecksumMismatchError);
|
||||
assert.equal(error.migrationId, migrationId);
|
||||
assert.equal(error.databaseChecksum, CHECKSUM_A);
|
||||
assert.equal(error.codeChecksum, CHECKSUM_B);
|
||||
assert.equal(
|
||||
error.message,
|
||||
`Migration checksum mismatch: ${migrationId} ` +
|
||||
`(database=${CHECKSUM_A}, code=${CHECKSUM_B})`,
|
||||
);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
|
||||
const corrupt = memoryStore([
|
||||
{
|
||||
streamId: 'postgresql-main',
|
||||
dialect: 'sqlite',
|
||||
migrationId,
|
||||
checksum: CHECKSUM_A,
|
||||
appliedAtMs: 1,
|
||||
},
|
||||
]);
|
||||
await assert.rejects(
|
||||
runMigrationStream({
|
||||
stream: stream([
|
||||
{ id: migrationId, checksum: CHECKSUM_A, async up() {} },
|
||||
]),
|
||||
store: corrupt.store,
|
||||
}),
|
||||
MigrationStreamHistoryCorruptionError,
|
||||
);
|
||||
});
|
||||
|
||||
test('rolls migration work and history back together when up fails', async () => {
|
||||
const state = memoryStore();
|
||||
await assert.rejects(
|
||||
runMigrationStream({
|
||||
stream: stream([
|
||||
{
|
||||
id: 'pg-0001-schema-history',
|
||||
checksum: CHECKSUM_A,
|
||||
async up(context) {
|
||||
context.statements.push('partial ddl');
|
||||
throw new Error('ddl failed');
|
||||
},
|
||||
},
|
||||
]),
|
||||
store: state.store,
|
||||
}),
|
||||
/ddl failed/,
|
||||
);
|
||||
assert.equal(state.records.size, 0);
|
||||
});
|
||||
|
||||
test('rechecks history inside the transaction for a concurrent leader winner', async () => {
|
||||
const migrationId = 'pg-0001-schema-history';
|
||||
const winner = {
|
||||
streamId: 'postgresql-main',
|
||||
dialect: 'postgresql',
|
||||
migrationId,
|
||||
checksum: CHECKSUM_A,
|
||||
appliedAtMs: 10,
|
||||
};
|
||||
let outsideReads = 0;
|
||||
let upCalls = 0;
|
||||
const store = {
|
||||
async ensureHistory() {},
|
||||
async listAll() {
|
||||
return [];
|
||||
},
|
||||
async findById() {
|
||||
outsideReads += 1;
|
||||
return null;
|
||||
},
|
||||
async transaction(work) {
|
||||
return work({
|
||||
context: {},
|
||||
async findById() {
|
||||
return winner;
|
||||
},
|
||||
async insert() {
|
||||
throw new Error('unreachable');
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
await runMigrationStream({
|
||||
stream: stream([
|
||||
{
|
||||
id: migrationId,
|
||||
checksum: CHECKSUM_A,
|
||||
async up() {
|
||||
upCalls += 1;
|
||||
},
|
||||
},
|
||||
]),
|
||||
store,
|
||||
});
|
||||
assert.equal(outsideReads, 1);
|
||||
assert.equal(upCalls, 0);
|
||||
});
|
||||
|
||||
test('rejects ahead and non-prefix history before new migration work', async () => {
|
||||
const first = {
|
||||
id: 'pg-0001-schema-history',
|
||||
checksum: CHECKSUM_A,
|
||||
async up() {},
|
||||
};
|
||||
const second = {
|
||||
id: 'pg-0002-run-core',
|
||||
checksum: CHECKSUM_B,
|
||||
async up() {},
|
||||
};
|
||||
const ahead = memoryStore([
|
||||
{
|
||||
streamId: 'postgresql-main',
|
||||
dialect: 'postgresql',
|
||||
migrationId: 'pg-0003-ahead',
|
||||
checksum: CHECKSUM_A,
|
||||
appliedAtMs: 1,
|
||||
},
|
||||
]);
|
||||
await assert.rejects(
|
||||
runMigrationStream({ stream: stream([first, second]), store: ahead.store }),
|
||||
MigrationStreamAheadOfCodeError,
|
||||
);
|
||||
|
||||
const gap = memoryStore([
|
||||
{
|
||||
streamId: 'postgresql-main',
|
||||
dialect: 'postgresql',
|
||||
migrationId: second.id,
|
||||
checksum: second.checksum,
|
||||
appliedAtMs: 2,
|
||||
},
|
||||
]);
|
||||
await assert.rejects(
|
||||
runMigrationStream({ stream: stream([first, second]), store: gap.store }),
|
||||
MigrationStreamHistoryCorruptionError,
|
||||
);
|
||||
});
|
||||
|
||||
test('validates stream identity, immutable checksum and unique prefixed ids first', async () => {
|
||||
const state = memoryStore();
|
||||
for (const definition of [
|
||||
{ ...stream([]), id: 'PostgreSQL' },
|
||||
{ ...stream([]), migrationIdScheme: 'sqlite-numbered' },
|
||||
{ ...stream([]), checksumScheme: 'unknown' },
|
||||
stream([{ id: '0001', checksum: CHECKSUM_A, async up() {} }]),
|
||||
stream([{ id: 'pg-0001', checksum: 'short', async up() {} }]),
|
||||
stream([
|
||||
{ id: 'pg-0001', checksum: CHECKSUM_A, async up() {} },
|
||||
{ id: 'pg-0001', checksum: CHECKSUM_A, async up() {} },
|
||||
]),
|
||||
]) {
|
||||
await assert.rejects(
|
||||
runMigrationStream({ stream: definition, store: state.store }),
|
||||
InvalidMigrationStreamError,
|
||||
);
|
||||
}
|
||||
assert.equal(state.ensured, 0);
|
||||
});
|
||||
|
||||
test('accepts the frozen numbered SQLite stream without rewriting its ids', async () => {
|
||||
const state = memoryStore();
|
||||
await runMigrationStream({
|
||||
stream: {
|
||||
id: 'sqlite-main',
|
||||
dialect: 'sqlite',
|
||||
migrationIdScheme: 'sqlite-numbered',
|
||||
checksumScheme: 'legacy-opaque',
|
||||
migrations: [
|
||||
{
|
||||
id: '0001-legacy-columns',
|
||||
checksum: 'legacy checksum v1',
|
||||
async up() {},
|
||||
},
|
||||
],
|
||||
},
|
||||
store: state.store,
|
||||
clock: () => 1,
|
||||
});
|
||||
assert.equal(
|
||||
state.records.get('0001-legacy-columns').migrationId,
|
||||
'0001-legacy-columns',
|
||||
);
|
||||
assert.equal(
|
||||
state.records.get('0001-legacy-columns').checksum,
|
||||
'legacy checksum v1',
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,85 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const { DataTypes, QueryTypes, Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const {
|
||||
legacyColumnsMigration,
|
||||
} = require('../../back/migrations/0001-legacy-columns');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
|
||||
const databases = [];
|
||||
|
||||
async function createDatabase() {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
const migrationModel = defineSchemaMigrationModel(database);
|
||||
databases.push(database);
|
||||
return { database, migrationModel };
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(databases.splice(0).map((database) => database.close()));
|
||||
});
|
||||
|
||||
test('preserves unknown tables, columns, indexes, and rows during legacy migration', async () => {
|
||||
const { database, migrationModel } = await createDatabase();
|
||||
const queryInterface = database.getQueryInterface();
|
||||
|
||||
for (const table of ['CrontabViews', 'Subscriptions', 'Envs']) {
|
||||
await queryInterface.createTable(table, {
|
||||
id: { type: DataTypes.INTEGER, primaryKey: true },
|
||||
});
|
||||
}
|
||||
await queryInterface.createTable('Crontabs', {
|
||||
id: { type: DataTypes.INTEGER, primaryKey: true },
|
||||
external_note: { type: DataTypes.STRING },
|
||||
});
|
||||
await queryInterface.addIndex('Crontabs', ['external_note'], {
|
||||
name: 'external_crontab_note_index',
|
||||
});
|
||||
await queryInterface.createTable('ExternalPluginState', {
|
||||
id: { type: DataTypes.INTEGER, primaryKey: true },
|
||||
payload: { type: DataTypes.TEXT, allowNull: false },
|
||||
});
|
||||
await queryInterface.bulkInsert('Crontabs', [
|
||||
{ id: 1, external_note: 'preserve-me' },
|
||||
]);
|
||||
await queryInterface.bulkInsert('ExternalPluginState', [
|
||||
{ id: 1, payload: '{"source":"fixture"}' },
|
||||
]);
|
||||
|
||||
const options = {
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [legacyColumnsMigration],
|
||||
logger: { info() {} },
|
||||
};
|
||||
await runMigrations(options);
|
||||
await runMigrations(options);
|
||||
|
||||
const tables = await queryInterface.showAllTables();
|
||||
const crontabs = await queryInterface.describeTable('Crontabs');
|
||||
const indexes = await queryInterface.showIndex('Crontabs');
|
||||
const cronRows = await database.query(
|
||||
'SELECT id, external_note FROM Crontabs ORDER BY id',
|
||||
{ type: QueryTypes.SELECT },
|
||||
);
|
||||
const pluginRows = await database.query(
|
||||
'SELECT id, payload FROM ExternalPluginState ORDER BY id',
|
||||
{ type: QueryTypes.SELECT },
|
||||
);
|
||||
|
||||
assert.ok(tables.includes('ExternalPluginState'));
|
||||
assert.ok(crontabs.external_note);
|
||||
assert.ok(indexes.some((index) => index.name === 'external_crontab_note_index'));
|
||||
assert.deepEqual(cronRows, [{ id: 1, external_note: 'preserve-me' }]);
|
||||
assert.deepEqual(pluginRows, [{ id: 1, payload: '{"source":"fixture"}' }]);
|
||||
assert.equal(await migrationModel.count(), 1);
|
||||
});
|
||||
@@ -0,0 +1,892 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const { DataTypes, QueryTypes, Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const {
|
||||
legacyColumnsMigration,
|
||||
} = require('../../back/migrations/0001-legacy-columns');
|
||||
const {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_EVENT_TABLE,
|
||||
RUN_TABLE,
|
||||
runSchemaMigration,
|
||||
} = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
RUNNING_INSTANCE_ATTEMPT_INDEX,
|
||||
RUNNING_INSTANCE_RUN_INDEX,
|
||||
RUNNING_INSTANCE_TABLE,
|
||||
runningInstanceRunReferenceMigration,
|
||||
} = require('../../back/migrations/0003-running-instance-run-reference');
|
||||
const {
|
||||
RUN_CANCELLATION_REQUEST_INDEX,
|
||||
runCancellationRequestMigration,
|
||||
} = require('../../back/migrations/0004-run-cancellation-request');
|
||||
const {
|
||||
RUN_CANCELLATION_DISPATCH_DUE_INDEX,
|
||||
RUN_CANCELLATION_DISPATCH_LEASE_INDEX,
|
||||
RUN_CANCELLATION_DISPATCH_TABLE,
|
||||
runCancellationDispatchMigration,
|
||||
} = require('../../back/migrations/0005-run-cancellation-dispatch');
|
||||
const {
|
||||
RUN_ATTEMPT_DEADLINE_INDEX,
|
||||
runAttemptDeadlineMigration,
|
||||
} = require('../../back/migrations/0006-run-attempt-deadline');
|
||||
const {
|
||||
COMPLETION_RECEIPT_JOURNAL_PURGE_INDEX,
|
||||
COMPLETION_RECEIPT_JOURNAL_SCAN_INDEX,
|
||||
COMPLETION_RECEIPT_JOURNAL_TABLE,
|
||||
completionReceiptJournalMigration,
|
||||
} = require('../../back/migrations/0007-completion-receipt-journal');
|
||||
const {
|
||||
WORKER_REGISTRY_CAPACITY_INDEX,
|
||||
WORKER_REGISTRY_LEASE_INDEX,
|
||||
WORKER_REGISTRY_TABLE,
|
||||
workerRegistryMigration,
|
||||
} = require('../../back/migrations/0008-worker-registry');
|
||||
const {
|
||||
RUN_DISPATCH_LEASE_EXPIRY_INDEX,
|
||||
RUN_DISPATCH_LEASE_TABLE,
|
||||
RUN_DISPATCH_LEASE_TOKEN_INDEX,
|
||||
RUN_DISPATCH_LEASE_WORKER_INDEX,
|
||||
runDispatchLeaseMigration,
|
||||
} = require('../../back/migrations/0009-run-dispatch-lease');
|
||||
const { migrations } = require('../../back/migrations');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
|
||||
const databases = [];
|
||||
|
||||
async function createDatabase() {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
const migrationModel = defineSchemaMigrationModel(database);
|
||||
databases.push(database);
|
||||
return { database, migrationModel };
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(databases.splice(0).map((database) => database.close()));
|
||||
});
|
||||
|
||||
test('runs a migration once and records its checksum', async () => {
|
||||
const { database, migrationModel } = await createDatabase();
|
||||
const queryInterface = database.getQueryInterface();
|
||||
await queryInterface.createTable('Examples', {
|
||||
id: { type: DataTypes.INTEGER, primaryKey: true },
|
||||
});
|
||||
|
||||
let calls = 0;
|
||||
const migrations = [
|
||||
{
|
||||
id: '0001-example',
|
||||
checksum: 'checksum-v1',
|
||||
async up({ queryInterface, transaction }) {
|
||||
calls += 1;
|
||||
await queryInterface.addColumn(
|
||||
'Examples',
|
||||
'name',
|
||||
{ type: DataTypes.STRING },
|
||||
{ transaction },
|
||||
);
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const options = {
|
||||
database,
|
||||
migrationModel,
|
||||
migrations,
|
||||
logger: { info() {} },
|
||||
};
|
||||
await runMigrations(options);
|
||||
await runMigrations(options);
|
||||
|
||||
const description = await queryInterface.describeTable('Examples');
|
||||
const applied = await migrationModel.findByPk('0001-example');
|
||||
assert.ok(description.name);
|
||||
assert.equal(calls, 1);
|
||||
assert.equal(applied.checksum, 'checksum-v1');
|
||||
});
|
||||
|
||||
test('rejects a changed checksum for an applied migration', async () => {
|
||||
const { database, migrationModel } = await createDatabase();
|
||||
const baseOptions = {
|
||||
database,
|
||||
migrationModel,
|
||||
logger: { info() {} },
|
||||
};
|
||||
|
||||
await runMigrations({
|
||||
...baseOptions,
|
||||
migrations: [
|
||||
{
|
||||
id: '0001-example',
|
||||
checksum: 'checksum-v1',
|
||||
async up() {},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
runMigrations({
|
||||
...baseOptions,
|
||||
migrations: [
|
||||
{
|
||||
id: '0001-example',
|
||||
checksum: 'checksum-v2',
|
||||
async up() {},
|
||||
},
|
||||
],
|
||||
}),
|
||||
/Migration checksum mismatch/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rolls back the migration record when migration work fails', async () => {
|
||||
const { database, migrationModel } = await createDatabase();
|
||||
|
||||
await assert.rejects(
|
||||
runMigrations({
|
||||
database,
|
||||
migrationModel,
|
||||
logger: { info() {} },
|
||||
migrations: [
|
||||
{
|
||||
id: '0001-failing',
|
||||
checksum: 'checksum-v1',
|
||||
async up() {
|
||||
throw new Error('migration failed');
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
/migration failed/,
|
||||
);
|
||||
|
||||
assert.equal(await migrationModel.count(), 0);
|
||||
});
|
||||
|
||||
test('rejects duplicate migration ids before touching the database', async () => {
|
||||
const { database, migrationModel } = await createDatabase();
|
||||
const duplicate = {
|
||||
id: '0001-duplicate',
|
||||
checksum: 'checksum-v1',
|
||||
async up() {},
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
runMigrations({
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [duplicate, duplicate],
|
||||
logger: { info() {} },
|
||||
}),
|
||||
/Duplicate migration id/,
|
||||
);
|
||||
});
|
||||
|
||||
test('upgrades the legacy QingLong tables without swallowing schema errors', async () => {
|
||||
const { database, migrationModel } = await createDatabase();
|
||||
const queryInterface = database.getQueryInterface();
|
||||
|
||||
for (const table of ['CrontabViews', 'Subscriptions', 'Crontabs', 'Envs']) {
|
||||
await queryInterface.createTable(table, {
|
||||
id: { type: DataTypes.INTEGER, primaryKey: true },
|
||||
});
|
||||
}
|
||||
|
||||
const options = {
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [legacyColumnsMigration],
|
||||
logger: { info() {} },
|
||||
};
|
||||
await runMigrations(options);
|
||||
await runMigrations(options);
|
||||
|
||||
const cronViews = await queryInterface.describeTable('CrontabViews');
|
||||
const subscriptions = await queryInterface.describeTable('Subscriptions');
|
||||
const crontabs = await queryInterface.describeTable('Crontabs');
|
||||
const envs = await queryInterface.describeTable('Envs');
|
||||
|
||||
assert.ok(cronViews.filterRelation);
|
||||
assert.ok(cronViews.type);
|
||||
assert.ok(subscriptions.proxy);
|
||||
assert.ok(subscriptions.autoAddCron);
|
||||
assert.ok(subscriptions.autoDelCron);
|
||||
assert.ok(crontabs.sub_id);
|
||||
assert.ok(crontabs.extra_schedules);
|
||||
assert.ok(crontabs.task_before);
|
||||
assert.ok(crontabs.task_after);
|
||||
assert.ok(crontabs.log_name);
|
||||
assert.ok(crontabs.allow_multiple_instances);
|
||||
assert.ok(crontabs.work_dir);
|
||||
assert.ok(envs.isPinned);
|
||||
assert.ok(envs.labels);
|
||||
assert.equal(await migrationModel.count(), 1);
|
||||
});
|
||||
|
||||
test('creates the Run aggregate schema with stable uniqueness constraints', async () => {
|
||||
const { database, migrationModel } = await createDatabase();
|
||||
const queryInterface = database.getQueryInterface();
|
||||
const options = {
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [runSchemaMigration],
|
||||
logger: { info() {} },
|
||||
};
|
||||
|
||||
await runMigrations(options);
|
||||
await runMigrations(options);
|
||||
|
||||
const runs = await queryInterface.describeTable(RUN_TABLE);
|
||||
const attempts = await queryInterface.describeTable(RUN_ATTEMPT_TABLE);
|
||||
const events = await queryInterface.describeTable(RUN_EVENT_TABLE);
|
||||
|
||||
for (const column of [
|
||||
'id',
|
||||
'project_id',
|
||||
'task_id',
|
||||
'task_revision',
|
||||
'legacy_cron_id',
|
||||
'execution_origin',
|
||||
'execution_owner',
|
||||
'status',
|
||||
'version',
|
||||
'event_sequence',
|
||||
'created_at_ms',
|
||||
]) {
|
||||
assert.ok(runs[column], `missing Runs.${column}`);
|
||||
}
|
||||
for (const column of [
|
||||
'id',
|
||||
'run_id',
|
||||
'attempt',
|
||||
'status',
|
||||
'executor_type',
|
||||
'callback_token_hash',
|
||||
'callback_sequence',
|
||||
'created_at_ms',
|
||||
]) {
|
||||
assert.ok(attempts[column], `missing RunAttempts.${column}`);
|
||||
}
|
||||
for (const column of [
|
||||
'id',
|
||||
'run_id',
|
||||
'sequence',
|
||||
'type',
|
||||
'dedupe_key',
|
||||
'actor_type',
|
||||
'payload',
|
||||
'created_at_ms',
|
||||
]) {
|
||||
assert.ok(events[column], `missing RunEvents.${column}`);
|
||||
}
|
||||
|
||||
const runId = '019f70a0-0000-7000-8000-000000000001';
|
||||
const attemptId = '019f70a0-0000-7000-8000-000000000002';
|
||||
const baseRun = {
|
||||
id: runId,
|
||||
project_id: 'default',
|
||||
task_id: 'legacy-cron:1',
|
||||
task_revision: 'revision-1',
|
||||
trigger_type: 'manual',
|
||||
execution_origin: 'manual',
|
||||
execution_owner: 'legacy',
|
||||
status: 'created',
|
||||
version: 0,
|
||||
event_sequence: 0,
|
||||
priority: 0,
|
||||
created_at_ms: 1_750_000_000_000,
|
||||
};
|
||||
|
||||
await queryInterface.bulkInsert(RUN_TABLE, [
|
||||
{ ...baseRun, idempotency_key: 'manual-request-1' },
|
||||
]);
|
||||
await queryInterface.bulkInsert(RUN_ATTEMPT_TABLE, [
|
||||
{
|
||||
id: attemptId,
|
||||
run_id: runId,
|
||||
attempt: 1,
|
||||
status: 'claimed',
|
||||
executor_type: 'legacy_local',
|
||||
callback_sequence: 0,
|
||||
created_at_ms: 1_750_000_000_000,
|
||||
},
|
||||
]);
|
||||
await queryInterface.bulkInsert(RUN_EVENT_TABLE, [
|
||||
{
|
||||
id: '019f70a0-0000-7000-8000-000000000003',
|
||||
run_id: runId,
|
||||
sequence: 1,
|
||||
type: 'run.created',
|
||||
dedupe_key: 'create',
|
||||
actor_type: 'compatibility',
|
||||
attempt_id: attemptId,
|
||||
payload: JSON.stringify({ source: 'migration-test' }),
|
||||
created_at_ms: 1_750_000_000_000,
|
||||
},
|
||||
]);
|
||||
|
||||
await assert.rejects(
|
||||
queryInterface.bulkInsert(RUN_TABLE, [
|
||||
{
|
||||
...baseRun,
|
||||
id: '019f70a0-0000-7000-8000-000000000004',
|
||||
idempotency_key: 'manual-request-1',
|
||||
},
|
||||
]),
|
||||
);
|
||||
await assert.rejects(
|
||||
queryInterface.bulkInsert(RUN_TABLE, [
|
||||
{
|
||||
...baseRun,
|
||||
id: '019f70a0-0000-7000-8000-000000000007',
|
||||
version: -1,
|
||||
idempotency_key: 'manual-request-2',
|
||||
},
|
||||
]),
|
||||
);
|
||||
await assert.rejects(
|
||||
queryInterface.bulkInsert(RUN_ATTEMPT_TABLE, [
|
||||
{
|
||||
id: '019f70a0-0000-7000-8000-000000000005',
|
||||
run_id: runId,
|
||||
attempt: 1,
|
||||
status: 'claimed',
|
||||
executor_type: 'legacy_local',
|
||||
callback_sequence: 0,
|
||||
created_at_ms: 1_750_000_000_000,
|
||||
},
|
||||
]),
|
||||
);
|
||||
await assert.rejects(
|
||||
queryInterface.bulkInsert(RUN_EVENT_TABLE, [
|
||||
{
|
||||
id: '019f70a0-0000-7000-8000-000000000006',
|
||||
run_id: runId,
|
||||
sequence: 2,
|
||||
type: 'run.created',
|
||||
dedupe_key: 'create',
|
||||
actor_type: 'compatibility',
|
||||
payload: '{}',
|
||||
created_at_ms: 1_750_000_000_001,
|
||||
},
|
||||
]),
|
||||
);
|
||||
|
||||
assert.equal(await migrationModel.count(), 1);
|
||||
});
|
||||
|
||||
test('creates a bounded completion receipt journal linked to Run Attempts', async () => {
|
||||
const { database, migrationModel } = await createDatabase();
|
||||
const queryInterface = database.getQueryInterface();
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [runSchemaMigration, completionReceiptJournalMigration],
|
||||
logger: { info() {} },
|
||||
});
|
||||
|
||||
const columns = await queryInterface.describeTable(
|
||||
COMPLETION_RECEIPT_JOURNAL_TABLE,
|
||||
);
|
||||
for (const column of [
|
||||
'attempt_id',
|
||||
'run_id',
|
||||
'state',
|
||||
'quarantine_ref',
|
||||
'purge_after_ms',
|
||||
'registered_at_ms',
|
||||
'updated_at_ms',
|
||||
]) {
|
||||
assert.ok(columns[column], `missing journal.${column}`);
|
||||
}
|
||||
const indexes = new Set(
|
||||
(await queryInterface.showIndex(COMPLETION_RECEIPT_JOURNAL_TABLE)).map(
|
||||
(index) => index.name,
|
||||
),
|
||||
);
|
||||
assert.ok(indexes.has(COMPLETION_RECEIPT_JOURNAL_SCAN_INDEX));
|
||||
assert.ok(indexes.has(COMPLETION_RECEIPT_JOURNAL_PURGE_INDEX));
|
||||
|
||||
await assert.rejects(
|
||||
queryInterface.bulkInsert(COMPLETION_RECEIPT_JOURNAL_TABLE, [
|
||||
{
|
||||
attempt_id: '019f70a0-0000-7000-8000-000000000099',
|
||||
run_id: '019f70a0-0000-7000-8000-000000000098',
|
||||
state: 'pending',
|
||||
registered_at_ms: 1,
|
||||
updated_at_ms: 1,
|
||||
},
|
||||
]),
|
||||
/FOREIGN KEY|constraint/i,
|
||||
);
|
||||
});
|
||||
|
||||
test('creates the durable fenced Worker registry and bounded lookup indexes', async () => {
|
||||
const { database, migrationModel } = await createDatabase();
|
||||
const queryInterface = database.getQueryInterface();
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [workerRegistryMigration],
|
||||
logger: { info() {} },
|
||||
});
|
||||
|
||||
const columns = await queryInterface.describeTable(WORKER_REGISTRY_TABLE);
|
||||
for (const column of [
|
||||
'id',
|
||||
'session_id',
|
||||
'generation',
|
||||
'status',
|
||||
'version',
|
||||
'capabilities_json',
|
||||
'capabilities_hash',
|
||||
'max_concurrent_runs',
|
||||
'available_slots',
|
||||
'registered_at_ms',
|
||||
'last_heartbeat_at_ms',
|
||||
'lease_expires_at_ms',
|
||||
'updated_at_ms',
|
||||
]) {
|
||||
assert.ok(columns[column], `missing workers.${column}`);
|
||||
}
|
||||
const indexes = new Set(
|
||||
(await queryInterface.showIndex(WORKER_REGISTRY_TABLE)).map(
|
||||
(index) => index.name,
|
||||
),
|
||||
);
|
||||
assert.ok(indexes.has(WORKER_REGISTRY_LEASE_INDEX));
|
||||
assert.ok(indexes.has(WORKER_REGISTRY_CAPACITY_INDEX));
|
||||
|
||||
await assert.rejects(
|
||||
queryInterface.bulkInsert(WORKER_REGISTRY_TABLE, [
|
||||
{
|
||||
id: 'worker-invalid',
|
||||
session_id: '019f7500-0000-7000-8000-000000000001',
|
||||
generation: 0,
|
||||
status: 'online',
|
||||
version: 0,
|
||||
capabilities_json: '{}',
|
||||
capabilities_hash: 'a'.repeat(64),
|
||||
max_concurrent_runs: 1,
|
||||
available_slots: 1,
|
||||
registered_at_ms: 1,
|
||||
last_heartbeat_at_ms: 1,
|
||||
lease_expires_at_ms: 2,
|
||||
updated_at_ms: 1,
|
||||
},
|
||||
]),
|
||||
/constraint/i,
|
||||
);
|
||||
});
|
||||
|
||||
test('creates attempt-scoped Run dispatch leases with Worker fencing indexes', async () => {
|
||||
const { database, migrationModel } = await createDatabase();
|
||||
const queryInterface = database.getQueryInterface();
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
workerRegistryMigration,
|
||||
runDispatchLeaseMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
|
||||
const columns = await queryInterface.describeTable(RUN_DISPATCH_LEASE_TABLE);
|
||||
for (const column of [
|
||||
'attempt_id',
|
||||
'run_id',
|
||||
'status',
|
||||
'version',
|
||||
'lease_generation',
|
||||
'worker_id',
|
||||
'worker_session_id',
|
||||
'worker_generation',
|
||||
'lease_token',
|
||||
'expires_at_ms',
|
||||
'completed_at_ms',
|
||||
]) {
|
||||
assert.ok(columns[column], `missing ${RUN_DISPATCH_LEASE_TABLE}.${column}`);
|
||||
}
|
||||
const indexes = await queryInterface.showIndex(RUN_DISPATCH_LEASE_TABLE);
|
||||
assert.ok(
|
||||
indexes.some((index) => index.name === RUN_DISPATCH_LEASE_EXPIRY_INDEX),
|
||||
);
|
||||
assert.ok(
|
||||
indexes.some((index) => index.name === RUN_DISPATCH_LEASE_WORKER_INDEX),
|
||||
);
|
||||
assert.ok(
|
||||
indexes.some(
|
||||
(index) => index.name === RUN_DISPATCH_LEASE_TOKEN_INDEX && index.unique,
|
||||
),
|
||||
);
|
||||
await assert.rejects(
|
||||
database.query(
|
||||
`INSERT INTO ${RUN_DISPATCH_LEASE_TABLE} (
|
||||
attempt_id, run_id, status, version, lease_generation,
|
||||
worker_id, worker_session_id, worker_generation, lease_token,
|
||||
acquired_at_ms, renewed_at_ms, expires_at_ms, updated_at_ms
|
||||
) VALUES (
|
||||
'missing-attempt', 'missing-run', 'leased', -1, 0,
|
||||
'missing-worker', '019f7800-0000-7000-8000-000000000001', 0,
|
||||
'lease_token_abcdefghijklmnopqrstuvwxyz0123456789', 1, 1, 2, 1
|
||||
)`,
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('adds stable Run references to legacy RunningInstance rows', async () => {
|
||||
const { database, migrationModel } = await createDatabase();
|
||||
const queryInterface = database.getQueryInterface();
|
||||
await queryInterface.createTable(RUNNING_INSTANCE_TABLE, {
|
||||
id: {
|
||||
type: DataTypes.INTEGER,
|
||||
primaryKey: true,
|
||||
autoIncrement: true,
|
||||
},
|
||||
cron_id: { type: DataTypes.INTEGER, allowNull: false },
|
||||
pid: { type: DataTypes.INTEGER, allowNull: true },
|
||||
log_path: { type: DataTypes.STRING, allowNull: true },
|
||||
started_at: { type: DataTypes.INTEGER, allowNull: false },
|
||||
finished_at: { type: DataTypes.INTEGER, allowNull: true },
|
||||
status: { type: DataTypes.INTEGER, allowNull: false },
|
||||
exit_code: { type: DataTypes.INTEGER, allowNull: true },
|
||||
});
|
||||
await queryInterface.bulkInsert(RUNNING_INSTANCE_TABLE, [
|
||||
{ cron_id: 7, started_at: 1_750_000_000, status: 1 },
|
||||
]);
|
||||
|
||||
const options = {
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [runningInstanceRunReferenceMigration],
|
||||
logger: { info() {} },
|
||||
};
|
||||
await runMigrations(options);
|
||||
await runMigrations(options);
|
||||
|
||||
const columns = await queryInterface.describeTable(RUNNING_INSTANCE_TABLE);
|
||||
assert.ok(columns.run_id);
|
||||
assert.ok(columns.attempt_id);
|
||||
assert.equal(columns.run_id.allowNull, true);
|
||||
assert.equal(columns.attempt_id.allowNull, true);
|
||||
const indexNames = new Set(
|
||||
(await queryInterface.showIndex(RUNNING_INSTANCE_TABLE)).map(
|
||||
(index) => index.name,
|
||||
),
|
||||
);
|
||||
assert.ok(indexNames.has(RUNNING_INSTANCE_RUN_INDEX));
|
||||
assert.ok(indexNames.has(RUNNING_INSTANCE_ATTEMPT_INDEX));
|
||||
|
||||
const fixture = database.define(
|
||||
'RunningInstanceMigrationFixture',
|
||||
{
|
||||
id: { type: DataTypes.INTEGER, primaryKey: true },
|
||||
cron_id: DataTypes.INTEGER,
|
||||
run_id: DataTypes.STRING(36),
|
||||
attempt_id: DataTypes.STRING(36),
|
||||
started_at: DataTypes.INTEGER,
|
||||
status: DataTypes.INTEGER,
|
||||
},
|
||||
{ tableName: RUNNING_INSTANCE_TABLE, timestamps: false },
|
||||
);
|
||||
const legacyRow = await fixture.findByPk(1, { raw: true });
|
||||
assert.equal(legacyRow.run_id, null);
|
||||
assert.equal(legacyRow.attempt_id, null);
|
||||
|
||||
const runId = '019f7110-0000-7000-8000-000000000001';
|
||||
const attemptId = '019f7110-0000-7000-8000-000000000002';
|
||||
await queryInterface.bulkInsert(RUNNING_INSTANCE_TABLE, [
|
||||
{
|
||||
cron_id: 7,
|
||||
run_id: runId,
|
||||
attempt_id: attemptId,
|
||||
started_at: 1_750_000_001,
|
||||
status: 0,
|
||||
},
|
||||
]);
|
||||
await assert.rejects(
|
||||
queryInterface.bulkInsert(RUNNING_INSTANCE_TABLE, [
|
||||
{
|
||||
cron_id: 7,
|
||||
run_id: runId,
|
||||
attempt_id: attemptId,
|
||||
started_at: 1_750_000_002,
|
||||
status: 0,
|
||||
},
|
||||
]),
|
||||
);
|
||||
assert.equal(await migrationModel.count(), 1);
|
||||
});
|
||||
|
||||
test('adds durable cancellation requests without rewriting legacy Run rows', async () => {
|
||||
const { database, migrationModel } = await createDatabase();
|
||||
const queryInterface = database.getQueryInterface();
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [runSchemaMigration],
|
||||
logger: { info() {} },
|
||||
});
|
||||
|
||||
const runId = '019f7110-0000-7000-8000-000000000004';
|
||||
await queryInterface.bulkInsert(RUN_TABLE, [
|
||||
{
|
||||
id: runId,
|
||||
project_id: 'default',
|
||||
task_id: 'legacy-cron:7',
|
||||
task_revision: 'revision-7',
|
||||
trigger_type: 'manual',
|
||||
execution_origin: 'manual',
|
||||
execution_owner: 'runtime',
|
||||
status: 'running',
|
||||
version: 3,
|
||||
event_sequence: 3,
|
||||
priority: 0,
|
||||
created_at_ms: 1_750_000_000_000,
|
||||
started_at_ms: 1_750_000_000_010,
|
||||
},
|
||||
]);
|
||||
|
||||
const options = {
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [runSchemaMigration, runCancellationRequestMigration],
|
||||
logger: { info() {} },
|
||||
};
|
||||
await runMigrations(options);
|
||||
await runMigrations(options);
|
||||
|
||||
const columns = await queryInterface.describeTable(RUN_TABLE);
|
||||
assert.ok(columns.cancel_requested_at_ms);
|
||||
assert.ok(columns.cancel_reason);
|
||||
assert.equal(columns.cancel_requested_at_ms.allowNull, true);
|
||||
assert.equal(columns.cancel_reason.allowNull, true);
|
||||
const indexNames = new Set(
|
||||
(await queryInterface.showIndex(RUN_TABLE)).map((index) => index.name),
|
||||
);
|
||||
assert.ok(indexNames.has(RUN_CANCELLATION_REQUEST_INDEX));
|
||||
|
||||
const [legacyRow] = await database.query(
|
||||
`SELECT cancel_requested_at_ms, cancel_reason FROM ${RUN_TABLE} WHERE id = :runId`,
|
||||
{
|
||||
replacements: { runId },
|
||||
type: QueryTypes.SELECT,
|
||||
},
|
||||
);
|
||||
assert.equal(legacyRow.cancel_requested_at_ms, null);
|
||||
assert.equal(legacyRow.cancel_reason, null);
|
||||
assert.equal(await migrationModel.count(), 2);
|
||||
});
|
||||
|
||||
test('adds nullable Attempt deadlines without rewriting existing attempts', async () => {
|
||||
const { database, migrationModel } = await createDatabase();
|
||||
const queryInterface = database.getQueryInterface();
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [runSchemaMigration],
|
||||
logger: { info() {} },
|
||||
});
|
||||
|
||||
const runId = '019f7110-0000-7000-8000-000000000014';
|
||||
const attemptId = '019f7110-0000-7000-8000-000000000015';
|
||||
await queryInterface.bulkInsert(RUN_TABLE, [
|
||||
{
|
||||
id: runId,
|
||||
project_id: 'default',
|
||||
task_id: 'legacy-cron:8',
|
||||
task_revision: 'revision-8',
|
||||
trigger_type: 'manual',
|
||||
execution_origin: 'manual',
|
||||
execution_owner: 'runtime',
|
||||
status: 'dispatching',
|
||||
version: 2,
|
||||
event_sequence: 2,
|
||||
priority: 0,
|
||||
created_at_ms: 1_750_000_000_000,
|
||||
},
|
||||
]);
|
||||
await queryInterface.bulkInsert(RUN_ATTEMPT_TABLE, [
|
||||
{
|
||||
id: attemptId,
|
||||
run_id: runId,
|
||||
attempt: 1,
|
||||
status: 'starting',
|
||||
executor_type: 'local_process',
|
||||
callback_sequence: 0,
|
||||
created_at_ms: 1_750_000_000_000,
|
||||
},
|
||||
]);
|
||||
|
||||
const options = {
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [runSchemaMigration, runAttemptDeadlineMigration],
|
||||
logger: { info() {} },
|
||||
};
|
||||
await runMigrations(options);
|
||||
await runMigrations(options);
|
||||
|
||||
const columns = await queryInterface.describeTable(RUN_ATTEMPT_TABLE);
|
||||
assert.ok(columns.deadline_at_ms);
|
||||
assert.equal(columns.deadline_at_ms.allowNull, true);
|
||||
const indexNames = new Set(
|
||||
(await queryInterface.showIndex(RUN_ATTEMPT_TABLE)).map(
|
||||
(index) => index.name,
|
||||
),
|
||||
);
|
||||
assert.ok(indexNames.has(RUN_ATTEMPT_DEADLINE_INDEX));
|
||||
const [legacyAttempt] = await database.query(
|
||||
`SELECT deadline_at_ms FROM ${RUN_ATTEMPT_TABLE} WHERE id = :attemptId`,
|
||||
{
|
||||
replacements: { attemptId },
|
||||
type: QueryTypes.SELECT,
|
||||
},
|
||||
);
|
||||
assert.equal(legacyAttempt.deadline_at_ms, null);
|
||||
assert.equal(await migrationModel.count(), 2);
|
||||
});
|
||||
|
||||
test('creates a fenced cancellation dispatch lease bound to a Run Attempt', async () => {
|
||||
const { database, migrationModel } = await createDatabase();
|
||||
const queryInterface = database.getQueryInterface();
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
runCancellationRequestMigration,
|
||||
runCancellationDispatchMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
|
||||
const columns = await queryInterface.describeTable(
|
||||
RUN_CANCELLATION_DISPATCH_TABLE,
|
||||
);
|
||||
for (const column of [
|
||||
'run_id',
|
||||
'attempt_id',
|
||||
'status',
|
||||
'version',
|
||||
'dispatch_count',
|
||||
'next_attempt_at_ms',
|
||||
'lease_owner',
|
||||
'lease_token',
|
||||
'lease_expires_at_ms',
|
||||
'last_result',
|
||||
'last_dispatched_at_ms',
|
||||
'created_at_ms',
|
||||
'updated_at_ms',
|
||||
]) {
|
||||
assert.ok(columns[column], `missing cancellation dispatch.${column}`);
|
||||
}
|
||||
const indexNames = new Set(
|
||||
(await queryInterface.showIndex(RUN_CANCELLATION_DISPATCH_TABLE)).map(
|
||||
(index) => index.name,
|
||||
),
|
||||
);
|
||||
assert.ok(indexNames.has(RUN_CANCELLATION_DISPATCH_DUE_INDEX));
|
||||
assert.ok(indexNames.has(RUN_CANCELLATION_DISPATCH_LEASE_INDEX));
|
||||
|
||||
const runId = '019f7110-0000-7000-8000-000000000011';
|
||||
const attemptId = '019f7110-0000-7000-8000-000000000012';
|
||||
await queryInterface.bulkInsert(RUN_TABLE, [
|
||||
{
|
||||
id: runId,
|
||||
project_id: 'default',
|
||||
task_id: 'legacy-cron:8',
|
||||
task_revision: 'revision-8',
|
||||
trigger_type: 'manual',
|
||||
execution_origin: 'manual',
|
||||
execution_owner: 'runtime',
|
||||
status: 'running',
|
||||
version: 2,
|
||||
event_sequence: 2,
|
||||
priority: 0,
|
||||
created_at_ms: 1_750_000_000_000,
|
||||
cancel_requested_at_ms: 1_750_000_000_100,
|
||||
cancel_reason: 'user',
|
||||
},
|
||||
]);
|
||||
await queryInterface.bulkInsert(RUN_ATTEMPT_TABLE, [
|
||||
{
|
||||
id: attemptId,
|
||||
run_id: runId,
|
||||
attempt: 1,
|
||||
status: 'running',
|
||||
executor_type: 'local_process',
|
||||
callback_sequence: 0,
|
||||
created_at_ms: 1_750_000_000_010,
|
||||
},
|
||||
]);
|
||||
await queryInterface.bulkInsert(RUN_CANCELLATION_DISPATCH_TABLE, [
|
||||
{
|
||||
run_id: runId,
|
||||
attempt_id: attemptId,
|
||||
status: 'pending',
|
||||
version: 0,
|
||||
dispatch_count: 0,
|
||||
next_attempt_at_ms: 1_750_000_000_100,
|
||||
created_at_ms: 1_750_000_000_100,
|
||||
updated_at_ms: 1_750_000_000_100,
|
||||
},
|
||||
]);
|
||||
await assert.rejects(
|
||||
queryInterface.bulkInsert(RUN_CANCELLATION_DISPATCH_TABLE, [
|
||||
{
|
||||
run_id: 'missing-run',
|
||||
attempt_id: attemptId,
|
||||
status: 'pending',
|
||||
version: 0,
|
||||
dispatch_count: 0,
|
||||
created_at_ms: 1_750_000_000_100,
|
||||
updated_at_ms: 1_750_000_000_100,
|
||||
},
|
||||
]),
|
||||
);
|
||||
await assert.rejects(
|
||||
queryInterface.bulkUpdate(
|
||||
RUN_CANCELLATION_DISPATCH_TABLE,
|
||||
{ version: -1 },
|
||||
{ run_id: runId },
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('runs the registered migration chain against a legacy database fixture', async () => {
|
||||
const { database, migrationModel } = await createDatabase();
|
||||
const queryInterface = database.getQueryInterface();
|
||||
|
||||
for (const table of ['CrontabViews', 'Subscriptions', 'Crontabs', 'Envs']) {
|
||||
await queryInterface.createTable(table, {
|
||||
id: { type: DataTypes.INTEGER, primaryKey: true },
|
||||
});
|
||||
}
|
||||
|
||||
const options = {
|
||||
database,
|
||||
migrationModel,
|
||||
logger: { info() {} },
|
||||
};
|
||||
await runMigrations(options);
|
||||
await runMigrations(options);
|
||||
|
||||
const tables = new Set(await queryInterface.showAllTables());
|
||||
assert.ok(tables.has(RUN_TABLE));
|
||||
assert.ok(tables.has(RUN_ATTEMPT_TABLE));
|
||||
assert.ok(tables.has(RUN_EVENT_TABLE));
|
||||
assert.ok(tables.has(RUN_CANCELLATION_DISPATCH_TABLE));
|
||||
assert.equal(await migrationModel.count(), migrations.length);
|
||||
});
|
||||
@@ -0,0 +1,108 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { afterEach, test } = require('node:test');
|
||||
|
||||
let nodeSqlite;
|
||||
try {
|
||||
nodeSqlite = require('node:sqlite');
|
||||
} catch {
|
||||
nodeSqlite = undefined;
|
||||
}
|
||||
|
||||
const temporaryDirectories = [];
|
||||
const nodeMajor = Number(process.versions.node.split('.')[0]);
|
||||
|
||||
async function temporaryRoot() {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-backup-'));
|
||||
temporaryDirectories.push(root);
|
||||
return root;
|
||||
}
|
||||
|
||||
function sha256(value) {
|
||||
return createHash('sha256').update(value).digest('hex');
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporaryDirectories
|
||||
.splice(0)
|
||||
.map((directory) => fs.rm(directory, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
test(
|
||||
'Node 24 Online Backup produces an integrity-checked restorable snapshot',
|
||||
{ skip: nodeMajor < 24 },
|
||||
async () => {
|
||||
assert.equal(typeof nodeSqlite?.backup, 'function');
|
||||
const root = await temporaryRoot();
|
||||
const sourcePath = path.join(root, 'source.sqlite');
|
||||
const backupPath = path.join(root, 'backup.sqlite');
|
||||
const restoredPath = path.join(root, 'restored.sqlite');
|
||||
const source = new nodeSqlite.DatabaseSync(sourcePath);
|
||||
source.exec(
|
||||
'PRAGMA journal_mode=WAL; CREATE TABLE facts(id INTEGER PRIMARY KEY, value TEXT NOT NULL);',
|
||||
);
|
||||
const insert = source.prepare('INSERT INTO facts(value) VALUES (?)');
|
||||
for (let index = 0; index < 2_000; index += 1) {
|
||||
insert.run('fact-' + index + '-' + 'x'.repeat(128));
|
||||
}
|
||||
|
||||
let progressCalls = 0;
|
||||
let insertedDuringBackup = false;
|
||||
const pages = await nodeSqlite.backup(source, backupPath, {
|
||||
rate: 1,
|
||||
progress() {
|
||||
progressCalls += 1;
|
||||
if (!insertedDuringBackup) {
|
||||
insertedDuringBackup = true;
|
||||
insert.run('inserted-during-online-backup');
|
||||
}
|
||||
},
|
||||
});
|
||||
source.close();
|
||||
|
||||
assert.ok(pages > 0);
|
||||
assert.ok(progressCalls > 0);
|
||||
assert.equal(insertedDuringBackup, true);
|
||||
const backup = new nodeSqlite.DatabaseSync(backupPath, {
|
||||
readOnly: true,
|
||||
});
|
||||
assert.equal(
|
||||
backup.prepare('PRAGMA integrity_check').get().integrity_check,
|
||||
'ok',
|
||||
);
|
||||
assert.equal(
|
||||
backup
|
||||
.prepare(
|
||||
"SELECT COUNT(*) AS count FROM facts WHERE value = 'inserted-during-online-backup'",
|
||||
)
|
||||
.get().count,
|
||||
1,
|
||||
);
|
||||
const expectedCount = backup
|
||||
.prepare('SELECT COUNT(*) AS count FROM facts')
|
||||
.get().count;
|
||||
backup.close();
|
||||
|
||||
await fs.copyFile(backupPath, restoredPath);
|
||||
const restoredBytes = await fs.readFile(restoredPath);
|
||||
const backupBytes = await fs.readFile(backupPath);
|
||||
assert.equal(sha256(restoredBytes), sha256(backupBytes));
|
||||
const restored = new nodeSqlite.DatabaseSync(restoredPath, {
|
||||
readOnly: true,
|
||||
});
|
||||
assert.equal(
|
||||
restored.prepare('PRAGMA quick_check').get().quick_check,
|
||||
'ok',
|
||||
);
|
||||
assert.equal(
|
||||
restored.prepare('SELECT COUNT(*) AS count FROM facts').get().count,
|
||||
expectedCount,
|
||||
);
|
||||
restored.close();
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,170 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { execFileSync } = require('node:child_process');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const { DataTypes, Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_EVENT_TABLE,
|
||||
RUN_TABLE,
|
||||
} = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
RUN_CANCELLATION_DISPATCH_TABLE,
|
||||
} = require('../../back/migrations/0005-run-cancellation-dispatch');
|
||||
const { migrations } = require('../../back/migrations');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
parseArguments: parseLegacySchemaAuditArguments,
|
||||
} = require('../../scripts/ql3-schema-audit.cjs');
|
||||
|
||||
let nodeSqlite;
|
||||
try {
|
||||
nodeSqlite = require('node:sqlite');
|
||||
} catch {
|
||||
nodeSqlite = undefined;
|
||||
}
|
||||
|
||||
const nodeMajor = Number(process.versions.node.split('.')[0]);
|
||||
const temporaryDirectories = [];
|
||||
|
||||
test('legacy schema audit never selects a live database implicitly', () => {
|
||||
assert.throws(
|
||||
() => parseLegacySchemaAuditArguments([]),
|
||||
/requires an explicit --database path/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
parseLegacySchemaAuditArguments([
|
||||
'--database=/tmp/a.sqlite',
|
||||
'--database=/tmp/b.sqlite',
|
||||
]),
|
||||
/must not be duplicated/,
|
||||
);
|
||||
});
|
||||
|
||||
async function createMigratedDatabase() {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-node-sqlite-'));
|
||||
temporaryDirectories.push(root);
|
||||
const storage = path.join(root, 'database.sqlite');
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
const queryInterface = database.getQueryInterface();
|
||||
for (const table of ['CrontabViews', 'Subscriptions', 'Crontabs', 'Envs']) {
|
||||
await queryInterface.createTable(table, {
|
||||
id: { type: DataTypes.INTEGER, primaryKey: true },
|
||||
});
|
||||
}
|
||||
await queryInterface.createTable('RunningInstances', {
|
||||
id: { type: DataTypes.INTEGER, primaryKey: true },
|
||||
started_at: { type: DataTypes.INTEGER, allowNull: false },
|
||||
});
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations,
|
||||
logger: { info() {} },
|
||||
});
|
||||
await database.close();
|
||||
return storage;
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporaryDirectories
|
||||
.splice(0)
|
||||
.map((directory) => fs.rm(directory, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
test(
|
||||
'Node 24 opens the Sequelize migration chain with defensive node:sqlite options',
|
||||
{ skip: nodeMajor < 24 },
|
||||
async () => {
|
||||
assert.equal(typeof nodeSqlite?.DatabaseSync, 'function');
|
||||
const storage = await createMigratedDatabase();
|
||||
const database = new nodeSqlite.DatabaseSync(storage, {
|
||||
allowExtension: false,
|
||||
allowUnknownNamedParameters: false,
|
||||
defensive: true,
|
||||
enableDoubleQuotedStringLiterals: false,
|
||||
enableForeignKeyConstraints: true,
|
||||
readOnly: true,
|
||||
timeout: 1_000,
|
||||
});
|
||||
|
||||
try {
|
||||
assert.equal(
|
||||
database.prepare('PRAGMA integrity_check').get().integrity_check,
|
||||
'ok',
|
||||
);
|
||||
const tables = database
|
||||
.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name",
|
||||
)
|
||||
.all()
|
||||
.map((row) => row.name);
|
||||
for (const table of [
|
||||
RUN_TABLE,
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_EVENT_TABLE,
|
||||
RUN_CANCELLATION_DISPATCH_TABLE,
|
||||
'SchemaMigrations',
|
||||
]) {
|
||||
assert.ok(tables.includes(table), `missing ${table}`);
|
||||
}
|
||||
|
||||
const runColumns = database
|
||||
.prepare(`PRAGMA table_info(${RUN_TABLE})`)
|
||||
.all()
|
||||
.map((row) => row.name);
|
||||
for (const column of [
|
||||
'id',
|
||||
'status',
|
||||
'version',
|
||||
'event_sequence',
|
||||
'cancel_requested_at_ms',
|
||||
]) {
|
||||
assert.ok(runColumns.includes(column), `missing Runs.${column}`);
|
||||
}
|
||||
const attemptColumns = database
|
||||
.prepare(`PRAGMA table_info(${RUN_ATTEMPT_TABLE})`)
|
||||
.all()
|
||||
.map((row) => row.name);
|
||||
assert.ok(
|
||||
attemptColumns.includes('deadline_at_ms'),
|
||||
'missing RunAttempts.deadline_at_ms',
|
||||
);
|
||||
|
||||
assert.throws(
|
||||
() => database.exec('CREATE TABLE must_not_write(id INTEGER)'),
|
||||
/read.?only/i,
|
||||
);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
|
||||
const audit = JSON.parse(
|
||||
execFileSync(
|
||||
process.execPath,
|
||||
[
|
||||
path.resolve(__dirname, '../../scripts/ql3-schema-audit.cjs'),
|
||||
'--json',
|
||||
`--database=${storage}`,
|
||||
],
|
||||
{ encoding: 'utf8' },
|
||||
),
|
||||
);
|
||||
assert.equal(audit.compatible, true);
|
||||
assert.equal(audit.driftDetected, false);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,77 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { EventEmitter } = require('node:events');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
observeLegacyChildProcess,
|
||||
} = require('../../back/runtime/compatibility/observeLegacyChildProcess');
|
||||
|
||||
function recordingObservation() {
|
||||
const facts = [];
|
||||
return {
|
||||
facts,
|
||||
observation: {
|
||||
spawned(fact) {
|
||||
facts.push(['spawned', fact]);
|
||||
},
|
||||
running(fact) {
|
||||
facts.push(['running', fact]);
|
||||
},
|
||||
startFailed(fact) {
|
||||
facts.push(['start_failed', fact]);
|
||||
},
|
||||
exited(fact) {
|
||||
facts.push(['exited', fact]);
|
||||
},
|
||||
cancelled(fact) {
|
||||
facts.push(['cancelled', fact]);
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('observes an existing child lifecycle without creating another process', () => {
|
||||
const child = new EventEmitter();
|
||||
child.pid = 4242;
|
||||
const { facts, observation } = recordingObservation();
|
||||
const timestamps = [100, 101];
|
||||
|
||||
observeLegacyChildProcess(child, observation, {
|
||||
now: () => timestamps.shift(),
|
||||
logArtifactId: 'legacy-log-1',
|
||||
});
|
||||
child.emit('spawn');
|
||||
child.emit('exit', 0, null);
|
||||
|
||||
assert.deepEqual(facts, [
|
||||
[
|
||||
'spawned',
|
||||
{
|
||||
atMs: 100,
|
||||
pid: 4242,
|
||||
executorHandle: 'legacy-local:4242',
|
||||
logArtifactId: 'legacy-log-1',
|
||||
},
|
||||
],
|
||||
['running', { atMs: 100 }],
|
||||
['exited', { atMs: 101, exitCode: 0 }],
|
||||
]);
|
||||
});
|
||||
|
||||
test('maps child errors and signals to bounded observation facts', () => {
|
||||
const child = new EventEmitter();
|
||||
const { facts, observation } = recordingObservation();
|
||||
const timestamps = [200, 201];
|
||||
|
||||
observeLegacyChildProcess(child, observation, {
|
||||
now: () => timestamps.shift(),
|
||||
});
|
||||
child.emit('error', new Error('must not leak'));
|
||||
child.emit('exit', null, 'SIGTERM');
|
||||
|
||||
assert.deepEqual(facts, [
|
||||
['start_failed', { atMs: 200, errorCode: 'LEGACY_PROCESS_ERROR' }],
|
||||
['exited', { atMs: 201, exitCode: null, signal: 'SIGTERM' }],
|
||||
]);
|
||||
});
|
||||
@@ -0,0 +1,145 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
createLocalProcessDurableHandle,
|
||||
} = require('../../back/runtime/adapters/local-process/localProcessIdentity');
|
||||
const {
|
||||
LocalProcessPersistedExecutionController,
|
||||
} = require('../../back/runtime/adapters/local-process/persistedLocalProcessController');
|
||||
|
||||
const IDENTITY = {
|
||||
platform: 'linux',
|
||||
bootId: '11111111-2222-3333-4444-555555555555',
|
||||
pid: 4321,
|
||||
processGroupId: 4321,
|
||||
startTimeTicks: '123456',
|
||||
};
|
||||
const HANDLE = createLocalProcessDurableHandle('handle-1', IDENTITY);
|
||||
|
||||
function controller(inspections, overrides = {}) {
|
||||
const signals = [];
|
||||
let index = 0;
|
||||
return {
|
||||
signals,
|
||||
value: new LocalProcessPersistedExecutionController({
|
||||
identityProvider: {
|
||||
async capture() {
|
||||
throw new Error('not used');
|
||||
},
|
||||
async inspect(identity) {
|
||||
assert.deepEqual(identity, IDENTITY);
|
||||
const status = inspections[Math.min(index, inspections.length - 1)];
|
||||
index += 1;
|
||||
return { status };
|
||||
},
|
||||
},
|
||||
sendSignal(pid, signal) {
|
||||
signals.push({ pid, signal });
|
||||
},
|
||||
graceMs: 10,
|
||||
pollIntervalMs: 5,
|
||||
sleep: async () => undefined,
|
||||
...overrides,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
const REASON = { kind: 'user', requestedAtMs: 1_750_000_000_000 };
|
||||
|
||||
test('refuses malformed, PID-mismatched, and non-leader handles without signals', async () => {
|
||||
const instance = controller(['running']);
|
||||
assert.equal(
|
||||
(await instance.value.stop({ durableHandle: 'bad', reason: REASON }))
|
||||
.status,
|
||||
'invalid',
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await instance.value.stop({
|
||||
durableHandle: HANDLE,
|
||||
expectedPid: 9999,
|
||||
reason: REASON,
|
||||
})
|
||||
).status,
|
||||
'pid_mismatch',
|
||||
);
|
||||
const nonLeader = createLocalProcessDurableHandle('handle-2', {
|
||||
...IDENTITY,
|
||||
processGroupId: 4000,
|
||||
});
|
||||
assert.equal(
|
||||
(await instance.value.stop({ durableHandle: nonLeader, reason: REASON }))
|
||||
.status,
|
||||
'identity_mismatch',
|
||||
);
|
||||
assert.deepEqual(instance.signals, []);
|
||||
});
|
||||
|
||||
test('sends TERM only when the persisted process exits during grace', async () => {
|
||||
const instance = controller(['running', 'exited']);
|
||||
const result = await instance.value.stop({
|
||||
durableHandle: HANDLE,
|
||||
expectedPid: 4321,
|
||||
reason: REASON,
|
||||
});
|
||||
assert.deepEqual(result, {
|
||||
status: 'termination_requested',
|
||||
termSignalSent: true,
|
||||
killSignalSent: false,
|
||||
});
|
||||
assert.deepEqual(instance.signals, [{ pid: -4321, signal: 'SIGTERM' }]);
|
||||
});
|
||||
|
||||
test('revalidates identity before escalating a persistent process to KILL', async () => {
|
||||
const persistent = controller(['running', 'running', 'running', 'running']);
|
||||
const killed = await persistent.value.stop({
|
||||
durableHandle: HANDLE,
|
||||
reason: REASON,
|
||||
});
|
||||
assert.equal(killed.killSignalSent, true);
|
||||
assert.deepEqual(persistent.signals, [
|
||||
{ pid: -4321, signal: 'SIGTERM' },
|
||||
{ pid: -4321, signal: 'SIGKILL' },
|
||||
]);
|
||||
|
||||
const changed = controller([
|
||||
'running',
|
||||
'running',
|
||||
'running',
|
||||
'identity_mismatch',
|
||||
]);
|
||||
const refused = await changed.value.stop({
|
||||
durableHandle: HANDLE,
|
||||
reason: REASON,
|
||||
});
|
||||
assert.equal(refused.status, 'identity_mismatch');
|
||||
assert.equal(refused.killSignalSent, false);
|
||||
assert.deepEqual(changed.signals, [{ pid: -4321, signal: 'SIGTERM' }]);
|
||||
});
|
||||
|
||||
test('treats an already exited process and ESRCH as idempotent completion', async () => {
|
||||
const exited = controller(['exited']);
|
||||
assert.deepEqual(
|
||||
await exited.value.stop({ durableHandle: HANDLE, reason: REASON }),
|
||||
{
|
||||
status: 'already_exited',
|
||||
termSignalSent: false,
|
||||
killSignalSent: false,
|
||||
},
|
||||
);
|
||||
|
||||
const noProcess = controller(['running'], {
|
||||
sendSignal() {
|
||||
const error = new Error('gone');
|
||||
error.code = 'ESRCH';
|
||||
throw error;
|
||||
},
|
||||
});
|
||||
assert.equal(
|
||||
(await noProcess.value.stop({ durableHandle: HANDLE, reason: REASON }))
|
||||
.status,
|
||||
'already_exited',
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,214 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PinnedTaskLocalRunDispatchPlanSource,
|
||||
} = require('../../back/runtime/application/pinnedTaskLocalRunDispatchPlanSource');
|
||||
const {
|
||||
MAX_EXECUTION_ENVIRONMENT_ENTRIES,
|
||||
normalizeExecutionContext,
|
||||
} = require('../../back/runtime/domain/executionContext');
|
||||
|
||||
function candidate() {
|
||||
return {
|
||||
runId: 'run-pinned',
|
||||
attemptId: 'attempt-2',
|
||||
projectId: 'default',
|
||||
taskId: 'task-pinned',
|
||||
taskRevision: 'revision-7',
|
||||
executorType: 'local_process',
|
||||
priority: 0,
|
||||
queuedAtMs: 1_760_000_000_000,
|
||||
attemptCreatedAtMs: 1_760_000_000_000,
|
||||
};
|
||||
}
|
||||
|
||||
function revision(overrides = {}) {
|
||||
return {
|
||||
projectId: 'default',
|
||||
taskId: 'task-pinned',
|
||||
taskRevision: 'revision-7',
|
||||
executorType: 'local_process',
|
||||
execution: {
|
||||
command: { kind: 'argv', file: '/bin/true', args: ['original'] },
|
||||
environmentPolicy: 'isolated',
|
||||
timeoutMs: 5_000,
|
||||
terminationGraceMs: 100,
|
||||
},
|
||||
contextRef: 'context:task-pinned:revision-7',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('materializes one pinned revision with fresh bounded context capabilities', async () => {
|
||||
const sourceRevision = revision();
|
||||
sourceRevision.execution.runId = 'must-not-override-attempt-identity';
|
||||
const environment = { TOKEN: 'in-memory-secret' };
|
||||
const output = { async write() {} };
|
||||
const revisionRequests = [];
|
||||
const contextRequests = [];
|
||||
const dispose = () => undefined;
|
||||
const source = new PinnedTaskLocalRunDispatchPlanSource(
|
||||
{
|
||||
async resolve(request) {
|
||||
revisionRequests.push(request);
|
||||
assert.equal(Object.isFrozen(request), true);
|
||||
return sourceRevision;
|
||||
},
|
||||
},
|
||||
{
|
||||
async prepare(request) {
|
||||
contextRequests.push(request);
|
||||
assert.equal(Object.isFrozen(request), true);
|
||||
assert.equal(Object.isFrozen(request.candidate), true);
|
||||
return { context: { environment, output }, dispose };
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const plan = await source.prepare(candidate());
|
||||
assert.deepEqual(revisionRequests, [
|
||||
{
|
||||
projectId: 'default',
|
||||
taskId: 'task-pinned',
|
||||
taskRevision: 'revision-7',
|
||||
},
|
||||
]);
|
||||
assert.equal(contextRequests[0].contextRef, sourceRevision.contextRef);
|
||||
assert.equal(plan.executionSpec.runId, 'run-pinned');
|
||||
assert.equal(plan.executionSpec.attemptId, 'attempt-2');
|
||||
assert.equal(plan.executionSpec.timeoutMs, 5_000);
|
||||
assert.equal(plan.context.output, output);
|
||||
assert.equal(Object.isFrozen(plan.context.environment), true);
|
||||
assert.equal(plan.dispose, dispose);
|
||||
|
||||
sourceRevision.execution.command.args[0] = 'mutated';
|
||||
environment.TOKEN = 'mutated-secret';
|
||||
assert.deepEqual(plan.executionSpec.command.args, ['original']);
|
||||
assert.equal(plan.context.environment.TOKEN, 'in-memory-secret');
|
||||
});
|
||||
|
||||
test('never falls back when the exact revision or context is unavailable', async () => {
|
||||
let contextCalls = 0;
|
||||
const missingRevision = new PinnedTaskLocalRunDispatchPlanSource(
|
||||
{
|
||||
async resolve() {
|
||||
return null;
|
||||
},
|
||||
},
|
||||
{
|
||||
async prepare() {
|
||||
contextCalls += 1;
|
||||
return null;
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(await missingRevision.prepare(candidate()), null);
|
||||
assert.equal(contextCalls, 0);
|
||||
|
||||
const missingContext = new PinnedTaskLocalRunDispatchPlanSource(
|
||||
{
|
||||
async resolve() {
|
||||
return revision();
|
||||
},
|
||||
},
|
||||
{
|
||||
async prepare() {
|
||||
return null;
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(await missingContext.prepare(candidate()), null);
|
||||
});
|
||||
|
||||
test('rejects revision or executor drift before materializing any context', async () => {
|
||||
let contextCalls = 0;
|
||||
const create = (value) =>
|
||||
new PinnedTaskLocalRunDispatchPlanSource(
|
||||
{
|
||||
async resolve() {
|
||||
return value;
|
||||
},
|
||||
},
|
||||
{
|
||||
async prepare() {
|
||||
contextCalls += 1;
|
||||
return null;
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
create(revision({ taskRevision: 'revision-latest' })).prepare(candidate()),
|
||||
/does not match/,
|
||||
);
|
||||
await assert.rejects(
|
||||
create(revision({ executorType: 'remote_worker' })).prepare(candidate()),
|
||||
/does not match/,
|
||||
);
|
||||
assert.equal(contextCalls, 0);
|
||||
});
|
||||
|
||||
test('bounds environment count, values, names and output capabilities', () => {
|
||||
const output = { async write() {} };
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeExecutionContext({
|
||||
environment: Object.fromEntries(
|
||||
Array.from(
|
||||
{ length: MAX_EXECUTION_ENVIRONMENT_ENTRIES + 1 },
|
||||
(_, index) => [`KEY_${index}`, 'value'],
|
||||
),
|
||||
),
|
||||
output,
|
||||
}),
|
||||
/too many entries/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeExecutionContext({
|
||||
environment: { 'INVALID=NAME': 'value' },
|
||||
output,
|
||||
}),
|
||||
/entry is invalid/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeExecutionContext({
|
||||
environment: {},
|
||||
output: {},
|
||||
}),
|
||||
/output sink is invalid/,
|
||||
);
|
||||
const prototypeSafe = normalizeExecutionContext({
|
||||
environment: JSON.parse('{"__proto__":"literal"}'),
|
||||
output,
|
||||
});
|
||||
assert.equal(Object.getPrototypeOf(prototypeSafe.environment), null);
|
||||
assert.equal(prototypeSafe.environment.__proto__, 'literal');
|
||||
});
|
||||
|
||||
test('disposes materialized capabilities when context validation fails', async () => {
|
||||
let disposeCalls = 0;
|
||||
const source = new PinnedTaskLocalRunDispatchPlanSource(
|
||||
{
|
||||
async resolve() {
|
||||
return revision();
|
||||
},
|
||||
},
|
||||
{
|
||||
async prepare() {
|
||||
return {
|
||||
context: { environment: {}, output: {} },
|
||||
dispose() {
|
||||
disposeCalls += 1;
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
await assert.rejects(source.prepare(candidate()), /output sink is invalid/);
|
||||
assert.equal(disposeCalls, 1);
|
||||
});
|
||||
@@ -0,0 +1,178 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PostgresMigrationLeaderUnavailableError,
|
||||
PostgresMigrationStreamStore,
|
||||
} = require('../../back/migrations/adapters/postgresMigrationStreamStore');
|
||||
const {
|
||||
runMigrationStream,
|
||||
} = require('../../back/migrations/core/migrationStream');
|
||||
|
||||
const CHECKSUM = 'a'.repeat(64);
|
||||
|
||||
function createPool(lockResults = [true, true]) {
|
||||
const records = new Map();
|
||||
const calls = [];
|
||||
let released = 0;
|
||||
let lockIndex = 0;
|
||||
|
||||
function selectRecord(values, source) {
|
||||
const record = source.get(values[0]);
|
||||
return { rows: record ? [{ ...record }] : [] };
|
||||
}
|
||||
|
||||
return {
|
||||
calls,
|
||||
records,
|
||||
get released() {
|
||||
return released;
|
||||
},
|
||||
pool: {
|
||||
async query(text, values = []) {
|
||||
calls.push({ scope: 'pool', text, values: [...values] });
|
||||
if (text.startsWith('SELECT\n stream_id')) {
|
||||
if (values.length === 0) {
|
||||
return {
|
||||
rows: [...records.values()].map((record) => ({ ...record })),
|
||||
};
|
||||
}
|
||||
return selectRecord(values, records);
|
||||
}
|
||||
throw new Error(`unexpected pool query: ${text}`);
|
||||
},
|
||||
async connect() {
|
||||
let staged = null;
|
||||
return {
|
||||
async query(text, values = []) {
|
||||
calls.push({ scope: 'client', text, values: [...values] });
|
||||
if (text === 'BEGIN') {
|
||||
staged = new Map(
|
||||
[...records].map(([id, record]) => [id, { ...record }]),
|
||||
);
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text === 'COMMIT') {
|
||||
records.clear();
|
||||
for (const [id, record] of staged) records.set(id, record);
|
||||
staged = null;
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text === 'ROLLBACK') {
|
||||
staged = null;
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text.startsWith('SELECT set_config')) return { rows: [] };
|
||||
if (text.startsWith('SELECT pg_try_advisory_xact_lock')) {
|
||||
return { rows: [{ acquired: lockResults[lockIndex++] ?? true }] };
|
||||
}
|
||||
if (text.startsWith('CREATE SCHEMA')) return { rows: [] };
|
||||
if (text.startsWith('CREATE TABLE')) return { rows: [] };
|
||||
if (text.startsWith('SELECT\n stream_id')) {
|
||||
return selectRecord(values, staged);
|
||||
}
|
||||
if (text.startsWith('INSERT INTO')) {
|
||||
const [migrationId, streamId, dialect, checksum, appliedAtMs] =
|
||||
values;
|
||||
staged.set(migrationId, {
|
||||
migrationId,
|
||||
streamId,
|
||||
dialect,
|
||||
checksum,
|
||||
appliedAtMs: String(appliedAtMs),
|
||||
});
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text === 'CREATE TABLE ql3.run_probe(id integer)') {
|
||||
return { rows: [] };
|
||||
}
|
||||
throw new Error(`unexpected client query: ${text}`);
|
||||
},
|
||||
release() {
|
||||
released += 1;
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function stream(up) {
|
||||
return {
|
||||
id: 'postgresql-main',
|
||||
dialect: 'postgresql',
|
||||
migrationIdScheme: 'postgres-prefixed',
|
||||
checksumScheme: 'sha256',
|
||||
migrations: [
|
||||
{
|
||||
id: 'pg-0001-schema-history',
|
||||
checksum: CHECKSUM,
|
||||
up,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
test('serializes PostgreSQL history bootstrap and migration in advisory-lock transactions', async () => {
|
||||
const state = createPool();
|
||||
let upCalls = 0;
|
||||
await runMigrationStream({
|
||||
stream: stream(async (context) => {
|
||||
upCalls += 1;
|
||||
await context.query('CREATE TABLE ql3.run_probe(id integer)');
|
||||
}),
|
||||
store: new PostgresMigrationStreamStore(state.pool),
|
||||
clock: () => 123,
|
||||
});
|
||||
|
||||
assert.equal(upCalls, 1);
|
||||
assert.equal(state.released, 2);
|
||||
assert.deepEqual(state.records.get('pg-0001-schema-history'), {
|
||||
migrationId: 'pg-0001-schema-history',
|
||||
streamId: 'postgresql-main',
|
||||
dialect: 'postgresql',
|
||||
checksum: CHECKSUM,
|
||||
appliedAtMs: '123',
|
||||
});
|
||||
assert.equal(
|
||||
state.calls.filter(({ text }) =>
|
||||
text.startsWith('SELECT pg_try_advisory_xact_lock'),
|
||||
).length,
|
||||
2,
|
||||
);
|
||||
assert.equal(state.calls.filter(({ text }) => text === 'COMMIT').length, 2);
|
||||
});
|
||||
|
||||
test('fails closed before migration work when another migration leader owns the lock', async () => {
|
||||
const state = createPool([true, false]);
|
||||
let upCalls = 0;
|
||||
await assert.rejects(
|
||||
runMigrationStream({
|
||||
stream: stream(async () => {
|
||||
upCalls += 1;
|
||||
}),
|
||||
store: new PostgresMigrationStreamStore(state.pool),
|
||||
}),
|
||||
PostgresMigrationLeaderUnavailableError,
|
||||
);
|
||||
assert.equal(upCalls, 0);
|
||||
assert.equal(state.records.size, 0);
|
||||
assert.equal(state.calls.filter(({ text }) => text === 'ROLLBACK').length, 1);
|
||||
assert.equal(state.released, 2);
|
||||
});
|
||||
|
||||
test('rolls PostgreSQL migration work and history back together', async () => {
|
||||
const state = createPool();
|
||||
await assert.rejects(
|
||||
runMigrationStream({
|
||||
stream: stream(async () => {
|
||||
throw new Error('migration work failed');
|
||||
}),
|
||||
store: new PostgresMigrationStreamStore(state.pool),
|
||||
}),
|
||||
/migration work failed/,
|
||||
);
|
||||
assert.equal(state.records.size, 0);
|
||||
assert.equal(state.calls.filter(({ text }) => text === 'ROLLBACK').length, 1);
|
||||
});
|
||||
@@ -0,0 +1,302 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
DuplicateIdempotencyKeyError,
|
||||
DuplicateRunAttemptError,
|
||||
DuplicateRunEventError,
|
||||
RunEventPayloadTooLargeError,
|
||||
RunRepositoryBusyError,
|
||||
RunRepositoryConstraintError,
|
||||
} = require('../../back/runtime/domain/repositoryErrors');
|
||||
const {
|
||||
MAX_RUN_EVENT_PAYLOAD_BYTES,
|
||||
} = require('../../back/runtime/ports/runRepository');
|
||||
const {
|
||||
PostgresRunRepository,
|
||||
PostgresRunTransaction,
|
||||
} = require('../../back/runtime/adapters/postgresql/runRepository');
|
||||
|
||||
const RUN = Object.freeze({
|
||||
id: '019f70b0-0000-7000-8000-000000000001',
|
||||
projectId: 'default',
|
||||
taskId: 'task-1',
|
||||
taskRevision: 'revision-1',
|
||||
taskName: 'test',
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
triggeredBy: 'user:1',
|
||||
status: 'created',
|
||||
version: 0,
|
||||
eventSequence: 0,
|
||||
priority: 0,
|
||||
idempotencyKey: 'request-1',
|
||||
createdAtMs: 1_750_000_000_000,
|
||||
});
|
||||
|
||||
const ATTEMPT = Object.freeze({
|
||||
id: '019f70b0-0000-7000-8000-000000000002',
|
||||
runId: RUN.id,
|
||||
attempt: 1,
|
||||
status: 'claimed',
|
||||
executorType: 'remote_worker',
|
||||
callbackSequence: 0,
|
||||
createdAtMs: 1_750_000_000_001,
|
||||
});
|
||||
|
||||
const EVENT = Object.freeze({
|
||||
id: '019f70b0-0000-7000-8000-000000000003',
|
||||
runId: RUN.id,
|
||||
sequence: 1,
|
||||
type: 'run.created',
|
||||
dedupeKey: 'run.created',
|
||||
actorType: 'system',
|
||||
payload: Object.freeze({ source: 'postgres-test' }),
|
||||
createdAtMs: 1_750_000_000_002,
|
||||
});
|
||||
|
||||
const RETRY_POLICY = Object.freeze({
|
||||
runId: RUN.id,
|
||||
maxAttempts: 3,
|
||||
retryOnLost: true,
|
||||
safety: 'idempotent',
|
||||
backoffBaseMs: 1_000,
|
||||
backoffMaxMs: 30_000,
|
||||
version: 0,
|
||||
createdAtMs: 1_750_000_000_002,
|
||||
updatedAtMs: 1_750_000_000_002,
|
||||
});
|
||||
|
||||
function driverError(code, constraint) {
|
||||
return Object.assign(new Error('driver failure'), { code, constraint });
|
||||
}
|
||||
|
||||
function clientHarness(handler = async () => ({ rows: [], rowCount: 0 })) {
|
||||
const queries = [];
|
||||
let released = 0;
|
||||
const client = {
|
||||
async query(text, values) {
|
||||
queries.push({ text, values });
|
||||
return handler(text, values, queries.length);
|
||||
},
|
||||
release() {
|
||||
released += 1;
|
||||
},
|
||||
};
|
||||
return {
|
||||
client,
|
||||
queries,
|
||||
released: () => released,
|
||||
pool: {
|
||||
query: (...args) => client.query(...args),
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('orders one bounded PostgreSQL transaction and releases its client', async () => {
|
||||
const harness = clientHarness();
|
||||
const repository = new PostgresRunRepository(harness.pool);
|
||||
const result = await repository.transaction(async () => 'committed');
|
||||
assert.equal(result, 'committed');
|
||||
assert.deepEqual(
|
||||
harness.queries.map(({ text }) => text),
|
||||
[
|
||||
'BEGIN',
|
||||
'SET TRANSACTION ISOLATION LEVEL READ COMMITTED',
|
||||
"SELECT set_config('statement_timeout', $1, true)",
|
||||
"SELECT set_config('lock_timeout', $1, true)",
|
||||
"SELECT set_config('idle_in_transaction_session_timeout', $1, true)",
|
||||
'COMMIT',
|
||||
],
|
||||
);
|
||||
assert.deepEqual(
|
||||
harness.queries.slice(2, 5).map(({ values }) => values[0]),
|
||||
['5000ms', '1000ms', '10000ms'],
|
||||
);
|
||||
assert.equal(harness.released(), 1);
|
||||
});
|
||||
|
||||
test('rolls back work errors unchanged and maps commit serialization failure', async () => {
|
||||
const workHarness = clientHarness();
|
||||
const workRepository = new PostgresRunRepository(workHarness.pool);
|
||||
const failure = new Error('work failed');
|
||||
await assert.rejects(
|
||||
workRepository.transaction(async () => {
|
||||
throw failure;
|
||||
}),
|
||||
(error) => error === failure,
|
||||
);
|
||||
assert.equal(workHarness.queries.at(-1).text, 'ROLLBACK');
|
||||
assert.equal(workHarness.released(), 1);
|
||||
|
||||
const commitHarness = clientHarness(async (text) => {
|
||||
if (text === 'COMMIT') throw driverError('40001');
|
||||
return { rows: [], rowCount: 0 };
|
||||
});
|
||||
await assert.rejects(
|
||||
new PostgresRunRepository(commitHarness.pool).transaction(async () => 1),
|
||||
RunRepositoryBusyError,
|
||||
);
|
||||
assert.equal(commitHarness.queries.at(-1).text, 'ROLLBACK');
|
||||
assert.equal(commitHarness.released(), 1);
|
||||
});
|
||||
|
||||
test('normalizes bigint rows and rejects corrupt enum data', async () => {
|
||||
const runRow = {
|
||||
...RUN,
|
||||
createdAtMs: String(RUN.createdAtMs),
|
||||
scheduledForMs: '1750000000100',
|
||||
taskSnapshotRef: null,
|
||||
legacyCronId: null,
|
||||
parentRunId: null,
|
||||
retryOfRunId: null,
|
||||
triggerId: null,
|
||||
requestId: null,
|
||||
queuedAtMs: null,
|
||||
startedAtMs: null,
|
||||
finishedAtMs: null,
|
||||
cancelRequestedAtMs: null,
|
||||
cancelReason: null,
|
||||
inputRef: null,
|
||||
outputRef: null,
|
||||
errorCode: null,
|
||||
errorSummary: null,
|
||||
};
|
||||
const harness = clientHarness(async (text) => {
|
||||
if (text.includes('FROM "ql3"."runs"')) {
|
||||
return { rows: [runRow], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`unexpected query: ${text}`);
|
||||
});
|
||||
const repository = new PostgresRunRepository(harness.pool);
|
||||
assert.deepEqual(await repository.findRunById(RUN.id), {
|
||||
...RUN,
|
||||
scheduledForMs: 1_750_000_000_100,
|
||||
});
|
||||
|
||||
runRow.status = 'invented';
|
||||
await assert.rejects(
|
||||
repository.findRunById(RUN.id),
|
||||
RunRepositoryConstraintError,
|
||||
);
|
||||
});
|
||||
|
||||
test('writes every aggregate shape and uses exact CAS predicates', async () => {
|
||||
const queries = [];
|
||||
const transaction = new PostgresRunTransaction({
|
||||
async query(text, values) {
|
||||
queries.push({ text, values });
|
||||
return text.startsWith('UPDATE')
|
||||
? { rows: [{ id: values[0] }], rowCount: 1 }
|
||||
: { rows: [], rowCount: 1 };
|
||||
},
|
||||
});
|
||||
await transaction.insertRun(RUN);
|
||||
await transaction.insertAttempt(ATTEMPT);
|
||||
await transaction.insertRetryPolicy(RETRY_POLICY);
|
||||
await transaction.appendEvent(EVENT);
|
||||
assert.equal(
|
||||
await transaction.compareAndSetRun(
|
||||
{ ...RUN, status: 'queued', version: 1 },
|
||||
0,
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
await transaction.compareAndSetAttempt(
|
||||
{ ...ATTEMPT, status: 'starting', callbackSequence: 1 },
|
||||
{ status: 'claimed', callbackSequence: 0 },
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
await transaction.compareAndSetRetryPolicy(
|
||||
{ ...RETRY_POLICY, version: 1, updatedAtMs: 1_750_000_000_003 },
|
||||
0,
|
||||
),
|
||||
true,
|
||||
);
|
||||
|
||||
const updates = queries.filter(({ text }) => text.startsWith('UPDATE'));
|
||||
assert.equal(updates.length, 3);
|
||||
assert.match(updates[0].text, /"version" = \$32/);
|
||||
assert.equal(updates[0].values.length, 32);
|
||||
assert.match(updates[1].text, /"status" = \$22/);
|
||||
assert.match(updates[1].text, /"callback_sequence" = \$23/);
|
||||
assert.equal(updates[1].values.length, 23);
|
||||
assert.match(updates[2].text, /"version" = \$11/);
|
||||
assert.equal(updates[2].values.length, 11);
|
||||
for (const { text } of updates) assert.doesNotMatch(text, /\$\$/);
|
||||
});
|
||||
|
||||
test('maps stable PostgreSQL constraints without leaking driver errors', async () => {
|
||||
const transactionFor = (error) =>
|
||||
new PostgresRunTransaction({
|
||||
async query() {
|
||||
throw error;
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
transactionFor(
|
||||
driverError('23505', 'ql3_runs_project_idempotency_uidx'),
|
||||
).insertRun(RUN),
|
||||
DuplicateIdempotencyKeyError,
|
||||
);
|
||||
await assert.rejects(
|
||||
transactionFor(
|
||||
driverError('23505', 'ql3_run_attempts_run_attempt_uidx'),
|
||||
).insertAttempt(ATTEMPT),
|
||||
DuplicateRunAttemptError,
|
||||
);
|
||||
await assert.rejects(
|
||||
transactionFor(
|
||||
driverError('23505', 'ql3_run_events_run_dedupe_uidx'),
|
||||
).appendEvent(EVENT),
|
||||
DuplicateRunEventError,
|
||||
);
|
||||
await assert.rejects(
|
||||
transactionFor(driverError('23514')).insertRetryPolicy(RETRY_POLICY),
|
||||
RunRepositoryConstraintError,
|
||||
);
|
||||
await assert.rejects(
|
||||
transactionFor(driverError('55P03')).compareAndSetRun(
|
||||
{ ...RUN, version: 1 },
|
||||
0,
|
||||
),
|
||||
RunRepositoryBusyError,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects non-incrementing CAS and oversized event payloads before SQL', async () => {
|
||||
let calls = 0;
|
||||
const transaction = new PostgresRunTransaction({
|
||||
async query() {
|
||||
calls += 1;
|
||||
return { rows: [], rowCount: 0 };
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
transaction.compareAndSetRun({ ...RUN, version: 2 }, 0),
|
||||
RunRepositoryConstraintError,
|
||||
);
|
||||
await assert.rejects(
|
||||
transaction.compareAndSetRetryPolicy(
|
||||
{ ...RETRY_POLICY, version: 2, updatedAtMs: 1_750_000_000_003 },
|
||||
0,
|
||||
),
|
||||
RunRepositoryConstraintError,
|
||||
);
|
||||
await assert.rejects(
|
||||
transaction.appendEvent({
|
||||
...EVENT,
|
||||
payload: { value: 'x'.repeat(MAX_RUN_EVENT_PAYLOAD_BYTES) },
|
||||
}),
|
||||
RunEventPayloadTooLargeError,
|
||||
);
|
||||
assert.equal(calls, 0);
|
||||
});
|
||||
@@ -0,0 +1,153 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
postgresqlMainMigrationStream,
|
||||
} = require('../../back/migrations/postgresql');
|
||||
const {
|
||||
postgresqlControlSchemaContract,
|
||||
} = require('../../back/migrations/postgresql/schemaContract');
|
||||
|
||||
const BANNED_CLUSTER_SCHEMA_NAMES = [
|
||||
'crontabs',
|
||||
'envs',
|
||||
'subscriptions',
|
||||
'runninginstances',
|
||||
'completionreceiptjournals',
|
||||
'localexecutioncontextrecipes',
|
||||
'localsecretenvelopes',
|
||||
'localartifactretentioncheckpoints',
|
||||
'projectownerbootstrapchallenges',
|
||||
'legacypanelidentitybindings',
|
||||
];
|
||||
|
||||
function tableDefinitionSql(statements, tableName) {
|
||||
const qualifiedName = `"ql3"."${tableName}"`;
|
||||
return statements
|
||||
.filter(
|
||||
(statement) =>
|
||||
statement.includes(qualifiedName) &&
|
||||
(/^CREATE TABLE /.test(statement) || /^ALTER TABLE /.test(statement)),
|
||||
)
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
test('defines the immutable PostgreSQL capability and Run core stream', async () => {
|
||||
assert.equal(postgresqlMainMigrationStream.id, 'postgresql-main');
|
||||
assert.equal(postgresqlMainMigrationStream.dialect, 'postgresql');
|
||||
assert.equal(
|
||||
postgresqlMainMigrationStream.migrationIdScheme,
|
||||
'postgres-prefixed',
|
||||
);
|
||||
assert.equal(postgresqlMainMigrationStream.checksumScheme, 'sha256');
|
||||
assert.deepEqual(
|
||||
postgresqlMainMigrationStream.migrations.map(({ id }) => id),
|
||||
[
|
||||
'pg-0001-schema-capability',
|
||||
'pg-0002-run-core',
|
||||
'pg-0003-run-retry-policy',
|
||||
],
|
||||
);
|
||||
for (const migration of postgresqlMainMigrationStream.migrations) {
|
||||
assert.match(migration.checksum, /^[0-9a-f]{64}$/);
|
||||
}
|
||||
});
|
||||
|
||||
test('keeps local-only and legacy tables out of the cluster baseline', async () => {
|
||||
const statements = [];
|
||||
for (const migration of postgresqlMainMigrationStream.migrations) {
|
||||
await migration.up({
|
||||
async query(statement) {
|
||||
statements.push(statement);
|
||||
return { rows: [] };
|
||||
},
|
||||
});
|
||||
}
|
||||
const canonical = statements.join('\n').toLowerCase();
|
||||
for (const table of BANNED_CLUSTER_SCHEMA_NAMES) {
|
||||
assert.equal(canonical.includes(table.toLowerCase()), false, table);
|
||||
}
|
||||
for (const table of [
|
||||
'schema_capabilities',
|
||||
'runs',
|
||||
'run_attempts',
|
||||
'run_events',
|
||||
'run_retry_policies',
|
||||
]) {
|
||||
assert.match(canonical, new RegExp(`"ql3"\\."${table}"`));
|
||||
}
|
||||
assert.match(canonical, /deferrable initially deferred/);
|
||||
assert.match(canonical, /'control-core'/);
|
||||
assert.match(canonical, /'pg-0003-run-retry-policy'/);
|
||||
assert.match(canonical, /"run_core":1,"run_retry_policy":1/);
|
||||
});
|
||||
|
||||
test('keeps the reviewed SQL and readiness schema contract in lockstep', async () => {
|
||||
const statements = [];
|
||||
for (const migration of postgresqlMainMigrationStream.migrations) {
|
||||
await migration.up({
|
||||
async query(statement) {
|
||||
statements.push(statement);
|
||||
return { rows: [] };
|
||||
},
|
||||
});
|
||||
}
|
||||
const canonical = statements.join('\n');
|
||||
for (const table of postgresqlControlSchemaContract.tables) {
|
||||
if (table.name === 'schema_migrations') continue;
|
||||
const definition = tableDefinitionSql(statements, table.name);
|
||||
assert.match(definition, new RegExp(`"ql3"\\."${table.name}"`));
|
||||
for (const column of table.columns) {
|
||||
assert.match(definition, new RegExp(`\\b${column}\\b`));
|
||||
}
|
||||
}
|
||||
for (const index of postgresqlControlSchemaContract.indexes) {
|
||||
if (index.endsWith('_pkey')) continue;
|
||||
assert.match(canonical, new RegExp(`CREATE (?:UNIQUE )?INDEX ${index}\\b`));
|
||||
}
|
||||
assert.doesNotMatch(canonical, /CREATE TABLE IF NOT EXISTS/);
|
||||
assert.doesNotMatch(canonical, /CREATE INDEX IF NOT EXISTS/);
|
||||
});
|
||||
|
||||
test('freezes every published PostgreSQL migration checksum', () => {
|
||||
assert.deepEqual(
|
||||
postgresqlMainMigrationStream.migrations.map(({ id, checksum }) => ({
|
||||
id,
|
||||
checksum,
|
||||
})),
|
||||
[
|
||||
{
|
||||
id: 'pg-0001-schema-capability',
|
||||
checksum:
|
||||
'9e3499e3bcdfe3d7b2559e64ea7bbf236a8a11ba32d6a45af131034887d5a8ab',
|
||||
},
|
||||
{
|
||||
id: 'pg-0002-run-core',
|
||||
checksum:
|
||||
'5b59a7f9323746e49c6c321e89007f553a0751f25d16ffd23c3ae37dd87f76e4',
|
||||
},
|
||||
{
|
||||
id: 'pg-0003-run-retry-policy',
|
||||
checksum:
|
||||
'621792cde917cc86809bbebff389443e790bdba60f73d04f7a1dc97a0ebf72db',
|
||||
},
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('advances capability v2 only from the exact v1 predecessor', async () => {
|
||||
const statements = [];
|
||||
await postgresqlMainMigrationStream.migrations.at(-1).up({
|
||||
async query(statement) {
|
||||
statements.push(statement);
|
||||
return { rows: [] };
|
||||
},
|
||||
});
|
||||
const capability = statements.at(-1);
|
||||
assert.match(capability, /contract_version = 1/);
|
||||
assert.match(capability, /migration_id = 'pg-0002-run-core'/);
|
||||
assert.match(capability, /capabilities = '\{"run_core":1\}'::jsonb/);
|
||||
assert.match(capability, /IF NOT FOUND THEN/);
|
||||
assert.match(capability, /RAISE EXCEPTION/);
|
||||
});
|
||||
@@ -0,0 +1,264 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
postgresqlMainMigrationStream,
|
||||
} = require('../../back/migrations/postgresql');
|
||||
const {
|
||||
postgresqlControlSchemaContract,
|
||||
} = require('../../back/migrations/postgresql/schemaContract');
|
||||
const {
|
||||
PostgresSchemaReadinessError,
|
||||
assertPostgresSchemaReady,
|
||||
} = require('../../back/migrations/postgresql/schemaReadiness');
|
||||
|
||||
function validHistory() {
|
||||
return postgresqlMainMigrationStream.migrations.map((migration, index) => ({
|
||||
streamId: 'postgresql-main',
|
||||
dialect: 'postgresql',
|
||||
migrationId: migration.id,
|
||||
checksum: migration.checksum,
|
||||
appliedAtMs: index + 1,
|
||||
}));
|
||||
}
|
||||
|
||||
function validPrivileges() {
|
||||
const expected = {
|
||||
schema_migrations: [true, false, false, false],
|
||||
schema_capabilities: [true, false, false, false],
|
||||
runs: [true, true, true, false],
|
||||
run_attempts: [true, true, true, false],
|
||||
run_events: [true, true, false, false],
|
||||
run_retry_policies: [true, true, true, false],
|
||||
};
|
||||
return Object.entries(expected).map(
|
||||
([
|
||||
tableName,
|
||||
[selectAllowed, insertAllowed, updateAllowed, deleteAllowed],
|
||||
]) => ({
|
||||
tableName,
|
||||
selectAllowed,
|
||||
insertAllowed,
|
||||
updateAllowed,
|
||||
deleteAllowed,
|
||||
isOwner: false,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function queryable(overrides = {}) {
|
||||
const contract = postgresqlControlSchemaContract;
|
||||
return {
|
||||
async query(text) {
|
||||
if (text.includes("current_setting('server_version_num')")) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
serverVersionNum: overrides.serverVersionNum ?? '160014',
|
||||
currentUser: 'ql3_runtime',
|
||||
inRecovery: overrides.inRecovery ?? false,
|
||||
transactionReadOnly: overrides.transactionReadOnly ?? 'off',
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."schema_migrations"')) {
|
||||
return { rows: overrides.history ?? validHistory() };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."schema_capabilities"')) {
|
||||
return {
|
||||
rows: [
|
||||
overrides.capability ?? {
|
||||
contractName: 'control-core',
|
||||
contractVersion: 2,
|
||||
migrationId: 'pg-0003-run-retry-policy',
|
||||
capabilities: { run_core: 1, run_retry_policy: 1 },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM information_schema.columns')) {
|
||||
const rows = contract.tables.flatMap((table) =>
|
||||
table.columns.map((columnName) => ({
|
||||
tableName: table.name,
|
||||
columnName,
|
||||
})),
|
||||
);
|
||||
if (overrides.extraTable) {
|
||||
rows.push({ tableName: overrides.extraTable, columnName: 'id' });
|
||||
}
|
||||
return { rows };
|
||||
}
|
||||
if (text.includes('FROM pg_indexes')) {
|
||||
return {
|
||||
rows: [
|
||||
...contract.indexes.map((indexName) => ({ indexName })),
|
||||
...(overrides.extraIndex
|
||||
? [{ indexName: overrides.extraIndex }]
|
||||
: []),
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM pg_constraint')) {
|
||||
return {
|
||||
rows: [
|
||||
...contract.checks.map((constraintName) => ({
|
||||
constraintName,
|
||||
constraintType: 'check',
|
||||
})),
|
||||
...contract.foreignKeys.map((constraintName) => ({
|
||||
constraintName,
|
||||
constraintType: 'foreign_key',
|
||||
})),
|
||||
...(overrides.extraConstraint
|
||||
? [
|
||||
{
|
||||
constraintName: overrides.extraConstraint,
|
||||
constraintType: 'check',
|
||||
},
|
||||
]
|
||||
: []),
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM pg_catalog.pg_roles')) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
canLogin: true,
|
||||
superuser: overrides.superuser ?? false,
|
||||
createDatabase: false,
|
||||
createRole: false,
|
||||
replication: false,
|
||||
bypassRowLevelSecurity: false,
|
||||
databaseConnect: true,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('has_schema_privilege')) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
schemaUsage: true,
|
||||
schemaCreate: overrides.schemaCreate ?? false,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('has_table_privilege')) {
|
||||
return { rows: overrides.privileges ?? validPrivileges() };
|
||||
}
|
||||
throw new Error(`unexpected query: ${text}`);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('accepts the exact PostgreSQL control schema and least-privilege runtime role', async () => {
|
||||
const report = await assertPostgresSchemaReady(queryable());
|
||||
assert.deepEqual(report, {
|
||||
ready: true,
|
||||
writablePrimary: true,
|
||||
serverVersionNum: 160014,
|
||||
serverMajor: 16,
|
||||
currentUser: 'ql3_runtime',
|
||||
contractName: 'control-core',
|
||||
contractVersion: 2,
|
||||
migrationIds: [
|
||||
'pg-0001-schema-capability',
|
||||
'pg-0002-run-core',
|
||||
'pg-0003-run-retry-policy',
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
test('rejects a standby or read-only endpoint before schema inspection', async () => {
|
||||
await assert.rejects(
|
||||
assertPostgresSchemaReady(queryable({ inRecovery: true })),
|
||||
(error) =>
|
||||
error instanceof PostgresSchemaReadinessError &&
|
||||
error.code === 'server_not_writable_primary' &&
|
||||
error.facts.includes('in-recovery:true'),
|
||||
);
|
||||
await assert.rejects(
|
||||
assertPostgresSchemaReady(queryable({ transactionReadOnly: 'on' })),
|
||||
(error) =>
|
||||
error instanceof PostgresSchemaReadinessError &&
|
||||
error.code === 'server_not_writable_primary' &&
|
||||
error.facts.includes('transaction-read-only:on'),
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects unsupported server versions and capability drift', async () => {
|
||||
await assert.rejects(
|
||||
assertPostgresSchemaReady(queryable({ serverVersionNum: '150018' })),
|
||||
(error) =>
|
||||
error instanceof PostgresSchemaReadinessError &&
|
||||
error.code === 'server_version_unsupported',
|
||||
);
|
||||
await assert.rejects(
|
||||
assertPostgresSchemaReady(
|
||||
queryable({
|
||||
capability: {
|
||||
contractName: 'control-core',
|
||||
contractVersion: 3,
|
||||
migrationId: 'pg-0003-run-retry-policy',
|
||||
capabilities: { run_core: 1, run_retry_policy: 1 },
|
||||
},
|
||||
}),
|
||||
),
|
||||
(error) =>
|
||||
error instanceof PostgresSchemaReadinessError &&
|
||||
error.code === 'capability_invalid',
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects unknown ql3 objects and an over-privileged runtime role', async () => {
|
||||
await assert.rejects(
|
||||
assertPostgresSchemaReady(
|
||||
queryable({
|
||||
extraTable: 'plugin_state',
|
||||
extraIndex: 'plugin_state_pkey',
|
||||
extraConstraint: 'plugin_state_payload_check',
|
||||
}),
|
||||
),
|
||||
(error) =>
|
||||
error instanceof PostgresSchemaReadinessError &&
|
||||
error.code === 'schema_contract_invalid' &&
|
||||
error.facts.includes('unknown-table:plugin_state') &&
|
||||
error.facts.includes('unknown-check:plugin_state_payload_check'),
|
||||
);
|
||||
await assert.rejects(
|
||||
assertPostgresSchemaReady(queryable({ schemaCreate: true })),
|
||||
(error) =>
|
||||
error instanceof PostgresSchemaReadinessError &&
|
||||
error.code === 'runtime_role_invalid',
|
||||
);
|
||||
});
|
||||
|
||||
test('preserves database availability errors for the outer readiness layer', async () => {
|
||||
const unavailable = new Error('database unavailable');
|
||||
let calls = 0;
|
||||
await assert.rejects(
|
||||
assertPostgresSchemaReady({
|
||||
async query() {
|
||||
calls += 1;
|
||||
if (calls === 1) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
serverVersionNum: '160014',
|
||||
currentUser: 'ql3_runtime',
|
||||
inRecovery: false,
|
||||
transactionReadOnly: 'off',
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
throw unavailable;
|
||||
},
|
||||
}),
|
||||
(error) => error === unavailable,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,409 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PrimaryCancellationDispatcher,
|
||||
} = require('../../back/runtime/application/primaryCancellationDispatcher');
|
||||
|
||||
const NOW_MS = 1_750_000_000_100;
|
||||
|
||||
function candidate(runId, overrides = {}) {
|
||||
return {
|
||||
runId,
|
||||
requestedAtMs: NOW_MS - 10,
|
||||
reason: 'user',
|
||||
attempts: [
|
||||
{
|
||||
attemptId: `${runId}-attempt`,
|
||||
executorType: 'local_process',
|
||||
executorHandle: `handle:${runId}`,
|
||||
pid: 4001,
|
||||
},
|
||||
],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fakeDispatchRepository(overrides = {}) {
|
||||
const claims = [];
|
||||
const results = [];
|
||||
return {
|
||||
claims,
|
||||
results,
|
||||
async findByRunId() {
|
||||
return null;
|
||||
},
|
||||
async claim(command) {
|
||||
claims.push(command);
|
||||
if (overrides.claim) return overrides.claim(command);
|
||||
return {
|
||||
status: 'claimed',
|
||||
dispatch: {
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
status: 'leased',
|
||||
version: 1,
|
||||
dispatchCount: 1,
|
||||
leaseOwner: command.owner,
|
||||
leaseToken: command.leaseToken,
|
||||
leaseExpiresAtMs: command.nowMs + command.leaseDurationMs,
|
||||
createdAtMs: command.nowMs,
|
||||
updatedAtMs: command.nowMs,
|
||||
},
|
||||
};
|
||||
},
|
||||
async recordResult(command) {
|
||||
results.push(command);
|
||||
if (overrides.recordResult) return overrides.recordResult(command);
|
||||
return {
|
||||
dispatch: {
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
status: command.nextAttemptAtMs ? 'retry_wait' : 'dispatched',
|
||||
version: command.expectedVersion + 1,
|
||||
dispatchCount: 1,
|
||||
lastResult: command.result,
|
||||
createdAtMs: NOW_MS,
|
||||
updatedAtMs: command.atMs,
|
||||
},
|
||||
event: {
|
||||
id: command.eventId,
|
||||
runId: command.runId,
|
||||
sequence: 1,
|
||||
type: 'fixture',
|
||||
actorType: 'worker',
|
||||
payload: {},
|
||||
createdAtMs: command.atMs,
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function dispatcherOptions(overrides = {}) {
|
||||
let id = 0;
|
||||
return {
|
||||
owner: 'worker-a',
|
||||
leaseDurationMs: 100,
|
||||
retryBaseMs: 1_000,
|
||||
retryMaxMs: 8_000,
|
||||
clock: () => NOW_MS,
|
||||
createId: () => `019f71d0-0000-7000-8000-${String(++id).padStart(12, '0')}`,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('leases before signalling and durably classifies every controller result', async () => {
|
||||
const stopped = [];
|
||||
const source = {
|
||||
async listCandidates(options) {
|
||||
assert.deepEqual(options, { limit: 8 });
|
||||
return {
|
||||
candidates: [
|
||||
candidate('run-1'),
|
||||
candidate('run-2'),
|
||||
candidate('run-3', { attempts: [] }),
|
||||
candidate('run-4', {
|
||||
attempts: [
|
||||
{
|
||||
attemptId: 'attempt-4a',
|
||||
executorType: 'local_process',
|
||||
executorHandle: 'handle:4a',
|
||||
},
|
||||
{
|
||||
attemptId: 'attempt-4b',
|
||||
executorType: 'local_process',
|
||||
executorHandle: 'handle:4b',
|
||||
},
|
||||
],
|
||||
}),
|
||||
candidate('run-5', {
|
||||
attempts: [
|
||||
{
|
||||
attemptId: 'attempt-5',
|
||||
executorType: 'remote_worker',
|
||||
executorHandle: 'remote:5',
|
||||
},
|
||||
],
|
||||
}),
|
||||
candidate('run-6'),
|
||||
candidate('run-7'),
|
||||
candidate('run-8'),
|
||||
],
|
||||
truncated: true,
|
||||
unsafeAttemptOverflow: false,
|
||||
nextCursor: { requestedAtMs: NOW_MS - 10, runId: 'run-8' },
|
||||
};
|
||||
},
|
||||
};
|
||||
const dispatches = fakeDispatchRepository({
|
||||
claim(command) {
|
||||
if (command.runId === 'run-8') {
|
||||
return {
|
||||
status: 'leased',
|
||||
dispatch: {
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
status: 'leased',
|
||||
version: 3,
|
||||
dispatchCount: 2,
|
||||
leaseOwner: 'worker-b',
|
||||
leaseToken: 'other-lease',
|
||||
leaseExpiresAtMs: NOW_MS + 1_000,
|
||||
createdAtMs: NOW_MS - 5_000,
|
||||
updatedAtMs: NOW_MS - 100,
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: 'claimed',
|
||||
dispatch: {
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
status: 'leased',
|
||||
version: 1,
|
||||
dispatchCount: 1,
|
||||
leaseOwner: command.owner,
|
||||
leaseToken: command.leaseToken,
|
||||
leaseExpiresAtMs: command.nowMs + command.leaseDurationMs,
|
||||
createdAtMs: command.nowMs,
|
||||
updatedAtMs: command.nowMs,
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
let calls = 0;
|
||||
const controller = {
|
||||
executorType: 'local_process',
|
||||
async stop(input) {
|
||||
calls += 1;
|
||||
stopped.push(input);
|
||||
if (calls === 1) {
|
||||
return {
|
||||
status: 'termination_requested',
|
||||
termSignalSent: true,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
if (calls === 2) {
|
||||
return {
|
||||
status: 'already_exited',
|
||||
termSignalSent: false,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
if (calls === 3) throw new Error('transient stop failure');
|
||||
return {
|
||||
status: 'identity_mismatch',
|
||||
termSignalSent: false,
|
||||
killSignalSent: false,
|
||||
};
|
||||
},
|
||||
};
|
||||
const dispatcher = new PrimaryCancellationDispatcher(
|
||||
source,
|
||||
dispatches,
|
||||
[controller],
|
||||
dispatcherOptions(),
|
||||
);
|
||||
|
||||
const summary = await dispatcher.dispatchBatch({ limit: 8 });
|
||||
assert.deepEqual(summary, {
|
||||
scanned: 8,
|
||||
claimed: 5,
|
||||
terminationRequested: 1,
|
||||
alreadyExited: 1,
|
||||
pending: 5,
|
||||
ambiguous: 1,
|
||||
blocked: 1,
|
||||
deferred: 1,
|
||||
alreadyResolved: 0,
|
||||
notEligible: 0,
|
||||
failed: 1,
|
||||
truncated: true,
|
||||
unsafeAttemptOverflow: false,
|
||||
nextCursor: { requestedAtMs: NOW_MS - 10, runId: 'run-8' },
|
||||
});
|
||||
assert.equal(stopped.length, 4);
|
||||
assert.deepEqual(stopped[0], {
|
||||
durableHandle: 'handle:run-1',
|
||||
expectedPid: 4001,
|
||||
reason: { kind: 'user', requestedAtMs: NOW_MS - 10 },
|
||||
});
|
||||
assert.deepEqual(
|
||||
dispatches.results.map((result) => result.result),
|
||||
[
|
||||
'termination_requested',
|
||||
'already_exited',
|
||||
'controller_missing',
|
||||
'dispatch_error',
|
||||
'identity_mismatch',
|
||||
],
|
||||
);
|
||||
assert.equal(
|
||||
dispatches.results.find((result) => result.result === 'dispatch_error')
|
||||
.nextAttemptAtMs,
|
||||
NOW_MS + 1_000,
|
||||
);
|
||||
});
|
||||
|
||||
test('fails closed on unsafe overflow, duplicate controllers, and corrupt reasons', async () => {
|
||||
let stopCalls = 0;
|
||||
const controller = {
|
||||
executorType: 'local_process',
|
||||
async stop() {
|
||||
stopCalls += 1;
|
||||
throw new Error('must not run');
|
||||
},
|
||||
};
|
||||
const dispatches = fakeDispatchRepository();
|
||||
assert.throws(
|
||||
() =>
|
||||
new PrimaryCancellationDispatcher(
|
||||
{ async listCandidates() {} },
|
||||
dispatches,
|
||||
[controller, controller],
|
||||
dispatcherOptions(),
|
||||
),
|
||||
/Duplicate persisted Executor controller/,
|
||||
);
|
||||
|
||||
const overflow = new PrimaryCancellationDispatcher(
|
||||
{
|
||||
async listCandidates() {
|
||||
return {
|
||||
candidates: [],
|
||||
truncated: true,
|
||||
unsafeAttemptOverflow: true,
|
||||
};
|
||||
},
|
||||
},
|
||||
dispatches,
|
||||
[controller],
|
||||
dispatcherOptions(),
|
||||
);
|
||||
const overflowSummary = await overflow.dispatchBatch();
|
||||
assert.equal(overflowSummary.unsafeAttemptOverflow, true);
|
||||
|
||||
const corrupt = new PrimaryCancellationDispatcher(
|
||||
{
|
||||
async listCandidates() {
|
||||
return {
|
||||
candidates: [candidate('run-corrupt', { reason: 'raw-corruption' })],
|
||||
truncated: false,
|
||||
unsafeAttemptOverflow: false,
|
||||
};
|
||||
},
|
||||
},
|
||||
dispatches,
|
||||
[controller],
|
||||
dispatcherOptions(),
|
||||
);
|
||||
const corruptSummary = await corrupt.dispatchBatch();
|
||||
assert.equal(corruptSummary.pending, 1);
|
||||
assert.equal(stopCalls, 0);
|
||||
assert.equal(dispatches.claims.length, 0);
|
||||
});
|
||||
|
||||
test('reports persisted terminal, deferred, stale, and failed claims without signalling', async () => {
|
||||
const statuses = new Map([
|
||||
['run-not-due', 'not_due'],
|
||||
['run-dispatched', 'dispatched'],
|
||||
['run-blocked', 'blocked'],
|
||||
['run-stale', 'not_eligible'],
|
||||
]);
|
||||
const dispatches = fakeDispatchRepository({
|
||||
claim(command) {
|
||||
if (command.runId === 'run-failed') throw new Error('database busy');
|
||||
const status = statuses.get(command.runId);
|
||||
if (status === 'not_eligible') return { status };
|
||||
return {
|
||||
status,
|
||||
dispatch: {
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
status: status === 'not_due' ? 'retry_wait' : status,
|
||||
version: 2,
|
||||
dispatchCount: 1,
|
||||
createdAtMs: NOW_MS - 1_000,
|
||||
updatedAtMs: NOW_MS - 500,
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
let stopCalls = 0;
|
||||
const dispatcher = new PrimaryCancellationDispatcher(
|
||||
{
|
||||
async listCandidates() {
|
||||
return {
|
||||
candidates: [
|
||||
candidate('run-not-due'),
|
||||
candidate('run-dispatched'),
|
||||
candidate('run-blocked'),
|
||||
candidate('run-stale'),
|
||||
candidate('run-failed'),
|
||||
],
|
||||
truncated: false,
|
||||
unsafeAttemptOverflow: false,
|
||||
};
|
||||
},
|
||||
},
|
||||
dispatches,
|
||||
[
|
||||
{
|
||||
executorType: 'local_process',
|
||||
async stop() {
|
||||
stopCalls += 1;
|
||||
throw new Error('must not run');
|
||||
},
|
||||
},
|
||||
],
|
||||
dispatcherOptions(),
|
||||
);
|
||||
const summary = await dispatcher.dispatchBatch();
|
||||
assert.equal(summary.deferred, 1);
|
||||
assert.equal(summary.alreadyResolved, 2);
|
||||
assert.equal(summary.blocked, 1);
|
||||
assert.equal(summary.notEligible, 1);
|
||||
assert.equal(summary.failed, 1);
|
||||
assert.equal(summary.pending, 2);
|
||||
assert.equal(stopCalls, 0);
|
||||
});
|
||||
|
||||
test('counts result persistence failure while retaining the signal outcome', async () => {
|
||||
const dispatches = fakeDispatchRepository({
|
||||
recordResult() {
|
||||
throw new Error('commit failed');
|
||||
},
|
||||
});
|
||||
const dispatcher = new PrimaryCancellationDispatcher(
|
||||
{
|
||||
async listCandidates() {
|
||||
return {
|
||||
candidates: [candidate('run-signal-before-commit')],
|
||||
truncated: false,
|
||||
unsafeAttemptOverflow: false,
|
||||
};
|
||||
},
|
||||
},
|
||||
dispatches,
|
||||
[
|
||||
{
|
||||
executorType: 'local_process',
|
||||
async stop() {
|
||||
return {
|
||||
status: 'termination_requested',
|
||||
termSignalSent: true,
|
||||
killSignalSent: false,
|
||||
};
|
||||
},
|
||||
},
|
||||
],
|
||||
dispatcherOptions(),
|
||||
);
|
||||
const summary = await dispatcher.dispatchBatch();
|
||||
assert.equal(summary.terminationRequested, 1);
|
||||
assert.equal(summary.failed, 1);
|
||||
assert.equal(summary.pending, 1);
|
||||
});
|
||||
@@ -0,0 +1,202 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PrimaryCancellationLifecycle,
|
||||
} = require('../../back/runtime/application/primaryCancellationLifecycle');
|
||||
|
||||
function deferred() {
|
||||
let resolve;
|
||||
let reject;
|
||||
const promise = new Promise((resolvePromise, rejectPromise) => {
|
||||
resolve = resolvePromise;
|
||||
reject = rejectPromise;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
function fakeScheduler() {
|
||||
let id = 0;
|
||||
const pending = new Map();
|
||||
const cleared = [];
|
||||
return {
|
||||
pending,
|
||||
cleared,
|
||||
scheduler: {
|
||||
setTimeout(callback, delayMs) {
|
||||
const timer = {
|
||||
id: ++id,
|
||||
delayMs,
|
||||
unrefCalls: 0,
|
||||
unref() {
|
||||
this.unrefCalls += 1;
|
||||
},
|
||||
};
|
||||
pending.set(timer.id, { timer, callback });
|
||||
return timer;
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
cleared.push(timer.id);
|
||||
pending.delete(timer.id);
|
||||
},
|
||||
},
|
||||
fireNext() {
|
||||
const next = pending.values().next().value;
|
||||
assert.ok(next, 'expected a scheduled timer');
|
||||
pending.delete(next.timer.id);
|
||||
next.callback();
|
||||
return next.timer;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function cycleSummary() {
|
||||
return {
|
||||
pages: 1,
|
||||
scanned: 1,
|
||||
claimed: 1,
|
||||
terminationRequested: 1,
|
||||
alreadyExited: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
blocked: 0,
|
||||
deferred: 0,
|
||||
alreadyResolved: 0,
|
||||
notEligible: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
}
|
||||
|
||||
async function flush() {
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
}
|
||||
|
||||
test('is inert until started and never overlaps slow cycles', async () => {
|
||||
const timers = fakeScheduler();
|
||||
const first = deferred();
|
||||
let calls = 0;
|
||||
const lifecycle = new PrimaryCancellationLifecycle(
|
||||
{
|
||||
async runCycle(options) {
|
||||
calls += 1;
|
||||
assert.deepEqual(options, { pageSize: 8, maxPages: 2 });
|
||||
return first.promise;
|
||||
},
|
||||
},
|
||||
{
|
||||
intervalMs: 2_000,
|
||||
initialDelayMs: 50,
|
||||
cycle: { pageSize: 8, maxPages: 2 },
|
||||
scheduler: timers.scheduler,
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(timers.pending.size, 0);
|
||||
assert.equal(lifecycle.start(), true);
|
||||
assert.equal(lifecycle.start(), false);
|
||||
assert.equal(timers.pending.size, 1);
|
||||
const initial = timers.fireNext();
|
||||
assert.equal(initial.delayMs, 50);
|
||||
assert.equal(initial.unrefCalls, 1);
|
||||
await flush();
|
||||
assert.equal(calls, 1);
|
||||
assert.equal(timers.pending.size, 0);
|
||||
|
||||
first.resolve(cycleSummary());
|
||||
await flush();
|
||||
assert.equal(timers.pending.size, 1);
|
||||
const next = timers.pending.values().next().value.timer;
|
||||
assert.equal(next.delayMs, 2_000);
|
||||
assert.equal(next.unrefCalls, 1);
|
||||
assert.equal(await lifecycle.stop(), 'drained');
|
||||
assert.deepEqual(timers.cleared, [next.id]);
|
||||
});
|
||||
|
||||
test('reports cycle errors and continues without callback failure loops', async () => {
|
||||
const timers = fakeScheduler();
|
||||
const errors = [];
|
||||
let calls = 0;
|
||||
const lifecycle = new PrimaryCancellationLifecycle(
|
||||
{
|
||||
async runCycle() {
|
||||
calls += 1;
|
||||
if (calls === 1) throw new Error('database busy');
|
||||
return cycleSummary();
|
||||
},
|
||||
},
|
||||
{
|
||||
intervalMs: 500,
|
||||
scheduler: timers.scheduler,
|
||||
onCycle() {
|
||||
throw new Error('metrics sink failed');
|
||||
},
|
||||
onError(error) {
|
||||
errors.push(error.message);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
lifecycle.start();
|
||||
timers.fireNext();
|
||||
await flush();
|
||||
assert.deepEqual(errors, ['database busy']);
|
||||
assert.equal(timers.pending.size, 1);
|
||||
|
||||
timers.fireNext();
|
||||
await flush();
|
||||
assert.deepEqual(errors, ['database busy', 'metrics sink failed']);
|
||||
assert.equal(timers.pending.size, 1);
|
||||
assert.equal(await lifecycle.stop(), 'drained');
|
||||
});
|
||||
|
||||
test('stops scheduling immediately and bounds waiting for an in-flight cycle', async () => {
|
||||
const timers = fakeScheduler();
|
||||
const running = deferred();
|
||||
const lifecycle = new PrimaryCancellationLifecycle(
|
||||
{
|
||||
async runCycle() {
|
||||
return running.promise;
|
||||
},
|
||||
},
|
||||
{
|
||||
intervalMs: 500,
|
||||
stopTimeoutMs: 5,
|
||||
scheduler: timers.scheduler,
|
||||
},
|
||||
);
|
||||
lifecycle.start();
|
||||
timers.fireNext();
|
||||
await flush();
|
||||
|
||||
assert.equal(await lifecycle.stop(), 'timed_out');
|
||||
assert.equal(timers.pending.size, 0);
|
||||
assert.equal(lifecycle.start(), false);
|
||||
running.resolve(cycleSummary());
|
||||
await flush();
|
||||
assert.equal(timers.pending.size, 0);
|
||||
assert.equal(lifecycle.start(), true);
|
||||
assert.equal(await lifecycle.stop(), 'drained');
|
||||
});
|
||||
|
||||
test('rejects cadences that could create hot loops or unbounded shutdown waits', () => {
|
||||
const supervisor = {
|
||||
async runCycle() {
|
||||
return cycleSummary();
|
||||
},
|
||||
};
|
||||
assert.throws(
|
||||
() => new PrimaryCancellationLifecycle(supervisor, { intervalMs: 249 }),
|
||||
RangeError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
new PrimaryCancellationLifecycle(supervisor, {
|
||||
intervalMs: 500,
|
||||
stopTimeoutMs: 60_001,
|
||||
}),
|
||||
RangeError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,179 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const { Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const { runSchemaMigration } = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
runCancellationRequestMigration,
|
||||
} = require('../../back/migrations/0004-run-cancellation-request');
|
||||
const {
|
||||
runAttemptDeadlineMigration,
|
||||
} = require('../../back/migrations/0006-run-attempt-deadline');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizePrimaryCancellationSource,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/primaryCancellationSource');
|
||||
const {
|
||||
LegacySequelizeRunRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/runRepository');
|
||||
|
||||
const databases = [];
|
||||
const BASE_TIME = 1_750_200_000_000;
|
||||
let idSequence = 1_500;
|
||||
|
||||
function nextId() {
|
||||
idSequence += 1;
|
||||
return `019f7130-0000-7000-8000-${String(idSequence).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
async function createRuntime() {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
const migrationModel = defineSchemaMigrationModel(database);
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
runCancellationRequestMigration,
|
||||
runAttemptDeadlineMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
databases.push(database);
|
||||
return {
|
||||
repository: new LegacySequelizeRunRepository(database),
|
||||
source: new LegacySequelizePrimaryCancellationSource(database),
|
||||
};
|
||||
}
|
||||
|
||||
function createRun(overrides = {}) {
|
||||
return {
|
||||
id: nextId(),
|
||||
projectId: 'default',
|
||||
taskId: 'cancel-task',
|
||||
taskRevision: 'revision-1',
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
status: 'running',
|
||||
version: 7,
|
||||
eventSequence: 7,
|
||||
priority: 0,
|
||||
createdAtMs: BASE_TIME,
|
||||
startedAtMs: BASE_TIME + 5,
|
||||
cancelRequestedAtMs: BASE_TIME + 10,
|
||||
cancelReason: 'user',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createAttempt(runId, overrides = {}) {
|
||||
return {
|
||||
id: nextId(),
|
||||
runId,
|
||||
attempt: 1,
|
||||
status: 'running',
|
||||
executorType: 'local_process',
|
||||
executorHandle: `durable:${runId}`,
|
||||
pid: 4321,
|
||||
callbackSequence: 0,
|
||||
createdAtMs: BASE_TIME,
|
||||
startedAtMs: BASE_TIME + 5,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function seed(repository, run, attempts = []) {
|
||||
await repository.transaction(async (transaction) => {
|
||||
await transaction.insertRun(run);
|
||||
for (const attempt of attempts) await transaction.insertAttempt(attempt);
|
||||
});
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(databases.splice(0).map((database) => database.close()));
|
||||
});
|
||||
|
||||
test('pages only runtime-owned non-terminal cancellation requests', async () => {
|
||||
const { repository, source } = await createRuntime();
|
||||
const first = createRun();
|
||||
const second = createRun({
|
||||
status: 'dispatching',
|
||||
startedAtMs: undefined,
|
||||
cancelRequestedAtMs: BASE_TIME + 11,
|
||||
cancelReason: 'shutdown',
|
||||
});
|
||||
const terminal = createRun({
|
||||
status: 'cancelled',
|
||||
cancelRequestedAtMs: BASE_TIME + 9,
|
||||
});
|
||||
const legacy = createRun({
|
||||
executionOwner: 'legacy',
|
||||
cancelRequestedAtMs: BASE_TIME + 8,
|
||||
});
|
||||
const untouched = createRun({
|
||||
cancelRequestedAtMs: undefined,
|
||||
cancelReason: undefined,
|
||||
});
|
||||
const firstAttempt = createAttempt(first.id);
|
||||
const secondAttempt = createAttempt(second.id, {
|
||||
status: 'starting',
|
||||
pid: undefined,
|
||||
});
|
||||
await seed(repository, first, [firstAttempt]);
|
||||
await seed(repository, second, [secondAttempt]);
|
||||
await seed(repository, terminal, [createAttempt(terminal.id)]);
|
||||
await seed(repository, legacy, [createAttempt(legacy.id)]);
|
||||
await seed(repository, untouched, [createAttempt(untouched.id)]);
|
||||
|
||||
const page1 = await source.listCandidates({ limit: 1 });
|
||||
assert.equal(page1.truncated, true);
|
||||
assert.deepEqual(page1.candidates, [
|
||||
{
|
||||
runId: first.id,
|
||||
requestedAtMs: BASE_TIME + 10,
|
||||
reason: 'user',
|
||||
attempts: [
|
||||
{
|
||||
attemptId: firstAttempt.id,
|
||||
executorType: 'local_process',
|
||||
executorHandle: `durable:${first.id}`,
|
||||
pid: 4321,
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
|
||||
const page2 = await source.listCandidates({
|
||||
limit: 1,
|
||||
cursor: page1.nextCursor,
|
||||
});
|
||||
assert.equal(page2.truncated, false);
|
||||
assert.equal(page2.candidates[0].runId, second.id);
|
||||
assert.equal(page2.candidates[0].reason, 'shutdown');
|
||||
assert.equal(page2.candidates[0].attempts[0].pid, undefined);
|
||||
await assert.rejects(source.listCandidates({ limit: 65 }), RangeError);
|
||||
});
|
||||
|
||||
test('fails closed when active Attempt corruption exceeds the bounded budget', async () => {
|
||||
const { repository, source } = await createRuntime();
|
||||
const run = createRun();
|
||||
await seed(repository, run, [
|
||||
createAttempt(run.id, { attempt: 1 }),
|
||||
createAttempt(run.id, { attempt: 2 }),
|
||||
createAttempt(run.id, { attempt: 3 }),
|
||||
]);
|
||||
|
||||
const page = await source.listCandidates({ limit: 1 });
|
||||
assert.equal(page.unsafeAttemptOverflow, true);
|
||||
assert.deepEqual(page.candidates, []);
|
||||
assert.equal(page.nextCursor, undefined);
|
||||
});
|
||||
@@ -0,0 +1,147 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PrimaryCancellationSupervisor,
|
||||
} = require('../../back/runtime/application/primaryCancellationSupervisor');
|
||||
|
||||
function summary(overrides = {}) {
|
||||
return {
|
||||
scanned: 1,
|
||||
claimed: 1,
|
||||
terminationRequested: 1,
|
||||
alreadyExited: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
blocked: 0,
|
||||
deferred: 0,
|
||||
alreadyResolved: 0,
|
||||
notEligible: 0,
|
||||
failed: 0,
|
||||
truncated: false,
|
||||
unsafeAttemptOverflow: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('paginates a bounded cancellation recovery cycle and aggregates results', async () => {
|
||||
const calls = [];
|
||||
const pages = [
|
||||
summary({
|
||||
scanned: 2,
|
||||
claimed: 1,
|
||||
pending: 1,
|
||||
truncated: true,
|
||||
nextCursor: { requestedAtMs: 100, runId: 'run-2' },
|
||||
}),
|
||||
summary({
|
||||
scanned: 2,
|
||||
claimed: 2,
|
||||
terminationRequested: 0,
|
||||
alreadyExited: 1,
|
||||
blocked: 1,
|
||||
}),
|
||||
];
|
||||
const supervisor = new PrimaryCancellationSupervisor({
|
||||
async dispatchBatch(options) {
|
||||
calls.push(options);
|
||||
return pages.shift();
|
||||
},
|
||||
});
|
||||
|
||||
const result = await supervisor.runCycle({ pageSize: 2, maxPages: 4 });
|
||||
assert.deepEqual(calls, [
|
||||
{ limit: 2 },
|
||||
{ cursor: { requestedAtMs: 100, runId: 'run-2' }, limit: 2 },
|
||||
]);
|
||||
assert.deepEqual(result, {
|
||||
pages: 2,
|
||||
scanned: 4,
|
||||
claimed: 3,
|
||||
terminationRequested: 1,
|
||||
alreadyExited: 1,
|
||||
pending: 1,
|
||||
ambiguous: 0,
|
||||
blocked: 1,
|
||||
deferred: 0,
|
||||
alreadyResolved: 0,
|
||||
notEligible: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
});
|
||||
});
|
||||
|
||||
test('stops at the page limit and exposes a resume cursor', async () => {
|
||||
let page = 0;
|
||||
const supervisor = new PrimaryCancellationSupervisor({
|
||||
async dispatchBatch() {
|
||||
page += 1;
|
||||
return summary({
|
||||
truncated: true,
|
||||
nextCursor: { requestedAtMs: 100 + page, runId: `run-${page}` },
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
const result = await supervisor.runCycle({ pageSize: 1, maxPages: 2 });
|
||||
assert.equal(result.pages, 2);
|
||||
assert.equal(result.stopReason, 'page_limit');
|
||||
assert.equal(result.remaining, true);
|
||||
assert.deepEqual(result.nextCursor, {
|
||||
requestedAtMs: 102,
|
||||
runId: 'run-2',
|
||||
});
|
||||
|
||||
const resumedCalls = [];
|
||||
const resumed = new PrimaryCancellationSupervisor({
|
||||
async dispatchBatch(options) {
|
||||
resumedCalls.push(options);
|
||||
return summary();
|
||||
},
|
||||
});
|
||||
await resumed.runCycle({ cursor: result.nextCursor, pageSize: 1 });
|
||||
assert.deepEqual(resumedCalls, [
|
||||
{
|
||||
cursor: { requestedAtMs: 102, runId: 'run-2' },
|
||||
limit: 1,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('fails closed when attempt rows overflow or pagination cannot advance', async () => {
|
||||
const unsafe = new PrimaryCancellationSupervisor({
|
||||
async dispatchBatch() {
|
||||
return summary({
|
||||
scanned: 0,
|
||||
claimed: 0,
|
||||
terminationRequested: 0,
|
||||
truncated: true,
|
||||
unsafeAttemptOverflow: true,
|
||||
});
|
||||
},
|
||||
});
|
||||
const unsafeResult = await unsafe.runCycle();
|
||||
assert.equal(unsafeResult.stopReason, 'unsafe_attempt_overflow');
|
||||
assert.equal(unsafeResult.remaining, true);
|
||||
|
||||
const stalled = new PrimaryCancellationSupervisor({
|
||||
async dispatchBatch() {
|
||||
return summary({ truncated: true });
|
||||
},
|
||||
});
|
||||
const stalledResult = await stalled.runCycle();
|
||||
assert.equal(stalledResult.stopReason, 'cursor_stalled');
|
||||
assert.equal(stalledResult.remaining, true);
|
||||
});
|
||||
|
||||
test('rejects unbounded cycle settings', async () => {
|
||||
const supervisor = new PrimaryCancellationSupervisor({
|
||||
async dispatchBatch() {
|
||||
throw new Error('must not run');
|
||||
},
|
||||
});
|
||||
await assert.rejects(supervisor.runCycle({ pageSize: 65 }), RangeError);
|
||||
await assert.rejects(supervisor.runCycle({ maxPages: 65 }), RangeError);
|
||||
});
|
||||
@@ -0,0 +1,148 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PrimaryCompletionReceiptJournalScanner,
|
||||
} = require('../../back/runtime/application/primaryCompletionReceiptJournalScanner');
|
||||
|
||||
const NOW = 1_751_000_000_000;
|
||||
|
||||
function candidate(attemptId, overrides = {}) {
|
||||
return {
|
||||
attemptId,
|
||||
runId: `run-${attemptId}`,
|
||||
state: 'pending',
|
||||
registeredAtMs: NOW - 1_000,
|
||||
updatedAtMs: NOW - 1_000,
|
||||
attemptStatus: 'running',
|
||||
executorType: 'local_process',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('scans pending journal rows and expires only old terminal missing receipts', async () => {
|
||||
const resolved = [];
|
||||
const consumed = [];
|
||||
const scanner = new PrimaryCompletionReceiptJournalScanner(
|
||||
{
|
||||
async listCandidates() {
|
||||
return {
|
||||
candidates: [
|
||||
candidate('applied'),
|
||||
candidate('terminal'),
|
||||
candidate('missing-active'),
|
||||
candidate('missing-terminal', {
|
||||
attemptStatus: 'succeeded',
|
||||
finishedAtMs: NOW - 61_000,
|
||||
}),
|
||||
candidate('quarantined'),
|
||||
candidate('remote', { executorType: 'remote_worker' }),
|
||||
candidate('failed'),
|
||||
],
|
||||
truncated: false,
|
||||
};
|
||||
},
|
||||
async resolve(attemptId) {
|
||||
resolved.push(attemptId);
|
||||
return true;
|
||||
},
|
||||
},
|
||||
{},
|
||||
{
|
||||
async consume(attemptId) {
|
||||
consumed.push(attemptId);
|
||||
if (attemptId === 'failed') throw new Error('transient database error');
|
||||
if (attemptId.startsWith('missing')) {
|
||||
return { status: 'missing', cleaned: false };
|
||||
}
|
||||
if (attemptId === 'terminal') {
|
||||
return { status: 'already_terminal', cleaned: true };
|
||||
}
|
||||
if (attemptId === 'quarantined') {
|
||||
return { status: 'quarantined', cleaned: true };
|
||||
}
|
||||
return { status: 'applied', cleaned: true };
|
||||
},
|
||||
},
|
||||
{ terminalMissingRetentionMs: 60_000, clock: { now: () => NOW } },
|
||||
);
|
||||
|
||||
assert.deepEqual(await scanner.scanBatch({ limit: 8 }), {
|
||||
scanned: 7,
|
||||
applied: 1,
|
||||
alreadyTerminal: 1,
|
||||
quarantined: 1,
|
||||
purgedQuarantines: 0,
|
||||
expiredMissing: 1,
|
||||
missing: 1,
|
||||
cleanupPending: 0,
|
||||
skipped: 1,
|
||||
ambiguous: 0,
|
||||
failed: 1,
|
||||
truncated: false,
|
||||
unsafeAttemptOverflow: false,
|
||||
});
|
||||
assert.deepEqual(resolved, ['missing-terminal']);
|
||||
assert.deepEqual(consumed, [
|
||||
'applied',
|
||||
'terminal',
|
||||
'missing-active',
|
||||
'missing-terminal',
|
||||
'quarantined',
|
||||
'failed',
|
||||
]);
|
||||
});
|
||||
|
||||
test('purges due quarantine through a known Attempt path and advances cursor', async () => {
|
||||
const calls = [];
|
||||
const scanner = new PrimaryCompletionReceiptJournalScanner(
|
||||
{
|
||||
async listCandidates(options) {
|
||||
calls.push(['list', options.cursor]);
|
||||
return {
|
||||
candidates: [
|
||||
candidate('due', {
|
||||
state: 'quarantined',
|
||||
quarantineRef: '.quarantine/du/due.json',
|
||||
purgeAfterMs: NOW,
|
||||
}),
|
||||
],
|
||||
truncated: true,
|
||||
nextCursor: { updatedAtMs: NOW - 1, attemptId: 'due' },
|
||||
};
|
||||
},
|
||||
async resolve(attemptId) {
|
||||
calls.push(['resolve', attemptId]);
|
||||
return true;
|
||||
},
|
||||
},
|
||||
{
|
||||
async quarantine(attemptId) {
|
||||
calls.push(['quarantine', attemptId]);
|
||||
return '.quarantine/du/due.json';
|
||||
},
|
||||
async purgeQuarantine(attemptId) {
|
||||
calls.push(['purge', attemptId]);
|
||||
return true;
|
||||
},
|
||||
},
|
||||
{ async consume() {} },
|
||||
{ clock: { now: () => NOW } },
|
||||
);
|
||||
|
||||
const result = await scanner.scanBatch({
|
||||
cursor: { createdAtMs: NOW - 2, runId: 'before' },
|
||||
});
|
||||
assert.equal(result.purgedQuarantines, 1);
|
||||
assert.deepEqual(result.nextCursor, {
|
||||
createdAtMs: NOW - 1,
|
||||
runId: 'due',
|
||||
});
|
||||
assert.deepEqual(calls, [
|
||||
['list', { updatedAtMs: NOW - 2, attemptId: 'before' }],
|
||||
['quarantine', 'due'],
|
||||
['purge', 'due'],
|
||||
['resolve', 'due'],
|
||||
]);
|
||||
});
|
||||
@@ -0,0 +1,262 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PrimaryCompletionReceiptLifecycle,
|
||||
} = require('../../back/runtime/application/primaryCompletionReceiptLifecycle');
|
||||
const {
|
||||
PrimaryCompletionReceiptScanner,
|
||||
} = require('../../back/runtime/application/primaryCompletionReceiptScanner');
|
||||
const {
|
||||
PrimaryCompletionReceiptSupervisor,
|
||||
} = require('../../back/runtime/application/primaryCompletionReceiptSupervisor');
|
||||
|
||||
function candidate(runId, attemptId, executorType = 'local_process') {
|
||||
return {
|
||||
runId,
|
||||
attempts: attemptId ? [{ attemptId, executorType }] : [],
|
||||
};
|
||||
}
|
||||
|
||||
function page(overrides = {}) {
|
||||
return {
|
||||
candidates: [],
|
||||
truncated: false,
|
||||
unsafeAttemptOverflow: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function scan(overrides = {}) {
|
||||
return {
|
||||
scanned: 1,
|
||||
applied: 0,
|
||||
alreadyTerminal: 0,
|
||||
quarantined: 0,
|
||||
purgedQuarantines: 0,
|
||||
expiredMissing: 0,
|
||||
missing: 1,
|
||||
cleanupPending: 0,
|
||||
skipped: 0,
|
||||
ambiguous: 0,
|
||||
failed: 0,
|
||||
truncated: false,
|
||||
unsafeAttemptOverflow: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('scans only database-discovered local Attempts and isolates failures', async () => {
|
||||
const consumed = [];
|
||||
const scanner = new PrimaryCompletionReceiptScanner(
|
||||
{
|
||||
async listCandidates() {
|
||||
return page({
|
||||
candidates: [
|
||||
candidate('run-1', 'applied'),
|
||||
candidate('run-2', 'terminal'),
|
||||
candidate('run-3', 'missing'),
|
||||
candidate('run-4', 'cleanup-pending'),
|
||||
candidate('run-5', 'failed'),
|
||||
candidate('run-6', 'quarantined'),
|
||||
candidate('run-7', 'remote', 'remote_worker'),
|
||||
candidate('run-8'),
|
||||
{
|
||||
runId: 'run-9',
|
||||
attempts: [
|
||||
{ attemptId: 'one', executorType: 'local_process' },
|
||||
{ attemptId: 'two', executorType: 'local_process' },
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
},
|
||||
},
|
||||
{
|
||||
async consume(attemptId) {
|
||||
consumed.push(attemptId);
|
||||
if (attemptId === 'failed') throw new Error('invalid receipt');
|
||||
if (attemptId === 'missing') {
|
||||
return { status: 'missing', cleaned: false };
|
||||
}
|
||||
if (attemptId === 'terminal') {
|
||||
return { status: 'already_terminal', cleaned: true };
|
||||
}
|
||||
if (attemptId === 'quarantined') {
|
||||
return {
|
||||
status: 'quarantined',
|
||||
cleaned: true,
|
||||
quarantineRef: '.quarantine/quarantined.json',
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: 'applied',
|
||||
cleaned: attemptId !== 'cleanup-pending',
|
||||
};
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
assert.deepEqual(await scanner.scanBatch({ limit: 8 }), {
|
||||
scanned: 9,
|
||||
applied: 2,
|
||||
alreadyTerminal: 1,
|
||||
quarantined: 1,
|
||||
purgedQuarantines: 0,
|
||||
expiredMissing: 0,
|
||||
missing: 1,
|
||||
cleanupPending: 1,
|
||||
skipped: 1,
|
||||
ambiguous: 2,
|
||||
failed: 1,
|
||||
truncated: false,
|
||||
unsafeAttemptOverflow: false,
|
||||
});
|
||||
assert.deepEqual(consumed, [
|
||||
'applied',
|
||||
'terminal',
|
||||
'missing',
|
||||
'cleanup-pending',
|
||||
'failed',
|
||||
'quarantined',
|
||||
]);
|
||||
});
|
||||
|
||||
test('supervisor aggregates bounded pages and exposes unsafe pagination', async () => {
|
||||
const cursors = [];
|
||||
const responses = [
|
||||
scan({
|
||||
applied: 1,
|
||||
missing: 0,
|
||||
truncated: true,
|
||||
nextCursor: { createdAtMs: 10, runId: 'run-1' },
|
||||
}),
|
||||
scan({ alreadyTerminal: 1, cleanupPending: 1 }),
|
||||
];
|
||||
const supervisor = new PrimaryCompletionReceiptSupervisor({
|
||||
async scanBatch(options) {
|
||||
cursors.push(options.cursor);
|
||||
return responses.shift();
|
||||
},
|
||||
});
|
||||
assert.deepEqual(await supervisor.run({ pageSize: 8, maxPages: 4 }), {
|
||||
pages: 2,
|
||||
scanned: 2,
|
||||
applied: 1,
|
||||
alreadyTerminal: 1,
|
||||
quarantined: 0,
|
||||
purgedQuarantines: 0,
|
||||
expiredMissing: 0,
|
||||
missing: 1,
|
||||
cleanupPending: 1,
|
||||
skipped: 0,
|
||||
ambiguous: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
});
|
||||
assert.deepEqual(cursors, [undefined, { createdAtMs: 10, runId: 'run-1' }]);
|
||||
|
||||
const overflow = new PrimaryCompletionReceiptSupervisor({
|
||||
async scanBatch() {
|
||||
return scan({ unsafeAttemptOverflow: true, truncated: true });
|
||||
},
|
||||
});
|
||||
assert.equal((await overflow.run()).stopReason, 'unsafe_attempt_overflow');
|
||||
await assert.rejects(supervisor.run({ pageSize: 0 }), RangeError);
|
||||
await assert.rejects(supervisor.run({ maxPages: 65 }), RangeError);
|
||||
});
|
||||
|
||||
test('lifecycle is explicit, unrefed, and never overlaps a slow scan', async () => {
|
||||
const scheduled = [];
|
||||
const scheduler = {
|
||||
setTimeout(callback, delayMs) {
|
||||
const timer = {
|
||||
callback,
|
||||
delayMs,
|
||||
unrefCalled: false,
|
||||
unref() {
|
||||
this.unrefCalled = true;
|
||||
},
|
||||
};
|
||||
scheduled.push(timer);
|
||||
return timer;
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
timer.cleared = true;
|
||||
},
|
||||
};
|
||||
let release;
|
||||
let runs = 0;
|
||||
const inFlight = new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const lifecycle = new PrimaryCompletionReceiptLifecycle(
|
||||
{
|
||||
async run() {
|
||||
runs += 1;
|
||||
await inFlight;
|
||||
return {};
|
||||
},
|
||||
},
|
||||
{ intervalMs: 1_000, scheduler },
|
||||
);
|
||||
|
||||
assert.equal(lifecycle.start(), true);
|
||||
assert.equal(lifecycle.start(), false);
|
||||
assert.equal(scheduled.length, 1);
|
||||
assert.equal(scheduled[0].unrefCalled, true);
|
||||
scheduled[0].callback();
|
||||
scheduled[0].callback();
|
||||
assert.equal(runs, 1);
|
||||
release();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.equal(scheduled.length, 2);
|
||||
assert.equal(await lifecycle.stop(), 'drained');
|
||||
assert.equal(scheduled[1].cleared, true);
|
||||
});
|
||||
|
||||
test('lifecycle resumes after a page limit and resets after reaching the tail', async () => {
|
||||
const scheduled = [];
|
||||
const scheduler = {
|
||||
setTimeout(callback) {
|
||||
const timer = { callback, unref() {} };
|
||||
scheduled.push(timer);
|
||||
return timer;
|
||||
},
|
||||
clearTimeout() {},
|
||||
};
|
||||
const cursors = [];
|
||||
let runs = 0;
|
||||
const lifecycle = new PrimaryCompletionReceiptLifecycle(
|
||||
{
|
||||
async run(options) {
|
||||
runs += 1;
|
||||
cursors.push(options.cursor);
|
||||
if (runs === 1) {
|
||||
return {
|
||||
remaining: true,
|
||||
nextCursor: { createdAtMs: 10, runId: 'run-10' },
|
||||
};
|
||||
}
|
||||
return { remaining: false };
|
||||
},
|
||||
},
|
||||
{ intervalMs: 1_000, scheduler },
|
||||
);
|
||||
|
||||
lifecycle.start();
|
||||
scheduled.shift().callback();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
scheduled.shift().callback();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
scheduled.shift().callback();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
await lifecycle.stop();
|
||||
assert.deepEqual(cursors, [
|
||||
undefined,
|
||||
{ createdAtMs: 10, runId: 'run-10' },
|
||||
undefined,
|
||||
]);
|
||||
});
|
||||
@@ -0,0 +1,485 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const { DataTypes, QueryTypes, Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const { runSchemaMigration } = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
runCancellationRequestMigration,
|
||||
} = require('../../back/migrations/0004-run-cancellation-request');
|
||||
const {
|
||||
runAttemptDeadlineMigration,
|
||||
} = require('../../back/migrations/0006-run-attempt-deadline');
|
||||
const {
|
||||
runningInstanceRunReferenceMigration,
|
||||
} = require('../../back/migrations/0003-running-instance-run-reference');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
PrimaryCronProjection,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/primaryCronProjection');
|
||||
const {
|
||||
LegacySequelizeProjectedRunRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/projectedRunRepository');
|
||||
const {
|
||||
LegacySequelizeRunRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/runRepository');
|
||||
const {
|
||||
PrimaryRunCreator,
|
||||
} = require('../../back/runtime/application/primaryRunCreator');
|
||||
const {
|
||||
PrimaryRunOrchestrator,
|
||||
} = require('../../back/runtime/application/primaryRunOrchestrator');
|
||||
const {
|
||||
RunCommandService,
|
||||
} = require('../../back/runtime/application/runCommandService');
|
||||
const {
|
||||
createLegacyLogOutputRef,
|
||||
} = require('../../back/runtime/compatibility/legacyLogOutputRef');
|
||||
|
||||
const databases = [];
|
||||
const BASE_TIME = 1_750_000_000_000;
|
||||
let idSequence = 900;
|
||||
|
||||
function nextId() {
|
||||
idSequence += 1;
|
||||
return '019f7110-0000-7000-8000-' + String(idSequence).padStart(12, '0');
|
||||
}
|
||||
|
||||
function deferred() {
|
||||
let resolve;
|
||||
let reject;
|
||||
const promise = new Promise((resolvePromise, rejectPromise) => {
|
||||
resolve = resolvePromise;
|
||||
reject = rejectPromise;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
async function createLegacyTables(database) {
|
||||
const queryInterface = database.getQueryInterface();
|
||||
await queryInterface.createTable('Crontabs', {
|
||||
id: {
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
autoIncrement: true,
|
||||
},
|
||||
name: { type: DataTypes.STRING, allowNull: true },
|
||||
command: { type: DataTypes.STRING, allowNull: false },
|
||||
status: { type: DataTypes.INTEGER, allowNull: true },
|
||||
pid: { type: DataTypes.INTEGER, allowNull: true },
|
||||
log_path: { type: DataTypes.STRING, allowNull: true },
|
||||
last_running_time: { type: DataTypes.INTEGER, allowNull: true },
|
||||
last_execution_time: { type: DataTypes.INTEGER, allowNull: true },
|
||||
});
|
||||
await queryInterface.createTable('RunningInstances', {
|
||||
id: {
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
autoIncrement: true,
|
||||
},
|
||||
cron_id: { type: DataTypes.INTEGER, allowNull: false },
|
||||
pid: { type: DataTypes.INTEGER, allowNull: true },
|
||||
log_path: { type: DataTypes.STRING, allowNull: true },
|
||||
started_at: { type: DataTypes.INTEGER, allowNull: false },
|
||||
finished_at: { type: DataTypes.INTEGER, allowNull: true },
|
||||
status: { type: DataTypes.INTEGER, allowNull: false },
|
||||
exit_code: { type: DataTypes.INTEGER, allowNull: true },
|
||||
});
|
||||
}
|
||||
|
||||
async function createDatabase() {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
await createLegacyTables(database);
|
||||
const migrationModel = defineSchemaMigrationModel(database);
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
runningInstanceRunReferenceMigration,
|
||||
runCancellationRequestMigration,
|
||||
runAttemptDeadlineMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
databases.push(database);
|
||||
return database;
|
||||
}
|
||||
|
||||
async function seedCron(database, id) {
|
||||
await database.getQueryInterface().bulkInsert('Crontabs', [
|
||||
{
|
||||
id,
|
||||
name: 'cron-' + id,
|
||||
command: 'echo projected',
|
||||
status: 1,
|
||||
pid: null,
|
||||
log_path: null,
|
||||
last_running_time: null,
|
||||
last_execution_time: null,
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
async function readCron(database, id) {
|
||||
const rows = await database.query(
|
||||
'SELECT id, status, pid, log_path, last_running_time, last_execution_time FROM Crontabs WHERE id = :id',
|
||||
{
|
||||
replacements: { id },
|
||||
type: QueryTypes.SELECT,
|
||||
},
|
||||
);
|
||||
return rows[0] || null;
|
||||
}
|
||||
|
||||
async function readInstances(database, cronId) {
|
||||
return database.query(
|
||||
'SELECT id, cron_id, run_id, attempt_id, pid, log_path, started_at, finished_at, status, exit_code FROM RunningInstances WHERE cron_id = :cronId ORDER BY id',
|
||||
{
|
||||
replacements: { cronId },
|
||||
type: QueryTypes.SELECT,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function projectedRepository(database, extraParticipants = []) {
|
||||
return new LegacySequelizeProjectedRunRepository(database, [
|
||||
new PrimaryCronProjection(database),
|
||||
...extraParticipants,
|
||||
]);
|
||||
}
|
||||
|
||||
class FakeExecutor {
|
||||
type = 'local_process';
|
||||
completion = deferred();
|
||||
|
||||
constructor(pid, startedAtMs) {
|
||||
this.pid = pid;
|
||||
this.startedAtMs = startedAtMs;
|
||||
}
|
||||
|
||||
capabilities() {
|
||||
return {
|
||||
timeout: true,
|
||||
processGroupTermination: true,
|
||||
workingDirectory: true,
|
||||
isolatedEnvironment: true,
|
||||
memoryLimit: 'none',
|
||||
cpuLimit: 'none',
|
||||
filesystemIsolation: 'none',
|
||||
networkIsolation: 'none',
|
||||
};
|
||||
}
|
||||
|
||||
async start(spec) {
|
||||
return {
|
||||
id: 'handle-' + this.pid,
|
||||
executorType: this.type,
|
||||
runId: spec.runId,
|
||||
attemptId: spec.attemptId,
|
||||
startedAtMs: this.startedAtMs,
|
||||
pid: this.pid,
|
||||
completion: this.completion.promise,
|
||||
};
|
||||
}
|
||||
|
||||
async stop() {
|
||||
return {
|
||||
status: 'termination_requested',
|
||||
termSignalSent: true,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
|
||||
async inspect() {
|
||||
return { status: 'running' };
|
||||
}
|
||||
}
|
||||
|
||||
function startCommand(cronId, acceptedAtMs, logPath) {
|
||||
return {
|
||||
definition: {
|
||||
projectId: 'default',
|
||||
taskId: 'legacy-cron:' + cronId,
|
||||
taskRevision: 'revision-' + cronId,
|
||||
taskName: 'projected run ' + cronId,
|
||||
legacyCronId: cronId,
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
outputRef: createLegacyLogOutputRef(logPath),
|
||||
acceptedAtMs,
|
||||
actor: { type: 'user', id: 'user:1' },
|
||||
},
|
||||
createSpec(reference) {
|
||||
return {
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
projectId: reference.run.projectId,
|
||||
taskId: reference.run.taskId,
|
||||
taskRevision: reference.run.taskRevision,
|
||||
command: { kind: 'argv', file: '/bin/true', args: [] },
|
||||
environmentPolicy: 'isolated',
|
||||
terminationGraceMs: 100,
|
||||
};
|
||||
},
|
||||
context: {
|
||||
environment: {},
|
||||
output: { async write() {} },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function result(outcome, startedAtMs, finishedAtMs, exitCode) {
|
||||
return {
|
||||
outcome,
|
||||
startedAtMs,
|
||||
finishedAtMs,
|
||||
...(exitCode === undefined ? {} : { exitCode }),
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(databases.splice(0).map((database) => database.close()));
|
||||
});
|
||||
|
||||
test('projects a Primary Run and RunningInstance through success', async () => {
|
||||
const database = await createDatabase();
|
||||
await seedCron(database, 7);
|
||||
const repository = projectedRepository(database);
|
||||
const executor = new FakeExecutor(4321, BASE_TIME + 2_000);
|
||||
const orchestrator = new PrimaryRunOrchestrator(repository, executor, {
|
||||
clock: { now: () => BASE_TIME + 3_000 },
|
||||
createId: nextId,
|
||||
});
|
||||
|
||||
const active = await orchestrator.start(
|
||||
startCommand(7, BASE_TIME, 'task-7/primary.log'),
|
||||
);
|
||||
const runningCron = await readCron(database, 7);
|
||||
assert.equal(runningCron.status, 0);
|
||||
assert.equal(runningCron.pid, 4321);
|
||||
assert.equal(runningCron.log_path, 'task-7/primary.log');
|
||||
assert.equal(runningCron.last_execution_time, 1_750_000_003);
|
||||
|
||||
let instances = await readInstances(database, 7);
|
||||
assert.equal(instances.length, 1);
|
||||
assert.equal(instances[0].run_id, active.run.id);
|
||||
assert.equal(instances[0].attempt_id, active.attempt.id);
|
||||
assert.equal(instances[0].status, 0);
|
||||
assert.equal(instances[0].pid, 4321);
|
||||
assert.equal(instances[0].log_path, 'task-7/primary.log');
|
||||
|
||||
executor.completion.resolve(
|
||||
result('succeeded', BASE_TIME + 2_000, BASE_TIME + 7_000, 0),
|
||||
);
|
||||
await active.completion;
|
||||
|
||||
const finishedCron = await readCron(database, 7);
|
||||
assert.equal(finishedCron.status, 1);
|
||||
assert.equal(finishedCron.pid, null);
|
||||
assert.equal(finishedCron.log_path, 'task-7/primary.log');
|
||||
assert.equal(finishedCron.last_running_time, 4);
|
||||
instances = await readInstances(database, 7);
|
||||
assert.equal(instances.length, 1);
|
||||
assert.equal(instances[0].status, 1);
|
||||
assert.equal(instances[0].finished_at, 1_750_000_007);
|
||||
assert.equal(instances[0].exit_code, 0);
|
||||
});
|
||||
|
||||
test('maps failed and cancelled attempts into legacy instance states', async () => {
|
||||
const cases = [
|
||||
{ cronId: 8, outcome: 'failed', expectedStatus: 3, exitCode: 17 },
|
||||
{ cronId: 9, outcome: 'cancelled', expectedStatus: 2, exitCode: 143 },
|
||||
];
|
||||
|
||||
for (const item of cases) {
|
||||
const database = await createDatabase();
|
||||
await seedCron(database, item.cronId);
|
||||
const repository = projectedRepository(database);
|
||||
const executor = new FakeExecutor(4400 + item.cronId, BASE_TIME + 1_000);
|
||||
const orchestrator = new PrimaryRunOrchestrator(repository, executor, {
|
||||
clock: { now: () => BASE_TIME + 2_000 },
|
||||
createId: nextId,
|
||||
});
|
||||
const active = await orchestrator.start(
|
||||
startCommand(
|
||||
item.cronId,
|
||||
BASE_TIME,
|
||||
'task-' + item.cronId + '/primary.log',
|
||||
),
|
||||
);
|
||||
|
||||
executor.completion.resolve(
|
||||
result(item.outcome, BASE_TIME + 1_000, BASE_TIME + 4_000, item.exitCode),
|
||||
);
|
||||
await active.completion;
|
||||
|
||||
const cron = await readCron(database, item.cronId);
|
||||
const instances = await readInstances(database, item.cronId);
|
||||
assert.equal(cron.status, 1);
|
||||
assert.equal(cron.pid, null);
|
||||
assert.equal(instances.length, 1);
|
||||
assert.equal(instances[0].status, item.expectedStatus);
|
||||
assert.equal(instances[0].exit_code, item.exitCode);
|
||||
}
|
||||
});
|
||||
|
||||
test('keeps Crontab running while another Primary instance remains active', async () => {
|
||||
const database = await createDatabase();
|
||||
await seedCron(database, 10);
|
||||
const repository = projectedRepository(database);
|
||||
const firstExecutor = new FakeExecutor(5101, BASE_TIME + 1_000);
|
||||
const firstOrchestrator = new PrimaryRunOrchestrator(
|
||||
repository,
|
||||
firstExecutor,
|
||||
{
|
||||
clock: { now: () => BASE_TIME + 2_000 },
|
||||
createId: nextId,
|
||||
},
|
||||
);
|
||||
const first = await firstOrchestrator.start(
|
||||
startCommand(10, BASE_TIME, 'task-10/first.log'),
|
||||
);
|
||||
|
||||
const secondExecutor = new FakeExecutor(5102, BASE_TIME + 101_000);
|
||||
const secondOrchestrator = new PrimaryRunOrchestrator(
|
||||
repository,
|
||||
secondExecutor,
|
||||
{
|
||||
clock: { now: () => BASE_TIME + 102_000 },
|
||||
createId: nextId,
|
||||
},
|
||||
);
|
||||
const second = await secondOrchestrator.start(
|
||||
startCommand(10, BASE_TIME + 100_000, 'task-10/second.log'),
|
||||
);
|
||||
assert.equal((await readCron(database, 10)).pid, 5102);
|
||||
|
||||
secondExecutor.completion.resolve(
|
||||
result('succeeded', BASE_TIME + 101_000, BASE_TIME + 104_000, 0),
|
||||
);
|
||||
await second.completion;
|
||||
const stillRunning = await readCron(database, 10);
|
||||
assert.equal(stillRunning.status, 0);
|
||||
assert.equal(stillRunning.pid, 5101);
|
||||
assert.equal(stillRunning.log_path, 'task-10/first.log');
|
||||
|
||||
firstExecutor.completion.resolve(
|
||||
result('succeeded', BASE_TIME + 1_000, BASE_TIME + 106_000, 0),
|
||||
);
|
||||
await first.completion;
|
||||
assert.equal((await readCron(database, 10)).status, 1);
|
||||
assert.equal((await readInstances(database, 10)).length, 2);
|
||||
});
|
||||
|
||||
test('rolls back Run, event and instance writes when projection fails', async () => {
|
||||
const database = await createDatabase();
|
||||
await seedCron(database, 11);
|
||||
const failure = {
|
||||
enabled: false,
|
||||
async apply() {
|
||||
if (this.enabled) throw new Error('injected projection failure');
|
||||
},
|
||||
};
|
||||
const repository = projectedRepository(database, [failure]);
|
||||
const creator = new PrimaryRunCreator(repository, nextId);
|
||||
const reference = await creator.create(
|
||||
{
|
||||
projectId: 'default',
|
||||
taskId: 'legacy-cron:11',
|
||||
taskRevision: 'revision-11',
|
||||
legacyCronId: 11,
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
outputRef: createLegacyLogOutputRef('task-11/rollback.log'),
|
||||
acceptedAtMs: BASE_TIME,
|
||||
actor: { type: 'user', id: 'user:1' },
|
||||
},
|
||||
'local_process',
|
||||
);
|
||||
const commands = new RunCommandService(repository, nextId);
|
||||
const dispatching = await commands.transitionRun({
|
||||
runId: reference.run.id,
|
||||
to: 'dispatching',
|
||||
expectedVersion: reference.run.version,
|
||||
atMs: BASE_TIME + 1_000,
|
||||
actor: { type: 'system' },
|
||||
});
|
||||
failure.enabled = true;
|
||||
|
||||
await assert.rejects(
|
||||
commands.transitionRunAttempt({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
to: 'starting',
|
||||
expectedRunVersion: dispatching.run.version,
|
||||
atMs: BASE_TIME + 2_000,
|
||||
actor: { type: 'executor', id: 'local_process' },
|
||||
}),
|
||||
/injected projection failure/,
|
||||
);
|
||||
|
||||
const run = await repository.findRunById(reference.run.id);
|
||||
const attempt = await repository.findAttemptById(reference.attempt.id);
|
||||
assert.equal(run.status, 'dispatching');
|
||||
assert.equal(run.version, dispatching.run.version);
|
||||
assert.equal(attempt.status, 'claimed');
|
||||
assert.equal((await repository.listEvents(run.id)).length, 3);
|
||||
assert.equal((await readInstances(database, 11)).length, 0);
|
||||
assert.equal((await readCron(database, 11)).status, 3);
|
||||
});
|
||||
|
||||
test('legacy Shadow repository never writes the Primary projection', async () => {
|
||||
const database = await createDatabase();
|
||||
await seedCron(database, 12);
|
||||
const repository = new LegacySequelizeRunRepository(database);
|
||||
const run = {
|
||||
id: nextId(),
|
||||
projectId: 'default',
|
||||
taskId: 'legacy-cron:12',
|
||||
taskRevision: 'revision-12',
|
||||
legacyCronId: 12,
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'legacy',
|
||||
status: 'running',
|
||||
version: 0,
|
||||
eventSequence: 0,
|
||||
priority: 0,
|
||||
outputRef: createLegacyLogOutputRef('task-12/shadow.log'),
|
||||
createdAtMs: BASE_TIME,
|
||||
startedAtMs: BASE_TIME + 1_000,
|
||||
};
|
||||
const attempt = {
|
||||
id: nextId(),
|
||||
runId: run.id,
|
||||
attempt: 1,
|
||||
status: 'running',
|
||||
executorType: 'legacy_local',
|
||||
pid: 6200,
|
||||
callbackSequence: 0,
|
||||
createdAtMs: BASE_TIME,
|
||||
startedAtMs: BASE_TIME + 1_000,
|
||||
};
|
||||
await repository.transaction(async (transaction) => {
|
||||
await transaction.insertRun(run);
|
||||
await transaction.insertAttempt(attempt);
|
||||
});
|
||||
|
||||
const cron = await readCron(database, 12);
|
||||
assert.equal(cron.status, 1);
|
||||
assert.equal(cron.pid, null);
|
||||
assert.equal(
|
||||
await readInstances(database, 12).then((rows) => rows.length),
|
||||
0,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,412 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const { Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const { runSchemaMigration } = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
runCancellationRequestMigration,
|
||||
} = require('../../back/migrations/0004-run-cancellation-request');
|
||||
const {
|
||||
runAttemptDeadlineMigration,
|
||||
} = require('../../back/migrations/0006-run-attempt-deadline');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
CompletionReceiptFileStore,
|
||||
} = require('../../back/runtime/adapters/fs/completionReceiptFileStore');
|
||||
const {
|
||||
LegacySequelizeRunRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/runRepository');
|
||||
const {
|
||||
PrimaryCompletionReceiptConsumer,
|
||||
} = require('../../back/runtime/application/primaryCompletionReceiptConsumer');
|
||||
const {
|
||||
PrimaryRunCompletionService,
|
||||
PrimaryCompletionUnauthorizedError,
|
||||
hashPrimaryCompletionToken,
|
||||
} = require('../../back/runtime/application/primaryRunCompletionService');
|
||||
const {
|
||||
PrimaryRunCreator,
|
||||
} = require('../../back/runtime/application/primaryRunCreator');
|
||||
const {
|
||||
RunCommandService,
|
||||
} = require('../../back/runtime/application/runCommandService');
|
||||
|
||||
const NOW = 1_750_000_000_000;
|
||||
const TOKEN = 'completion_token_abcdefghijklmnopqrstuvwxyz0123456789';
|
||||
let idSequence = 900;
|
||||
|
||||
function nextId() {
|
||||
idSequence += 1;
|
||||
return `019f70f0-0000-7000-8000-${String(idSequence).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
async function setup(t) {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
runCancellationRequestMigration,
|
||||
runAttemptDeadlineMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
return new LegacySequelizeRunRepository(database);
|
||||
}
|
||||
|
||||
async function runningReference(repository) {
|
||||
const creator = new PrimaryRunCreator(repository, nextId);
|
||||
const commands = new RunCommandService(repository, nextId);
|
||||
const created = await creator.create(
|
||||
{
|
||||
projectId: 'default',
|
||||
taskId: 'legacy-cron:9',
|
||||
taskRevision: 'revision-9',
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
acceptedAtMs: NOW,
|
||||
actor: { type: 'user', id: 'user:1' },
|
||||
},
|
||||
'local_process',
|
||||
);
|
||||
const dispatching = await commands.transitionRun({
|
||||
runId: created.run.id,
|
||||
to: 'dispatching',
|
||||
expectedVersion: created.run.version,
|
||||
atMs: NOW + 1,
|
||||
actor: { type: 'scheduler' },
|
||||
});
|
||||
const starting = await commands.transitionRunAttempt({
|
||||
runId: created.run.id,
|
||||
attemptId: created.attempt.id,
|
||||
to: 'starting',
|
||||
expectedRunVersion: dispatching.run.version,
|
||||
atMs: NOW + 2,
|
||||
callbackTokenHash: hashPrimaryCompletionToken(TOKEN),
|
||||
actor: { type: 'worker', id: 'local_process' },
|
||||
});
|
||||
const attempt = await commands.transitionRunAttempt({
|
||||
runId: created.run.id,
|
||||
attemptId: created.attempt.id,
|
||||
to: 'running',
|
||||
expectedRunVersion: starting.run.version,
|
||||
atMs: NOW + 3,
|
||||
executorHandle: 'ql3lp1.test-handle',
|
||||
actor: { type: 'executor', id: 'local_process' },
|
||||
});
|
||||
const run = await commands.transitionRun({
|
||||
runId: created.run.id,
|
||||
to: 'running',
|
||||
expectedVersion: attempt.run.version,
|
||||
atMs: NOW + 3,
|
||||
actor: { type: 'executor', id: 'local_process' },
|
||||
});
|
||||
return { run: run.run, attempt: attempt.attempt, commands };
|
||||
}
|
||||
|
||||
function successfulResult() {
|
||||
return {
|
||||
outcome: 'succeeded',
|
||||
startedAtMs: NOW + 3,
|
||||
finishedAtMs: NOW + 10,
|
||||
exitCode: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function receipt(reference) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
callbackSequence: 1,
|
||||
token: TOKEN,
|
||||
startedAtMs: NOW + 3,
|
||||
finishedAtMs: NOW + 10,
|
||||
exitCode: 0,
|
||||
};
|
||||
}
|
||||
|
||||
class FailAfterWorkRepository {
|
||||
fail = true;
|
||||
|
||||
constructor(delegate) {
|
||||
this.delegate = delegate;
|
||||
}
|
||||
|
||||
findRunById(runId) {
|
||||
return this.delegate.findRunById(runId);
|
||||
}
|
||||
|
||||
findAttemptById(attemptId) {
|
||||
return this.delegate.findAttemptById(attemptId);
|
||||
}
|
||||
|
||||
listEvents(runId, options) {
|
||||
return this.delegate.listEvents(runId, options);
|
||||
}
|
||||
|
||||
listCancellationRequested(options) {
|
||||
return this.delegate.listCancellationRequested(options);
|
||||
}
|
||||
|
||||
transaction(work) {
|
||||
return this.delegate.transaction(async (transaction) => {
|
||||
const result = await work(transaction);
|
||||
if (this.fail) {
|
||||
this.fail = false;
|
||||
throw new Error('simulated crash before commit');
|
||||
}
|
||||
return result;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
class FailFirstRemoveStore {
|
||||
fail = true;
|
||||
|
||||
constructor(delegate) {
|
||||
this.delegate = delegate;
|
||||
}
|
||||
|
||||
publish(value) {
|
||||
return this.delegate.publish(value);
|
||||
}
|
||||
|
||||
read(attemptId) {
|
||||
return this.delegate.read(attemptId);
|
||||
}
|
||||
|
||||
async remove(attemptId) {
|
||||
if (this.fail) {
|
||||
this.fail = false;
|
||||
throw new Error('simulated crash before receipt cleanup');
|
||||
}
|
||||
return this.delegate.remove(attemptId);
|
||||
}
|
||||
}
|
||||
|
||||
async function receiptStore(t) {
|
||||
const root = await fs.mkdtemp(
|
||||
path.join(os.tmpdir(), 'ql3-completion-service-'),
|
||||
);
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
return new CompletionReceiptFileStore(root);
|
||||
}
|
||||
|
||||
test('atomically applies one completion and treats an identical replay as terminal', async (t) => {
|
||||
const repository = await setup(t);
|
||||
const reference = await runningReference(repository);
|
||||
const service = new PrimaryRunCompletionService(repository, nextId);
|
||||
const command = {
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
callbackSequence: 1,
|
||||
result: successfulResult(),
|
||||
source: { kind: 'receipt', token: TOKEN },
|
||||
};
|
||||
|
||||
const applied = await service.complete(command);
|
||||
assert.equal(applied.status, 'applied');
|
||||
assert.equal(applied.run.status, 'succeeded');
|
||||
assert.equal(applied.attempt.status, 'succeeded');
|
||||
assert.equal(applied.attempt.callbackSequence, 1);
|
||||
const eventsAfterApply = await repository.listEvents(reference.run.id);
|
||||
assert.doesNotMatch(JSON.stringify(eventsAfterApply), new RegExp(TOKEN));
|
||||
assert.deepEqual(
|
||||
eventsAfterApply.slice(-2).map((event) => event.type),
|
||||
['attempt.succeeded', 'run.succeeded'],
|
||||
);
|
||||
|
||||
const replay = await service.complete(command);
|
||||
assert.equal(replay.status, 'already_terminal');
|
||||
assert.equal(
|
||||
(await repository.listEvents(reference.run.id)).length,
|
||||
eventsAfterApply.length,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects an invalid receipt token without changing durable state', async (t) => {
|
||||
const repository = await setup(t);
|
||||
const reference = await runningReference(repository);
|
||||
const service = new PrimaryRunCompletionService(repository, nextId);
|
||||
const beforeEvents = await repository.listEvents(reference.run.id);
|
||||
|
||||
await assert.rejects(
|
||||
service.complete({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
callbackSequence: 1,
|
||||
result: successfulResult(),
|
||||
source: {
|
||||
kind: 'receipt',
|
||||
token: 'wrong_token_abcdefghijklmnopqrstuvwxyz0123456789',
|
||||
},
|
||||
}),
|
||||
PrimaryCompletionUnauthorizedError,
|
||||
);
|
||||
assert.equal(
|
||||
(await repository.findRunById(reference.run.id)).status,
|
||||
'running',
|
||||
);
|
||||
assert.equal(
|
||||
(await repository.findAttemptById(reference.attempt.id)).callbackSequence,
|
||||
0,
|
||||
);
|
||||
assert.equal(
|
||||
(await repository.listEvents(reference.run.id)).length,
|
||||
beforeEvents.length,
|
||||
);
|
||||
});
|
||||
|
||||
test('quarantines an unauthorized receipt instead of retrying it forever', async (t) => {
|
||||
const repository = await setup(t);
|
||||
const reference = await runningReference(repository);
|
||||
const store = await receiptStore(t);
|
||||
await store.publish({
|
||||
...receipt(reference),
|
||||
token: 'wrong_token_abcdefghijklmnopqrstuvwxyz0123456789',
|
||||
});
|
||||
const quarantines = [];
|
||||
const consumer = new PrimaryCompletionReceiptConsumer(
|
||||
store,
|
||||
new PrimaryRunCompletionService(repository, nextId),
|
||||
{
|
||||
journal: {
|
||||
async markQuarantined(command) {
|
||||
quarantines.push(command);
|
||||
},
|
||||
async resolve() {
|
||||
return true;
|
||||
},
|
||||
},
|
||||
quarantineRetentionMs: 5_000,
|
||||
clock: { now: () => NOW + 20 },
|
||||
},
|
||||
);
|
||||
|
||||
const result = await consumer.consume(reference.attempt.id);
|
||||
assert.equal(result.status, 'quarantined');
|
||||
assert.equal(result.cleaned, true);
|
||||
assert.match(result.quarantineRef, /^\.quarantine\//);
|
||||
assert.deepEqual(quarantines, [
|
||||
{
|
||||
attemptId: reference.attempt.id,
|
||||
quarantineRef: store.quarantineReference(reference.attempt.id),
|
||||
updatedAtMs: NOW + 20,
|
||||
purgeAfterMs: NOW + 5_020,
|
||||
},
|
||||
]);
|
||||
assert.equal(await store.read(reference.attempt.id), undefined);
|
||||
assert.equal(
|
||||
(await repository.findRunById(reference.run.id)).status,
|
||||
'running',
|
||||
);
|
||||
});
|
||||
|
||||
test('maps a persisted timeout request before a late successful receipt', async (t) => {
|
||||
const repository = await setup(t);
|
||||
const reference = await runningReference(repository);
|
||||
await reference.commands.requestCancellation({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
atMs: NOW + 5,
|
||||
reason: 'timeout',
|
||||
actor: { type: 'system', id: 'runtime:timeout' },
|
||||
});
|
||||
|
||||
const completed = await new PrimaryRunCompletionService(
|
||||
repository,
|
||||
nextId,
|
||||
).complete({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
callbackSequence: 1,
|
||||
result: successfulResult(),
|
||||
source: { kind: 'receipt', token: TOKEN },
|
||||
});
|
||||
assert.equal(completed.run.status, 'timed_out');
|
||||
assert.equal(completed.attempt.status, 'timed_out');
|
||||
});
|
||||
|
||||
test('keeps a receipt when the terminal transaction crashes before commit', async (t) => {
|
||||
const repository = await setup(t);
|
||||
const reference = await runningReference(repository);
|
||||
const store = await receiptStore(t);
|
||||
await store.publish(receipt(reference));
|
||||
const failing = new PrimaryCompletionReceiptConsumer(
|
||||
store,
|
||||
new PrimaryRunCompletionService(
|
||||
new FailAfterWorkRepository(repository),
|
||||
nextId,
|
||||
),
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
failing.consume(reference.attempt.id),
|
||||
/simulated crash before commit/,
|
||||
);
|
||||
assert.ok(await store.read(reference.attempt.id));
|
||||
assert.equal(
|
||||
(await repository.findRunById(reference.run.id)).status,
|
||||
'running',
|
||||
);
|
||||
|
||||
const recovered = await new PrimaryCompletionReceiptConsumer(
|
||||
store,
|
||||
new PrimaryRunCompletionService(repository, nextId),
|
||||
).consume(reference.attempt.id);
|
||||
assert.equal(recovered.status, 'applied');
|
||||
assert.equal(recovered.cleaned, true);
|
||||
assert.equal(await store.read(reference.attempt.id), undefined);
|
||||
assert.equal(
|
||||
(await repository.listEvents(reference.run.id)).filter(
|
||||
(event) => event.type === 'run.succeeded',
|
||||
).length,
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
test('replays terminal state and cleans a receipt after cleanup crashes', async (t) => {
|
||||
const repository = await setup(t);
|
||||
const reference = await runningReference(repository);
|
||||
const fileStore = await receiptStore(t);
|
||||
const store = new FailFirstRemoveStore(fileStore);
|
||||
await store.publish(receipt(reference));
|
||||
const consumer = new PrimaryCompletionReceiptConsumer(
|
||||
store,
|
||||
new PrimaryRunCompletionService(repository, nextId),
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
consumer.consume(reference.attempt.id),
|
||||
/simulated crash before receipt cleanup/,
|
||||
);
|
||||
assert.equal(
|
||||
(await repository.findRunById(reference.run.id)).status,
|
||||
'succeeded',
|
||||
);
|
||||
assert.ok(await store.read(reference.attempt.id));
|
||||
const eventCount = (await repository.listEvents(reference.run.id)).length;
|
||||
|
||||
const replay = await consumer.consume(reference.attempt.id);
|
||||
assert.equal(replay.status, 'already_terminal');
|
||||
assert.equal(replay.cleaned, true);
|
||||
assert.equal(await store.read(reference.attempt.id), undefined);
|
||||
assert.equal(
|
||||
(await repository.listEvents(reference.run.id)).length,
|
||||
eventCount,
|
||||
);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,282 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const { Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const { runSchemaMigration } = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
runCancellationRequestMigration,
|
||||
} = require('../../back/migrations/0004-run-cancellation-request');
|
||||
const {
|
||||
runAttemptDeadlineMigration,
|
||||
} = require('../../back/migrations/0006-run-attempt-deadline');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
CompletionReceiptFileStore,
|
||||
} = require('../../back/runtime/adapters/fs/completionReceiptFileStore');
|
||||
const {
|
||||
LegacySequelizePrimaryRunRecoverySource,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/primaryRunRecoverySource');
|
||||
const {
|
||||
LegacySequelizeRunRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/runRepository');
|
||||
const {
|
||||
PrimaryCompletionReceiptConsumer,
|
||||
} = require('../../back/runtime/application/primaryCompletionReceiptConsumer');
|
||||
const {
|
||||
hashPrimaryCompletionToken,
|
||||
PrimaryRunCompletionService,
|
||||
} = require('../../back/runtime/application/primaryRunCompletionService');
|
||||
const {
|
||||
PrimaryRunStartupReconciler,
|
||||
} = require('../../back/runtime/application/primaryRunStartupReconciler');
|
||||
|
||||
const BASE_TIME = 1_750_300_000_000;
|
||||
const TOKEN = 'c'.repeat(43);
|
||||
const databases = [];
|
||||
const roots = [];
|
||||
let idSequence = 1_800;
|
||||
|
||||
function nextId() {
|
||||
idSequence += 1;
|
||||
return `019f7400-0000-7000-8000-${String(idSequence).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
async function createRuntime() {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
runCancellationRequestMigration,
|
||||
runAttemptDeadlineMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
databases.push(database);
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-startup-receipt-'));
|
||||
roots.push(root);
|
||||
const repository = new LegacySequelizeRunRepository(database);
|
||||
const store = new CompletionReceiptFileStore(root);
|
||||
const consumer = new PrimaryCompletionReceiptConsumer(
|
||||
store,
|
||||
new PrimaryRunCompletionService(repository, nextId),
|
||||
);
|
||||
return { database, repository, store, consumer };
|
||||
}
|
||||
|
||||
function aggregate() {
|
||||
const run = {
|
||||
id: nextId(),
|
||||
projectId: 'default',
|
||||
taskId: 'receipt-recovery',
|
||||
taskRevision: 'revision-1',
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
status: 'running',
|
||||
version: 0,
|
||||
eventSequence: 0,
|
||||
priority: 0,
|
||||
createdAtMs: BASE_TIME,
|
||||
startedAtMs: BASE_TIME + 1,
|
||||
};
|
||||
const attempt = {
|
||||
id: nextId(),
|
||||
runId: run.id,
|
||||
attempt: 1,
|
||||
status: 'running',
|
||||
executorType: 'local_process',
|
||||
executorHandle: `durable:${run.id}`,
|
||||
pid: 4321,
|
||||
callbackTokenHash: hashPrimaryCompletionToken(TOKEN),
|
||||
callbackSequence: 0,
|
||||
createdAtMs: BASE_TIME,
|
||||
startedAtMs: BASE_TIME + 1,
|
||||
};
|
||||
return { run, attempt };
|
||||
}
|
||||
|
||||
function receipt(run, attempt) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
runId: run.id,
|
||||
attemptId: attempt.id,
|
||||
callbackSequence: 1,
|
||||
token: TOKEN,
|
||||
startedAtMs: BASE_TIME + 1,
|
||||
finishedAtMs: BASE_TIME + 20,
|
||||
exitCode: 0,
|
||||
};
|
||||
}
|
||||
|
||||
async function seed(repository, run, attempt) {
|
||||
await repository.transaction(async (transaction) => {
|
||||
await transaction.insertRun(run);
|
||||
await transaction.insertAttempt(attempt);
|
||||
});
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(databases.splice(0).map((database) => database.close()));
|
||||
await Promise.all(
|
||||
roots
|
||||
.splice(0)
|
||||
.map((root) => fs.rm(root, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
test('startup consumes a receipt before consulting durable identity', async () => {
|
||||
const { database, repository, store, consumer } = await createRuntime();
|
||||
const { run, attempt } = aggregate();
|
||||
await seed(repository, run, attempt);
|
||||
await store.publish(receipt(run, attempt));
|
||||
let inspections = 0;
|
||||
const registrations = [];
|
||||
const reconciler = new PrimaryRunStartupReconciler(
|
||||
repository,
|
||||
new LegacySequelizePrimaryRunRecoverySource(database),
|
||||
[
|
||||
{
|
||||
executorType: 'local_process',
|
||||
async inspect() {
|
||||
inspections += 1;
|
||||
return { status: 'exited', identityPid: 4321 };
|
||||
},
|
||||
},
|
||||
],
|
||||
{
|
||||
completionReceipts: consumer,
|
||||
completionReceiptJournal: {
|
||||
async register(command) {
|
||||
registrations.push(command);
|
||||
},
|
||||
},
|
||||
createEventId: nextId,
|
||||
},
|
||||
);
|
||||
|
||||
const summary = await reconciler.reconcileBatch();
|
||||
assert.equal(summary.completedFromReceipt, 1);
|
||||
assert.equal(summary.markedLost, 0);
|
||||
assert.equal(inspections, 0);
|
||||
assert.deepEqual(registrations, [
|
||||
{
|
||||
runId: run.id,
|
||||
attemptId: attempt.id,
|
||||
registeredAtMs: attempt.createdAtMs,
|
||||
},
|
||||
]);
|
||||
assert.equal((await repository.findRunById(run.id)).status, 'succeeded');
|
||||
assert.equal(
|
||||
(await repository.findAttemptById(attempt.id)).status,
|
||||
'succeeded',
|
||||
);
|
||||
assert.equal(await store.read(attempt.id), undefined);
|
||||
});
|
||||
|
||||
test('startup rechecks a receipt after identity reports process exit', async () => {
|
||||
const { database, repository, store, consumer } = await createRuntime();
|
||||
const { run, attempt } = aggregate();
|
||||
await seed(repository, run, attempt);
|
||||
let inspections = 0;
|
||||
const reconciler = new PrimaryRunStartupReconciler(
|
||||
repository,
|
||||
new LegacySequelizePrimaryRunRecoverySource(database),
|
||||
[
|
||||
{
|
||||
executorType: 'local_process',
|
||||
async inspect() {
|
||||
inspections += 1;
|
||||
await store.publish(receipt(run, attempt));
|
||||
return { status: 'exited', identityPid: 4321 };
|
||||
},
|
||||
},
|
||||
],
|
||||
{ completionReceipts: consumer, createEventId: nextId },
|
||||
);
|
||||
|
||||
const summary = await reconciler.reconcileBatch();
|
||||
assert.equal(summary.completedFromReceipt, 1);
|
||||
assert.equal(summary.markedLost, 0);
|
||||
assert.equal(inspections, 1);
|
||||
assert.equal((await repository.findRunById(run.id)).status, 'succeeded');
|
||||
assert.equal(await store.read(attempt.id), undefined);
|
||||
});
|
||||
|
||||
test('startup waits once for a late receipt after observing process exit', async () => {
|
||||
const { database, repository, store, consumer } = await createRuntime();
|
||||
const { run, attempt } = aggregate();
|
||||
await seed(repository, run, attempt);
|
||||
const waits = [];
|
||||
const reconciler = new PrimaryRunStartupReconciler(
|
||||
repository,
|
||||
new LegacySequelizePrimaryRunRecoverySource(database),
|
||||
[
|
||||
{
|
||||
executorType: 'local_process',
|
||||
async inspect() {
|
||||
return { status: 'exited', identityPid: 4321 };
|
||||
},
|
||||
},
|
||||
],
|
||||
{
|
||||
completionReceipts: consumer,
|
||||
createEventId: nextId,
|
||||
receiptPublishGraceMs: 50,
|
||||
async wait(delayMs) {
|
||||
waits.push(delayMs);
|
||||
await store.publish(receipt(run, attempt));
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const summary = await reconciler.reconcileBatch();
|
||||
assert.deepEqual(waits, [50]);
|
||||
assert.equal(summary.publishGraceWaits, 1);
|
||||
assert.equal(summary.completedFromReceipt, 1);
|
||||
assert.equal(summary.markedLost, 0);
|
||||
assert.equal((await repository.findRunById(run.id)).status, 'succeeded');
|
||||
});
|
||||
|
||||
test('startup quarantines an unauthorized receipt and still verifies the process', async () => {
|
||||
const { database, repository, store, consumer } = await createRuntime();
|
||||
const { run, attempt } = aggregate();
|
||||
await seed(repository, run, attempt);
|
||||
await store.publish({
|
||||
...receipt(run, attempt),
|
||||
token: 'd'.repeat(43),
|
||||
});
|
||||
const reconciler = new PrimaryRunStartupReconciler(
|
||||
repository,
|
||||
new LegacySequelizePrimaryRunRecoverySource(database),
|
||||
[
|
||||
{
|
||||
executorType: 'local_process',
|
||||
async inspect() {
|
||||
return { status: 'running', identityPid: 4321 };
|
||||
},
|
||||
},
|
||||
],
|
||||
{ completionReceipts: consumer, createEventId: nextId },
|
||||
);
|
||||
|
||||
const summary = await reconciler.reconcileBatch();
|
||||
assert.equal(summary.quarantinedReceipts, 1);
|
||||
assert.equal(summary.completedFromReceipt, 0);
|
||||
assert.equal(summary.verifiedRunning, 1);
|
||||
assert.equal(summary.markedLost, 0);
|
||||
assert.equal((await repository.findRunById(run.id)).status, 'running');
|
||||
assert.equal(await store.read(attempt.id), undefined);
|
||||
});
|
||||
@@ -0,0 +1,376 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const { Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const { runSchemaMigration } = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
runCancellationRequestMigration,
|
||||
} = require('../../back/migrations/0004-run-cancellation-request');
|
||||
const {
|
||||
runAttemptDeadlineMigration,
|
||||
} = require('../../back/migrations/0006-run-attempt-deadline');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeRunRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/runRepository');
|
||||
const {
|
||||
LegacySequelizePrimaryRunRecoverySource,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/primaryRunRecoverySource');
|
||||
const {
|
||||
PrimaryRunStartupReconciler,
|
||||
} = require('../../back/runtime/application/primaryRunStartupReconciler');
|
||||
|
||||
const databases = [];
|
||||
const BASE_TIME = 1_750_100_000_000;
|
||||
let idSequence = 1200;
|
||||
|
||||
function nextId() {
|
||||
idSequence += 1;
|
||||
return `019f7100-0000-7000-8000-${String(idSequence).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
async function createRuntime() {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
const migrationModel = defineSchemaMigrationModel(database);
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel,
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
runCancellationRequestMigration,
|
||||
runAttemptDeadlineMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
databases.push(database);
|
||||
return {
|
||||
database,
|
||||
repository: new LegacySequelizeRunRepository(database),
|
||||
source: new LegacySequelizePrimaryRunRecoverySource(database),
|
||||
};
|
||||
}
|
||||
|
||||
function createRun(overrides = {}) {
|
||||
return {
|
||||
id: nextId(),
|
||||
projectId: 'default',
|
||||
taskId: 'recovery-task',
|
||||
taskRevision: 'revision-1',
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
status: 'running',
|
||||
version: 0,
|
||||
eventSequence: 0,
|
||||
priority: 0,
|
||||
createdAtMs: BASE_TIME,
|
||||
startedAtMs: BASE_TIME + 5,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createAttempt(runId, overrides = {}) {
|
||||
return {
|
||||
id: nextId(),
|
||||
runId,
|
||||
attempt: 1,
|
||||
status: 'running',
|
||||
executorType: 'local_process',
|
||||
executorHandle: `durable:${runId}`,
|
||||
pid: 4321,
|
||||
callbackSequence: 0,
|
||||
createdAtMs: BASE_TIME,
|
||||
startedAtMs: BASE_TIME + 5,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function seed(repository, run, attempts = []) {
|
||||
await repository.transaction(async (transaction) => {
|
||||
await transaction.insertRun(run);
|
||||
for (const attempt of attempts) await transaction.insertAttempt(attempt);
|
||||
});
|
||||
}
|
||||
|
||||
function inspector(inspect) {
|
||||
return {
|
||||
executorType: 'local_process',
|
||||
inspect,
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(databases.splice(0).map((database) => database.close()));
|
||||
});
|
||||
|
||||
test('scans only active runtime-owned Runs with a bounded stable cursor', async () => {
|
||||
const { repository, source } = await createRuntime();
|
||||
const first = createRun({ createdAtMs: BASE_TIME });
|
||||
const second = createRun({
|
||||
status: 'dispatching',
|
||||
createdAtMs: BASE_TIME + 1,
|
||||
startedAtMs: undefined,
|
||||
});
|
||||
const terminal = createRun({
|
||||
status: 'succeeded',
|
||||
createdAtMs: BASE_TIME + 2,
|
||||
});
|
||||
const legacy = createRun({
|
||||
executionOwner: 'legacy',
|
||||
createdAtMs: BASE_TIME + 3,
|
||||
});
|
||||
const firstAttempt = createAttempt(first.id);
|
||||
const secondAttempt = createAttempt(second.id);
|
||||
await seed(repository, first, [firstAttempt]);
|
||||
await seed(repository, second, [secondAttempt]);
|
||||
await seed(repository, terminal, [createAttempt(terminal.id)]);
|
||||
await seed(repository, legacy, [createAttempt(legacy.id)]);
|
||||
|
||||
const page1 = await source.listCandidates({ limit: 1 });
|
||||
assert.equal(page1.truncated, true);
|
||||
assert.equal(page1.unsafeAttemptOverflow, false);
|
||||
assert.deepEqual(page1.candidates, [
|
||||
{
|
||||
runId: first.id,
|
||||
attempts: [
|
||||
{
|
||||
attemptId: firstAttempt.id,
|
||||
executorType: 'local_process',
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
|
||||
const page2 = await source.listCandidates({
|
||||
limit: 1,
|
||||
cursor: page1.nextCursor,
|
||||
});
|
||||
assert.equal(page2.candidates[0].runId, second.id);
|
||||
assert.equal(page2.truncated, false);
|
||||
await assert.rejects(source.listCandidates({ limit: 65 }), RangeError);
|
||||
});
|
||||
|
||||
test('fails closed when corrupt data exceeds the bounded active-attempt budget', async () => {
|
||||
const { repository, source } = await createRuntime();
|
||||
const run = createRun();
|
||||
await seed(repository, run, [
|
||||
createAttempt(run.id, { attempt: 1 }),
|
||||
createAttempt(run.id, { attempt: 2 }),
|
||||
createAttempt(run.id, { attempt: 3 }),
|
||||
]);
|
||||
|
||||
const page = await source.listCandidates({ limit: 1 });
|
||||
assert.equal(page.unsafeAttemptOverflow, true);
|
||||
assert.deepEqual(page.candidates, []);
|
||||
assert.equal(page.nextCursor, undefined);
|
||||
});
|
||||
|
||||
test('audits a verified running process without changing its status', async () => {
|
||||
const { repository, source } = await createRuntime();
|
||||
const run = createRun();
|
||||
const attempt = createAttempt(run.id);
|
||||
await seed(repository, run, [attempt]);
|
||||
const reconciler = new PrimaryRunStartupReconciler(
|
||||
repository,
|
||||
source,
|
||||
[inspector(async () => ({ status: 'running', identityPid: 4321 }))],
|
||||
{ clock: { now: () => BASE_TIME + 10 }, createEventId: nextId },
|
||||
);
|
||||
|
||||
const summary = await reconciler.reconcileBatch();
|
||||
assert.equal(summary.verifiedRunning, 1);
|
||||
assert.equal(summary.markedLost, 0);
|
||||
const persisted = await repository.findRunById(run.id);
|
||||
assert.equal(persisted.status, 'running');
|
||||
assert.equal(persisted.version, 1);
|
||||
assert.deepEqual(
|
||||
(await repository.listEvents(run.id)).map((event) => event.type),
|
||||
['run.reconciled'],
|
||||
);
|
||||
assert.deepEqual((await repository.listEvents(run.id))[0].payload, {
|
||||
status: 'running',
|
||||
executor_type: 'local_process',
|
||||
evidence: 'durable_handle',
|
||||
version: 1,
|
||||
});
|
||||
});
|
||||
|
||||
test('recovers a dispatching Run when its durable Attempt is still running', async () => {
|
||||
const { repository, source } = await createRuntime();
|
||||
const run = createRun({ status: 'dispatching', startedAtMs: undefined });
|
||||
const attempt = createAttempt(run.id);
|
||||
await seed(repository, run, [attempt]);
|
||||
const reconciler = new PrimaryRunStartupReconciler(
|
||||
repository,
|
||||
source,
|
||||
[inspector(async () => ({ status: 'running', identityPid: 4321 }))],
|
||||
{ clock: { now: () => BASE_TIME + 10 }, createEventId: nextId },
|
||||
);
|
||||
|
||||
const summary = await reconciler.reconcileBatch();
|
||||
assert.equal(summary.recoveredRunning, 1);
|
||||
assert.equal((await repository.findRunById(run.id)).status, 'running');
|
||||
assert.deepEqual(
|
||||
(await repository.listEvents(run.id)).map((event) => event.type),
|
||||
['run.running'],
|
||||
);
|
||||
});
|
||||
|
||||
test('marks unprovable processes lost without issuing any process action', async () => {
|
||||
const { repository, source } = await createRuntime();
|
||||
const cases = [
|
||||
{
|
||||
handle: 'invalid-handle',
|
||||
inspection: { status: 'invalid' },
|
||||
code: 'RECOVERY_HANDLE_INVALID',
|
||||
},
|
||||
{
|
||||
handle: 'mismatched-handle',
|
||||
inspection: { status: 'identity_mismatch', identityPid: 4321 },
|
||||
code: 'RECOVERY_IDENTITY_MISMATCH',
|
||||
},
|
||||
{
|
||||
handle: 'unsupported-handle',
|
||||
inspection: { status: 'unsupported', identityPid: 4321 },
|
||||
code: 'RECOVERY_IDENTITY_UNSUPPORTED',
|
||||
},
|
||||
{
|
||||
handle: 'exited-handle',
|
||||
inspection: { status: 'exited', identityPid: 4321 },
|
||||
code: 'RECOVERY_PROCESS_EXITED_UNOBSERVED',
|
||||
},
|
||||
{
|
||||
handle: 'reused-pid-handle',
|
||||
inspection: { status: 'running', identityPid: 9999 },
|
||||
code: 'RECOVERY_IDENTITY_PID_MISMATCH',
|
||||
},
|
||||
];
|
||||
const inspections = new Map();
|
||||
for (const [index, item] of cases.entries()) {
|
||||
const run = createRun({ createdAtMs: BASE_TIME + index });
|
||||
const attempt = createAttempt(run.id, { executorHandle: item.handle });
|
||||
await seed(repository, run, [attempt]);
|
||||
inspections.set(item.handle, item.inspection);
|
||||
item.run = run;
|
||||
item.attempt = attempt;
|
||||
}
|
||||
const reconciler = new PrimaryRunStartupReconciler(
|
||||
repository,
|
||||
source,
|
||||
[inspector(async (handle) => inspections.get(handle))],
|
||||
{ clock: { now: () => BASE_TIME + 20 }, createEventId: nextId },
|
||||
);
|
||||
|
||||
const summary = await reconciler.reconcileBatch();
|
||||
assert.equal(summary.markedLost, cases.length);
|
||||
for (const item of cases) {
|
||||
const run = await repository.findRunById(item.run.id);
|
||||
const attempt = await repository.findAttemptById(item.attempt.id);
|
||||
assert.equal(run.status, 'lost');
|
||||
assert.equal(attempt.status, 'lost');
|
||||
assert.equal(run.errorCode, item.code);
|
||||
assert.equal(attempt.errorCode, item.code);
|
||||
}
|
||||
});
|
||||
|
||||
test('marks incomplete ownership lost before consulting the OS', async () => {
|
||||
const { repository, source } = await createRuntime();
|
||||
const run = createRun({ status: 'dispatching', startedAtMs: undefined });
|
||||
const attempt = createAttempt(run.id, {
|
||||
status: 'starting',
|
||||
executorHandle: undefined,
|
||||
pid: undefined,
|
||||
startedAtMs: undefined,
|
||||
});
|
||||
await seed(repository, run, [attempt]);
|
||||
const missingAttemptRun = createRun({
|
||||
status: 'dispatching',
|
||||
startedAtMs: undefined,
|
||||
createdAtMs: BASE_TIME + 1,
|
||||
});
|
||||
await seed(repository, missingAttemptRun);
|
||||
let inspections = 0;
|
||||
const reconciler = new PrimaryRunStartupReconciler(
|
||||
repository,
|
||||
source,
|
||||
[
|
||||
inspector(async () => {
|
||||
inspections += 1;
|
||||
return { status: 'running' };
|
||||
}),
|
||||
],
|
||||
{ clock: { now: () => BASE_TIME + 10 }, createEventId: nextId },
|
||||
);
|
||||
|
||||
const summary = await reconciler.reconcileBatch();
|
||||
assert.equal(summary.markedLost, 2);
|
||||
assert.equal(inspections, 0);
|
||||
assert.equal(
|
||||
(await repository.findRunById(run.id)).errorCode,
|
||||
'RECOVERY_ATTEMPT_INCOMPLETE',
|
||||
);
|
||||
assert.equal(
|
||||
(await repository.findRunById(missingAttemptRun.id)).errorCode,
|
||||
'RECOVERY_ATTEMPT_MISSING',
|
||||
);
|
||||
});
|
||||
|
||||
test('leaves ambiguous, unsupported, and transient probe failures untouched', async () => {
|
||||
const { repository, source } = await createRuntime();
|
||||
const ambiguous = createRun({ createdAtMs: BASE_TIME });
|
||||
await seed(repository, ambiguous, [
|
||||
createAttempt(ambiguous.id, { attempt: 1 }),
|
||||
createAttempt(ambiguous.id, { attempt: 2 }),
|
||||
]);
|
||||
const unsupported = createRun({ createdAtMs: BASE_TIME + 1 });
|
||||
await seed(repository, unsupported, [
|
||||
createAttempt(unsupported.id, {
|
||||
executorType: 'remote_worker',
|
||||
executorHandle: 'remote-handle',
|
||||
}),
|
||||
]);
|
||||
const transient = createRun({ createdAtMs: BASE_TIME + 2 });
|
||||
const transientAttempt = createAttempt(transient.id);
|
||||
await seed(repository, transient, [transientAttempt]);
|
||||
const healthy = createRun({ createdAtMs: BASE_TIME + 3 });
|
||||
const healthyAttempt = createAttempt(healthy.id);
|
||||
await seed(repository, healthy, [healthyAttempt]);
|
||||
const reconciler = new PrimaryRunStartupReconciler(
|
||||
repository,
|
||||
source,
|
||||
[
|
||||
inspector(async (handle) => {
|
||||
if (handle === transientAttempt.executorHandle) {
|
||||
throw new Error('temporary /proc failure');
|
||||
}
|
||||
return { status: 'running', identityPid: 4321 };
|
||||
}),
|
||||
],
|
||||
{ clock: { now: () => BASE_TIME + 10 }, createEventId: nextId },
|
||||
);
|
||||
|
||||
const summary = await reconciler.reconcileBatch();
|
||||
assert.equal(summary.ambiguous, 1);
|
||||
assert.equal(summary.skipped, 1);
|
||||
assert.equal(summary.failed, 1);
|
||||
assert.equal(summary.verifiedRunning, 1);
|
||||
assert.equal((await repository.findRunById(ambiguous.id)).status, 'running');
|
||||
assert.equal(
|
||||
(await repository.findRunById(unsupported.id)).status,
|
||||
'running',
|
||||
);
|
||||
assert.equal((await repository.findRunById(transient.id)).status, 'running');
|
||||
assert.equal((await repository.findRunById(healthy.id)).status, 'running');
|
||||
assert.deepEqual(
|
||||
(await repository.listEvents(healthy.id)).map((event) => event.type),
|
||||
['run.reconciled'],
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,104 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PrimaryRunStartupSupervisor,
|
||||
} = require('../../back/runtime/application/primaryRunStartupSupervisor');
|
||||
|
||||
function page(overrides = {}) {
|
||||
return {
|
||||
scanned: 1,
|
||||
verifiedRunning: 0,
|
||||
recoveredRunning: 1,
|
||||
completedFromReceipt: 0,
|
||||
quarantinedReceipts: 0,
|
||||
publishGraceWaits: 0,
|
||||
markedLost: 0,
|
||||
skipped: 0,
|
||||
ambiguous: 0,
|
||||
failed: 0,
|
||||
truncated: false,
|
||||
unsafeAttemptOverflow: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('startup supervisor aggregates bounded pages until recovery completes', async () => {
|
||||
const cursors = [];
|
||||
const responses = [
|
||||
page({
|
||||
truncated: true,
|
||||
nextCursor: { createdAtMs: 10, runId: 'run-1' },
|
||||
}),
|
||||
page({ verifiedRunning: 1, recoveredRunning: 0, markedLost: 1 }),
|
||||
];
|
||||
const supervisor = new PrimaryRunStartupSupervisor({
|
||||
async reconcileBatch(options) {
|
||||
cursors.push(options.cursor);
|
||||
return responses.shift();
|
||||
},
|
||||
});
|
||||
|
||||
const result = await supervisor.run({ pageSize: 8, maxPages: 4 });
|
||||
assert.deepEqual(cursors, [undefined, { createdAtMs: 10, runId: 'run-1' }]);
|
||||
assert.deepEqual(result, {
|
||||
pages: 2,
|
||||
scanned: 2,
|
||||
verifiedRunning: 1,
|
||||
recoveredRunning: 1,
|
||||
completedFromReceipt: 0,
|
||||
quarantinedReceipts: 0,
|
||||
publishGraceWaits: 0,
|
||||
markedLost: 1,
|
||||
skipped: 0,
|
||||
ambiguous: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
});
|
||||
});
|
||||
|
||||
test('startup supervisor fails closed for overflow, stalled cursor, and page limit', async () => {
|
||||
const overflow = new PrimaryRunStartupSupervisor({
|
||||
async reconcileBatch() {
|
||||
return page({ unsafeAttemptOverflow: true, truncated: true });
|
||||
},
|
||||
});
|
||||
assert.equal((await overflow.run()).stopReason, 'unsafe_attempt_overflow');
|
||||
|
||||
const stalled = new PrimaryRunStartupSupervisor({
|
||||
async reconcileBatch() {
|
||||
return page({ truncated: true });
|
||||
},
|
||||
});
|
||||
assert.equal((await stalled.run()).stopReason, 'cursor_stalled');
|
||||
|
||||
let sequence = 0;
|
||||
const limited = new PrimaryRunStartupSupervisor({
|
||||
async reconcileBatch() {
|
||||
sequence += 1;
|
||||
return page({
|
||||
truncated: true,
|
||||
nextCursor: { createdAtMs: sequence, runId: `run-${sequence}` },
|
||||
});
|
||||
},
|
||||
});
|
||||
const limitedResult = await limited.run({ maxPages: 2 });
|
||||
assert.equal(limitedResult.stopReason, 'page_limit');
|
||||
assert.equal(limitedResult.remaining, true);
|
||||
assert.deepEqual(limitedResult.nextCursor, {
|
||||
createdAtMs: 2,
|
||||
runId: 'run-2',
|
||||
});
|
||||
});
|
||||
|
||||
test('startup supervisor rejects unbounded settings', async () => {
|
||||
const supervisor = new PrimaryRunStartupSupervisor({
|
||||
async reconcileBatch() {
|
||||
return page();
|
||||
},
|
||||
});
|
||||
await assert.rejects(supervisor.run({ pageSize: 0 }), RangeError);
|
||||
await assert.rejects(supervisor.run({ maxPages: 65 }), RangeError);
|
||||
});
|
||||
@@ -0,0 +1,191 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PrimaryTimeoutLifecycle,
|
||||
} = require('../../back/runtime/application/primaryTimeoutLifecycle');
|
||||
|
||||
function deferred() {
|
||||
let resolve;
|
||||
let reject;
|
||||
const promise = new Promise((resolvePromise, rejectPromise) => {
|
||||
resolve = resolvePromise;
|
||||
reject = rejectPromise;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
function fakeScheduler() {
|
||||
let id = 0;
|
||||
const pending = new Map();
|
||||
const cleared = [];
|
||||
return {
|
||||
pending,
|
||||
cleared,
|
||||
scheduler: {
|
||||
setTimeout(callback, delayMs) {
|
||||
const timer = {
|
||||
id: ++id,
|
||||
delayMs,
|
||||
unrefCalls: 0,
|
||||
unref() {
|
||||
this.unrefCalls += 1;
|
||||
},
|
||||
};
|
||||
pending.set(timer.id, { timer, callback });
|
||||
return timer;
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
cleared.push(timer.id);
|
||||
pending.delete(timer.id);
|
||||
},
|
||||
},
|
||||
fireNext() {
|
||||
const next = pending.values().next().value;
|
||||
assert.ok(next, 'expected a scheduled timer');
|
||||
pending.delete(next.timer.id);
|
||||
next.callback();
|
||||
return next.timer;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function cycleSummary() {
|
||||
return {
|
||||
pages: 1,
|
||||
scanned: 1,
|
||||
accepted: 1,
|
||||
alreadyRequested: 0,
|
||||
alreadyTerminal: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
}
|
||||
|
||||
async function flush() {
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
}
|
||||
|
||||
test('is inert until started and never overlaps timeout scans', async () => {
|
||||
const timers = fakeScheduler();
|
||||
const first = deferred();
|
||||
let calls = 0;
|
||||
const lifecycle = new PrimaryTimeoutLifecycle(
|
||||
{
|
||||
async run(options) {
|
||||
calls += 1;
|
||||
assert.deepEqual(options, { pageSize: 8, maxPages: 2 });
|
||||
return first.promise;
|
||||
},
|
||||
},
|
||||
{
|
||||
intervalMs: 30_000,
|
||||
initialDelayMs: 100,
|
||||
cycle: { pageSize: 8, maxPages: 2 },
|
||||
scheduler: timers.scheduler,
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(timers.pending.size, 0);
|
||||
assert.equal(lifecycle.start(), true);
|
||||
assert.equal(lifecycle.start(), false);
|
||||
const initial = timers.fireNext();
|
||||
assert.equal(initial.delayMs, 100);
|
||||
assert.equal(initial.unrefCalls, 1);
|
||||
await flush();
|
||||
assert.equal(calls, 1);
|
||||
assert.equal(timers.pending.size, 0);
|
||||
|
||||
first.resolve(cycleSummary());
|
||||
await flush();
|
||||
const next = timers.pending.values().next().value.timer;
|
||||
assert.equal(next.delayMs, 30_000);
|
||||
assert.equal(next.unrefCalls, 1);
|
||||
assert.equal(await lifecycle.stop(), 'drained');
|
||||
assert.deepEqual(timers.cleared, [next.id]);
|
||||
});
|
||||
|
||||
test('reports errors and resumes without callback failure loops', async () => {
|
||||
const timers = fakeScheduler();
|
||||
const errors = [];
|
||||
let calls = 0;
|
||||
const lifecycle = new PrimaryTimeoutLifecycle(
|
||||
{
|
||||
async run() {
|
||||
calls += 1;
|
||||
if (calls === 1) throw new Error('database busy');
|
||||
return cycleSummary();
|
||||
},
|
||||
},
|
||||
{
|
||||
intervalMs: 5_000,
|
||||
scheduler: timers.scheduler,
|
||||
onCycle() {
|
||||
throw new Error('metrics sink failed');
|
||||
},
|
||||
onError(error) {
|
||||
errors.push(error.message);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
lifecycle.start();
|
||||
timers.fireNext();
|
||||
await flush();
|
||||
assert.deepEqual(errors, ['database busy']);
|
||||
|
||||
timers.fireNext();
|
||||
await flush();
|
||||
assert.deepEqual(errors, ['database busy', 'metrics sink failed']);
|
||||
assert.equal(await lifecycle.stop(), 'drained');
|
||||
});
|
||||
|
||||
test('bounds shutdown and refuses restart while an old scan is in flight', async () => {
|
||||
const timers = fakeScheduler();
|
||||
const running = deferred();
|
||||
const lifecycle = new PrimaryTimeoutLifecycle(
|
||||
{
|
||||
async run() {
|
||||
return running.promise;
|
||||
},
|
||||
},
|
||||
{
|
||||
intervalMs: 5_000,
|
||||
stopTimeoutMs: 5,
|
||||
scheduler: timers.scheduler,
|
||||
},
|
||||
);
|
||||
lifecycle.start();
|
||||
timers.fireNext();
|
||||
await flush();
|
||||
|
||||
assert.equal(await lifecycle.stop(), 'timed_out');
|
||||
assert.equal(timers.pending.size, 0);
|
||||
assert.equal(lifecycle.start(), false);
|
||||
running.resolve(cycleSummary());
|
||||
await flush();
|
||||
assert.equal(lifecycle.start(), true);
|
||||
assert.equal(await lifecycle.stop(), 'drained');
|
||||
});
|
||||
|
||||
test('rejects hot loops and unbounded shutdown waits', () => {
|
||||
const supervisor = {
|
||||
async run() {
|
||||
return cycleSummary();
|
||||
},
|
||||
};
|
||||
assert.throws(
|
||||
() => new PrimaryTimeoutLifecycle(supervisor, { intervalMs: 249 }),
|
||||
RangeError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
new PrimaryTimeoutLifecycle(supervisor, {
|
||||
intervalMs: 500,
|
||||
stopTimeoutMs: 60_001,
|
||||
}),
|
||||
RangeError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,213 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const { Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const { runSchemaMigration } = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
runCancellationRequestMigration,
|
||||
} = require('../../back/migrations/0004-run-cancellation-request');
|
||||
const {
|
||||
runAttemptDeadlineMigration,
|
||||
} = require('../../back/migrations/0006-run-attempt-deadline');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeRunRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/runRepository');
|
||||
const {
|
||||
LegacySequelizePrimaryTimeoutSource,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/primaryTimeoutSource');
|
||||
const {
|
||||
PrimaryTimeoutRequester,
|
||||
} = require('../../back/runtime/application/primaryTimeoutRequester');
|
||||
const {
|
||||
RunCommandService,
|
||||
} = require('../../back/runtime/application/runCommandService');
|
||||
|
||||
const NOW = 1_750_300_000_000;
|
||||
const databases = [];
|
||||
|
||||
function page(candidates, overrides = {}) {
|
||||
return { candidates, truncated: false, ...overrides };
|
||||
}
|
||||
|
||||
function candidate(index, overrides = {}) {
|
||||
return {
|
||||
runId: `run-${index}`,
|
||||
attemptId: `attempt-${index}`,
|
||||
deadlineAtMs: NOW - index,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(databases.splice(0).map((database) => database.close()));
|
||||
});
|
||||
|
||||
test('commits a durable timeout request through the real Repository boundary', async () => {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
databases.push(database);
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
runCancellationRequestMigration,
|
||||
runAttemptDeadlineMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
const repository = new LegacySequelizeRunRepository(database);
|
||||
const runId = '019f7300-0000-7000-8000-000000000101';
|
||||
const attemptId = '019f7300-0000-7000-8000-000000000102';
|
||||
await repository.transaction(async (transaction) => {
|
||||
await transaction.insertRun({
|
||||
id: runId,
|
||||
projectId: 'default',
|
||||
taskId: 'timeout-task',
|
||||
taskRevision: 'revision-1',
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
status: 'running',
|
||||
version: 3,
|
||||
eventSequence: 3,
|
||||
priority: 0,
|
||||
createdAtMs: NOW - 10_000,
|
||||
startedAtMs: NOW - 9_000,
|
||||
});
|
||||
await transaction.insertAttempt({
|
||||
id: attemptId,
|
||||
runId,
|
||||
attempt: 1,
|
||||
status: 'running',
|
||||
executorType: 'local_process',
|
||||
callbackSequence: 0,
|
||||
createdAtMs: NOW - 10_000,
|
||||
startedAtMs: NOW - 9_000,
|
||||
deadlineAtMs: NOW - 1,
|
||||
});
|
||||
});
|
||||
const requester = new PrimaryTimeoutRequester(
|
||||
new LegacySequelizePrimaryTimeoutSource(database),
|
||||
new RunCommandService(repository),
|
||||
);
|
||||
|
||||
const result = await requester.requestBatch({ nowMs: NOW });
|
||||
assert.equal(result.accepted, 1);
|
||||
assert.equal(result.failed, 0);
|
||||
const persisted = await repository.findRunById(runId);
|
||||
assert.equal(persisted.cancelRequestedAtMs, NOW);
|
||||
assert.equal(persisted.cancelReason, 'timeout');
|
||||
assert.deepEqual(
|
||||
(await repository.listEvents(runId)).map((event) => ({
|
||||
type: event.type,
|
||||
actorType: event.actorType,
|
||||
actorId: event.actorId,
|
||||
})),
|
||||
[
|
||||
{
|
||||
type: 'run.cancel_requested',
|
||||
actorType: 'system',
|
||||
actorId: 'runtime:timeout',
|
||||
},
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('persists timeout intent for each due candidate without side effects', async () => {
|
||||
const commands = [];
|
||||
const requester = new PrimaryTimeoutRequester(
|
||||
{
|
||||
async listOverdue(options) {
|
||||
assert.deepEqual(options, { nowMs: NOW, limit: 8 });
|
||||
return page([candidate(1), candidate(2), candidate(3)]);
|
||||
},
|
||||
},
|
||||
{
|
||||
async requestCancellation(command) {
|
||||
commands.push(command);
|
||||
if (command.runId === 'run-1') return { status: 'accepted' };
|
||||
if (command.runId === 'run-2') return { status: 'already_requested' };
|
||||
return { status: 'already_terminal' };
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const result = await requester.requestBatch({ nowMs: NOW, limit: 8 });
|
||||
assert.deepEqual(result, {
|
||||
scanned: 3,
|
||||
accepted: 1,
|
||||
alreadyRequested: 1,
|
||||
alreadyTerminal: 1,
|
||||
failed: 0,
|
||||
truncated: false,
|
||||
});
|
||||
assert.deepEqual(
|
||||
commands.map(({ runId, attemptId, atMs, reason, actor }) => ({
|
||||
runId,
|
||||
attemptId,
|
||||
atMs,
|
||||
reason,
|
||||
actor,
|
||||
})),
|
||||
[1, 2, 3].map((index) => ({
|
||||
runId: `run-${index}`,
|
||||
attemptId: `attempt-${index}`,
|
||||
atMs: NOW,
|
||||
reason: 'timeout',
|
||||
actor: { type: 'system', id: 'runtime:timeout' },
|
||||
})),
|
||||
);
|
||||
});
|
||||
|
||||
test('isolates candidate failures and refuses future deadline output', async () => {
|
||||
let calls = 0;
|
||||
const requester = new PrimaryTimeoutRequester(
|
||||
{
|
||||
async listOverdue() {
|
||||
return page([
|
||||
candidate(1),
|
||||
candidate(2, { deadlineAtMs: NOW + 1 }),
|
||||
candidate(3),
|
||||
]);
|
||||
},
|
||||
},
|
||||
{
|
||||
async requestCancellation() {
|
||||
calls += 1;
|
||||
if (calls === 1) throw new Error('database unavailable');
|
||||
return { status: 'accepted' };
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const result = await requester.requestBatch({ nowMs: NOW });
|
||||
assert.equal(result.scanned, 3);
|
||||
assert.equal(result.accepted, 1);
|
||||
assert.equal(result.failed, 2);
|
||||
assert.equal(calls, 2);
|
||||
});
|
||||
|
||||
test('validates its clock before querying the source', async () => {
|
||||
let queried = false;
|
||||
const requester = new PrimaryTimeoutRequester(
|
||||
{
|
||||
async listOverdue() {
|
||||
queried = true;
|
||||
return page([]);
|
||||
},
|
||||
},
|
||||
{ async requestCancellation() {} },
|
||||
{ now: () => -1 },
|
||||
);
|
||||
await assert.rejects(requester.requestBatch(), /nowMs/);
|
||||
assert.equal(queried, false);
|
||||
});
|
||||
@@ -0,0 +1,167 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const { Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
runSchemaMigration,
|
||||
} = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
runCancellationRequestMigration,
|
||||
} = require('../../back/migrations/0004-run-cancellation-request');
|
||||
const {
|
||||
runAttemptDeadlineMigration,
|
||||
} = require('../../back/migrations/0006-run-attempt-deadline');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizePrimaryTimeoutSource,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/primaryTimeoutSource');
|
||||
|
||||
const databases = [];
|
||||
const NOW = 1_750_300_000_000;
|
||||
let sequence = 2000;
|
||||
|
||||
function nextId() {
|
||||
sequence += 1;
|
||||
return `019f7300-0000-7000-8000-${String(sequence).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
async function createRuntime() {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
runCancellationRequestMigration,
|
||||
runAttemptDeadlineMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
databases.push(database);
|
||||
return {
|
||||
database,
|
||||
source: new LegacySequelizePrimaryTimeoutSource(database),
|
||||
};
|
||||
}
|
||||
|
||||
async function seed(database, overrides = {}) {
|
||||
const runId = overrides.runId ?? nextId();
|
||||
const attemptId = overrides.attemptId ?? nextId();
|
||||
await database.getQueryInterface().bulkInsert(RUN_TABLE, [
|
||||
{
|
||||
id: runId,
|
||||
project_id: 'default',
|
||||
task_id: 'timeout-task',
|
||||
task_revision: 'revision-1',
|
||||
trigger_type: 'manual',
|
||||
execution_origin: 'manual',
|
||||
execution_owner: overrides.executionOwner ?? 'runtime',
|
||||
status: overrides.runStatus ?? 'running',
|
||||
version: 3,
|
||||
event_sequence: 3,
|
||||
priority: 0,
|
||||
created_at_ms: NOW - 10_000,
|
||||
started_at_ms: NOW - 9_000,
|
||||
cancel_requested_at_ms: overrides.cancelRequestedAtMs ?? null,
|
||||
},
|
||||
]);
|
||||
await database.getQueryInterface().bulkInsert(RUN_ATTEMPT_TABLE, [
|
||||
{
|
||||
id: attemptId,
|
||||
run_id: runId,
|
||||
attempt: 1,
|
||||
status: overrides.attemptStatus ?? 'running',
|
||||
executor_type: 'local_process',
|
||||
callback_sequence: 0,
|
||||
created_at_ms: NOW - 10_000,
|
||||
started_at_ms: NOW - 9_000,
|
||||
deadline_at_ms:
|
||||
overrides.deadlineAtMs === undefined ? NOW - 1 : overrides.deadlineAtMs,
|
||||
},
|
||||
]);
|
||||
return { runId, attemptId };
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(databases.splice(0).map((database) => database.close()));
|
||||
});
|
||||
|
||||
test('lists only overdue active runtime-owned Attempts without cancellation', async () => {
|
||||
const { database, source } = await createRuntime();
|
||||
const due = await seed(database, { deadlineAtMs: NOW - 10 });
|
||||
await seed(database, { deadlineAtMs: NOW + 1 });
|
||||
await seed(database, { deadlineAtMs: null });
|
||||
await seed(database, { executionOwner: 'legacy' });
|
||||
await seed(database, { runStatus: 'succeeded' });
|
||||
await seed(database, { attemptStatus: 'claimed' });
|
||||
await seed(database, { cancelRequestedAtMs: NOW - 20 });
|
||||
|
||||
assert.deepEqual(await source.listOverdue({ nowMs: NOW }), {
|
||||
candidates: [
|
||||
{
|
||||
runId: due.runId,
|
||||
attemptId: due.attemptId,
|
||||
deadlineAtMs: NOW - 10,
|
||||
},
|
||||
],
|
||||
truncated: false,
|
||||
});
|
||||
});
|
||||
|
||||
test('paginates equal deadlines by Attempt id with a stable bounded cursor', async () => {
|
||||
const { database, source } = await createRuntime();
|
||||
const first = await seed(database, { deadlineAtMs: NOW - 5 });
|
||||
const second = await seed(database, { deadlineAtMs: NOW - 5 });
|
||||
|
||||
const page1 = await source.listOverdue({ nowMs: NOW, limit: 1 });
|
||||
assert.equal(page1.truncated, true);
|
||||
assert.deepEqual(
|
||||
page1.candidates.map((item) => item.attemptId),
|
||||
[first.attemptId],
|
||||
);
|
||||
assert.deepEqual(page1.nextCursor, {
|
||||
deadlineAtMs: NOW - 5,
|
||||
attemptId: first.attemptId,
|
||||
});
|
||||
|
||||
const page2 = await source.listOverdue({
|
||||
nowMs: NOW,
|
||||
limit: 1,
|
||||
cursor: page1.nextCursor,
|
||||
});
|
||||
assert.equal(page2.truncated, false);
|
||||
assert.deepEqual(
|
||||
page2.candidates.map((item) => item.attemptId),
|
||||
[second.attemptId],
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects unbounded timeout scans and malformed cursors', async () => {
|
||||
const { source } = await createRuntime();
|
||||
await assert.rejects(source.listOverdue({ nowMs: -1 }), /nowMs/);
|
||||
await assert.rejects(source.listOverdue({ nowMs: NOW, limit: 65 }), /limit/);
|
||||
await assert.rejects(
|
||||
source.listOverdue({
|
||||
nowMs: NOW,
|
||||
cursor: { deadlineAtMs: -1, attemptId: 'attempt' },
|
||||
}),
|
||||
/deadlineAtMs/,
|
||||
);
|
||||
await assert.rejects(
|
||||
source.listOverdue({
|
||||
nowMs: NOW,
|
||||
cursor: { deadlineAtMs: NOW, attemptId: '' },
|
||||
}),
|
||||
/attemptId/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,140 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PrimaryTimeoutSupervisor,
|
||||
} = require('../../back/runtime/application/primaryTimeoutSupervisor');
|
||||
|
||||
function summary(overrides = {}) {
|
||||
return {
|
||||
scanned: 1,
|
||||
accepted: 1,
|
||||
alreadyRequested: 0,
|
||||
alreadyTerminal: 0,
|
||||
failed: 0,
|
||||
truncated: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('aggregates bounded timeout pages with one fixed observation time', async () => {
|
||||
const calls = [];
|
||||
const responses = [
|
||||
summary({
|
||||
truncated: true,
|
||||
nextCursor: { deadlineAtMs: 10, attemptId: 'attempt-1' },
|
||||
}),
|
||||
summary({ accepted: 0, alreadyTerminal: 1 }),
|
||||
];
|
||||
const supervisor = new PrimaryTimeoutSupervisor({
|
||||
async requestBatch(options) {
|
||||
calls.push(options);
|
||||
return responses.shift();
|
||||
},
|
||||
});
|
||||
|
||||
const result = await supervisor.run({ nowMs: 100, pageSize: 8, maxPages: 4 });
|
||||
assert.deepEqual(calls, [
|
||||
{ nowMs: 100, limit: 8 },
|
||||
{
|
||||
nowMs: 100,
|
||||
cursor: { deadlineAtMs: 10, attemptId: 'attempt-1' },
|
||||
limit: 8,
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(result, {
|
||||
pages: 2,
|
||||
scanned: 2,
|
||||
accepted: 1,
|
||||
alreadyRequested: 0,
|
||||
alreadyTerminal: 1,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
});
|
||||
});
|
||||
|
||||
test('samples its clock once when the caller omits an observation time', async () => {
|
||||
const calls = [];
|
||||
let clockCalls = 0;
|
||||
const responses = [
|
||||
summary({
|
||||
truncated: true,
|
||||
nextCursor: { deadlineAtMs: 10, attemptId: 'attempt-1' },
|
||||
}),
|
||||
summary(),
|
||||
];
|
||||
const supervisor = new PrimaryTimeoutSupervisor(
|
||||
{
|
||||
async requestBatch(options) {
|
||||
calls.push(options);
|
||||
return responses.shift();
|
||||
},
|
||||
},
|
||||
{
|
||||
now() {
|
||||
clockCalls += 1;
|
||||
return 123;
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
await supervisor.run({ pageSize: 8 });
|
||||
assert.equal(clockCalls, 1);
|
||||
assert.equal(calls.length, 2);
|
||||
assert.equal(calls[0].nowMs, 123);
|
||||
assert.equal(calls[1].nowMs, 123);
|
||||
});
|
||||
|
||||
test('fails closed for a stalled cursor and bounded page exhaustion', async () => {
|
||||
const stalledCursor = { deadlineAtMs: 10, attemptId: 'attempt-1' };
|
||||
const stalled = new PrimaryTimeoutSupervisor({
|
||||
async requestBatch() {
|
||||
return summary({ truncated: true, nextCursor: stalledCursor });
|
||||
},
|
||||
});
|
||||
assert.deepEqual(await stalled.run({ cursor: stalledCursor }), {
|
||||
pages: 1,
|
||||
scanned: 1,
|
||||
accepted: 1,
|
||||
alreadyRequested: 0,
|
||||
alreadyTerminal: 0,
|
||||
failed: 0,
|
||||
stopReason: 'cursor_stalled',
|
||||
remaining: true,
|
||||
nextCursor: stalledCursor,
|
||||
});
|
||||
|
||||
let sequence = 0;
|
||||
const limited = new PrimaryTimeoutSupervisor({
|
||||
async requestBatch() {
|
||||
sequence += 1;
|
||||
return summary({
|
||||
truncated: true,
|
||||
nextCursor: {
|
||||
deadlineAtMs: sequence,
|
||||
attemptId: `attempt-${sequence}`,
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
const limitedResult = await limited.run({ maxPages: 2 });
|
||||
assert.equal(limitedResult.stopReason, 'page_limit');
|
||||
assert.equal(limitedResult.remaining, true);
|
||||
assert.deepEqual(limitedResult.nextCursor, {
|
||||
deadlineAtMs: 2,
|
||||
attemptId: 'attempt-2',
|
||||
});
|
||||
});
|
||||
|
||||
test('rejects unbounded timeout supervisor settings', async () => {
|
||||
const supervisor = new PrimaryTimeoutSupervisor({
|
||||
async requestBatch() {
|
||||
return summary();
|
||||
},
|
||||
});
|
||||
await assert.rejects(supervisor.run({ pageSize: 0 }), /pageSize/);
|
||||
await assert.rejects(supervisor.run({ maxPages: 65 }), /maxPages/);
|
||||
await assert.rejects(supervisor.run({ nowMs: -1 }), /nowMs/);
|
||||
});
|
||||
@@ -0,0 +1,606 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const { QueryTypes, Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const {
|
||||
PROJECT_ROLE_BINDING_TABLE,
|
||||
PROJECT_TABLE,
|
||||
projectPolicyMigration,
|
||||
} = require('../../back/migrations/0017-project-policy');
|
||||
const {
|
||||
PROJECT_OWNER_BOOTSTRAP_CHALLENGE_ID_INDEX,
|
||||
PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE,
|
||||
PROJECT_OWNER_BOOTSTRAP_CURRENT_INDEX,
|
||||
projectOwnerBootstrapMigration,
|
||||
} = require('../../back/migrations/0018-project-owner-bootstrap');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeProjectOwnerBootstrapRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/projectOwnerBootstrapRepository');
|
||||
const {
|
||||
LegacySequelizeProjectPolicyRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/projectPolicyRepository');
|
||||
const {
|
||||
ProjectOwnerBootstrapService,
|
||||
} = require('../../back/runtime/application/projectOwnerBootstrapService');
|
||||
const {
|
||||
AuthenticatedPrincipalExpiredError,
|
||||
normalizeAuthenticatedPrincipal,
|
||||
} = require('../../back/runtime/domain/authenticatedPrincipal');
|
||||
const {
|
||||
ProjectOwnerBootstrapChallengeActiveError,
|
||||
ProjectOwnerBootstrapClaimRejectedError,
|
||||
ProjectOwnerBootstrapProjectInactiveError,
|
||||
ProjectOwnerBootstrapProjectNotFoundError,
|
||||
ProjectOwnerBootstrapProjectNotPristineError,
|
||||
ProjectOwnerBootstrapUnauthorizedError,
|
||||
ProjectOwnerBootstrapUnavailableError,
|
||||
} = require('../../back/runtime/domain/projectOwnerBootstrap');
|
||||
|
||||
const PROJECT_ID = 'default';
|
||||
const NOW = 100_000;
|
||||
const TTL_MS = 60_000;
|
||||
|
||||
function principal(subject = { type: 'user', id: 'owner-1' }, overrides = {}) {
|
||||
return {
|
||||
subject,
|
||||
authenticationId: 'auth-1',
|
||||
authenticatedAtMs: 0,
|
||||
expiresAtMs: NOW + TTL_MS * 10,
|
||||
assurance: 'multi_factor',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function localConsolePrincipal(overrides = {}) {
|
||||
return principal(
|
||||
{ type: 'system', id: 'owner-bootstrap' },
|
||||
{ assurance: 'local_console', ...overrides },
|
||||
);
|
||||
}
|
||||
|
||||
function deterministicRandomSource(seed = 1) {
|
||||
let next = seed;
|
||||
return {
|
||||
bytes(size) {
|
||||
const value = Buffer.alloc(size, next);
|
||||
next += 1;
|
||||
return value;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function setup(t, storage = ':memory:') {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [projectPolicyMigration, projectOwnerBootstrapMigration],
|
||||
logger: { info() {} },
|
||||
});
|
||||
const repository = new LegacySequelizeProjectOwnerBootstrapRepository(
|
||||
database,
|
||||
);
|
||||
return {
|
||||
database,
|
||||
repository,
|
||||
service: new ProjectOwnerBootstrapService(
|
||||
repository,
|
||||
deterministicRandomSource(),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
test('normalizes an exact, bounded and live authenticated principal', () => {
|
||||
const value = principal();
|
||||
assert.deepEqual(normalizeAuthenticatedPrincipal(value), value);
|
||||
assert.throws(
|
||||
() => normalizeAuthenticatedPrincipal({ ...value, scopes: ['*'] }),
|
||||
/shape is invalid/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeAuthenticatedPrincipal({
|
||||
...value,
|
||||
authenticationId: 'auth with spaces',
|
||||
}),
|
||||
/authenticationId is invalid/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeAuthenticatedPrincipal({
|
||||
...value,
|
||||
expiresAtMs: value.authenticatedAtMs,
|
||||
}),
|
||||
/lifetime is invalid/,
|
||||
);
|
||||
});
|
||||
|
||||
test('migration records versioned challenges and both bounded lookup indexes', async (t) => {
|
||||
const { database, service } = await setup(t);
|
||||
const issued = await service.issue({
|
||||
projectId: PROJECT_ID,
|
||||
issuer: localConsolePrincipal(),
|
||||
nowMs: NOW,
|
||||
ttlMs: TTL_MS,
|
||||
});
|
||||
assert.match(issued.challengeId, /^[A-Za-z0-9_-]{22}$/);
|
||||
assert.match(issued.token, /^[A-Za-z0-9_-]{43}$/);
|
||||
assert.equal(issued.expiresAtMs, NOW + TTL_MS);
|
||||
|
||||
const rows = await database
|
||||
.getQueryInterface()
|
||||
.select(null, PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE);
|
||||
assert.equal(rows.length, 1);
|
||||
assert.equal(rows[0].version, 1);
|
||||
assert.equal(rows[0].challenge_id, issued.challengeId);
|
||||
assert.match(rows[0].token_digest, /^[0-9a-f]{64}$/);
|
||||
assert.notEqual(rows[0].token_digest, issued.token);
|
||||
assert.equal(JSON.stringify(rows).includes(issued.token), false);
|
||||
assert.equal(rows[0].consumed_at_ms, null);
|
||||
assert.equal(
|
||||
(
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.select(null, PROJECT_ROLE_BINDING_TABLE)
|
||||
).length,
|
||||
0,
|
||||
);
|
||||
const indexes = new Set(
|
||||
(
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.showIndex(PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE)
|
||||
).map((index) => index.name),
|
||||
);
|
||||
assert.ok(indexes.has(PROJECT_OWNER_BOOTSTRAP_CHALLENGE_ID_INDEX));
|
||||
assert.ok(indexes.has(PROJECT_OWNER_BOOTSTRAP_CURRENT_INDEX));
|
||||
});
|
||||
|
||||
test('only an active local-console bootstrap principal can issue', async (t) => {
|
||||
const { service } = await setup(t);
|
||||
for (const issuer of [
|
||||
principal(),
|
||||
localConsolePrincipal({ assurance: 'service' }),
|
||||
principal(
|
||||
{ type: 'system', id: 'different-system' },
|
||||
{ assurance: 'local_console' },
|
||||
),
|
||||
]) {
|
||||
await assert.rejects(
|
||||
service.issue({
|
||||
projectId: PROJECT_ID,
|
||||
issuer,
|
||||
nowMs: NOW,
|
||||
ttlMs: TTL_MS,
|
||||
}),
|
||||
ProjectOwnerBootstrapUnauthorizedError,
|
||||
);
|
||||
}
|
||||
await assert.rejects(
|
||||
service.issue({
|
||||
projectId: PROJECT_ID,
|
||||
issuer: localConsolePrincipal({ expiresAtMs: NOW }),
|
||||
nowMs: NOW,
|
||||
ttlMs: TTL_MS,
|
||||
}),
|
||||
AuthenticatedPrincipalExpiredError,
|
||||
);
|
||||
await assert.rejects(
|
||||
service.issue({
|
||||
projectId: PROJECT_ID,
|
||||
issuer: localConsolePrincipal(),
|
||||
nowMs: NOW,
|
||||
ttlMs: TTL_MS,
|
||||
subject: { type: 'user', id: 'injected-owner' },
|
||||
}),
|
||||
/request shape is invalid/,
|
||||
);
|
||||
});
|
||||
|
||||
test('does not replace a live challenge and versions a replacement only after expiry', async (t) => {
|
||||
const { database, service } = await setup(t);
|
||||
const request = {
|
||||
projectId: PROJECT_ID,
|
||||
issuer: localConsolePrincipal(),
|
||||
nowMs: NOW,
|
||||
ttlMs: TTL_MS,
|
||||
};
|
||||
const first = await service.issue(request);
|
||||
await assert.rejects(
|
||||
service.issue({ ...request, nowMs: NOW + 1 }),
|
||||
ProjectOwnerBootstrapChallengeActiveError,
|
||||
);
|
||||
const second = await service.issue({
|
||||
...request,
|
||||
nowMs: NOW + TTL_MS,
|
||||
});
|
||||
assert.notEqual(second.challengeId, first.challengeId);
|
||||
const rows = await database.query(
|
||||
`SELECT version, challenge_id FROM "${PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE}" ORDER BY version`,
|
||||
{ type: QueryTypes.SELECT },
|
||||
);
|
||||
assert.deepEqual(rows, [
|
||||
{ version: 1, challenge_id: first.challengeId },
|
||||
{ version: 2, challenge_id: second.challengeId },
|
||||
]);
|
||||
await assert.rejects(
|
||||
service.claim({
|
||||
projectId: PROJECT_ID,
|
||||
challengeId: first.challengeId,
|
||||
token: first.token,
|
||||
principal: principal(),
|
||||
nowMs: NOW + TTL_MS + 1,
|
||||
}),
|
||||
ProjectOwnerBootstrapClaimRejectedError,
|
||||
);
|
||||
});
|
||||
|
||||
test('claims one owner atomically and permits only exact idempotent replay', async (t) => {
|
||||
const { database, service } = await setup(t);
|
||||
const issued = await service.issue({
|
||||
projectId: PROJECT_ID,
|
||||
issuer: localConsolePrincipal(),
|
||||
nowMs: NOW,
|
||||
ttlMs: TTL_MS,
|
||||
});
|
||||
const request = {
|
||||
projectId: PROJECT_ID,
|
||||
challengeId: issued.challengeId,
|
||||
token: issued.token,
|
||||
principal: principal(),
|
||||
nowMs: NOW + 1,
|
||||
};
|
||||
const claimed = await service.claim(request);
|
||||
assert.equal(claimed.status, 'claimed');
|
||||
assert.deepEqual(claimed.binding, {
|
||||
projectId: PROJECT_ID,
|
||||
subject: { type: 'user', id: 'owner-1' },
|
||||
version: 1,
|
||||
state: 'active',
|
||||
role: 'owner',
|
||||
mutationId: `owner-bootstrap:${issued.challengeId}`,
|
||||
changedBy: { type: 'system', id: 'owner-bootstrap' },
|
||||
createdAtMs: NOW + 1,
|
||||
});
|
||||
const replay = await service.claim({ ...request, nowMs: NOW + 2 });
|
||||
assert.equal(replay.status, 'existing');
|
||||
assert.deepEqual(replay.binding, claimed.binding);
|
||||
|
||||
const policyRepository = new LegacySequelizeProjectPolicyRepository(database);
|
||||
assert.deepEqual(
|
||||
(await policyRepository.resolve(PROJECT_ID, principal().subject)).binding,
|
||||
claimed.binding,
|
||||
);
|
||||
const challenge = (
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.select(null, PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE)
|
||||
)[0];
|
||||
assert.equal(Number(challenge.consumed_at_ms), NOW + 1);
|
||||
assert.equal(challenge.claimed_subject_type, 'user');
|
||||
assert.equal(challenge.claimed_subject_id, 'owner-1');
|
||||
|
||||
await assert.rejects(
|
||||
service.claim({
|
||||
...request,
|
||||
principal: principal({ type: 'user', id: 'owner-2' }),
|
||||
nowMs: NOW + 2,
|
||||
}),
|
||||
ProjectOwnerBootstrapClaimRejectedError,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects expired, malformed, mismatched and non-user claims without leaking the token', async (t) => {
|
||||
const { service } = await setup(t);
|
||||
const issued = await service.issue({
|
||||
projectId: PROJECT_ID,
|
||||
issuer: localConsolePrincipal(),
|
||||
nowMs: NOW,
|
||||
ttlMs: TTL_MS,
|
||||
});
|
||||
await assert.rejects(
|
||||
service.claim({
|
||||
projectId: PROJECT_ID,
|
||||
challengeId: issued.challengeId,
|
||||
token: Buffer.alloc(32, 99).toString('base64url'),
|
||||
principal: principal(),
|
||||
nowMs: NOW + 1,
|
||||
}),
|
||||
(error) => {
|
||||
assert.ok(error instanceof ProjectOwnerBootstrapClaimRejectedError);
|
||||
assert.equal(error.message.includes(issued.token), false);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
await assert.rejects(
|
||||
service.claim({
|
||||
projectId: PROJECT_ID,
|
||||
challengeId: issued.challengeId,
|
||||
token: issued.token,
|
||||
principal: principal(),
|
||||
nowMs: NOW + TTL_MS,
|
||||
}),
|
||||
ProjectOwnerBootstrapClaimRejectedError,
|
||||
);
|
||||
await assert.rejects(
|
||||
service.claim({
|
||||
projectId: PROJECT_ID,
|
||||
challengeId: issued.challengeId,
|
||||
token: issued.token,
|
||||
principal: principal({ type: 'api_app', id: 'app-1' }),
|
||||
nowMs: NOW + 1,
|
||||
}),
|
||||
ProjectOwnerBootstrapUnauthorizedError,
|
||||
);
|
||||
await assert.rejects(
|
||||
service.claim({
|
||||
projectId: PROJECT_ID,
|
||||
challengeId: issued.challengeId,
|
||||
token: `${issued.token}x`,
|
||||
principal: principal(),
|
||||
nowMs: NOW + 1,
|
||||
}),
|
||||
(error) => {
|
||||
assert.equal(error.message.includes(issued.token), false);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
await assert.rejects(
|
||||
service.claim({
|
||||
projectId: PROJECT_ID,
|
||||
challengeId: issued.challengeId,
|
||||
token: issued.token,
|
||||
principal: principal(),
|
||||
nowMs: NOW + 1,
|
||||
subject: { type: 'user', id: 'injected-owner' },
|
||||
}),
|
||||
/request shape is invalid/,
|
||||
);
|
||||
});
|
||||
|
||||
test('refuses bootstrap whenever the Project already has any role binding', async (t) => {
|
||||
const first = await setup(t);
|
||||
await first.database.query(
|
||||
`INSERT INTO "${PROJECT_ROLE_BINDING_TABLE}"
|
||||
(project_id, subject_type, subject_id, version, state, role,
|
||||
mutation_id, changed_by_type, changed_by_id, created_at_ms)
|
||||
VALUES
|
||||
('default', 'user', 'existing', 1, 'active', 'viewer',
|
||||
'existing-binding', 'system', 'owner-bootstrap', 1)`,
|
||||
);
|
||||
await assert.rejects(
|
||||
first.service.issue({
|
||||
projectId: PROJECT_ID,
|
||||
issuer: localConsolePrincipal(),
|
||||
nowMs: NOW,
|
||||
ttlMs: TTL_MS,
|
||||
}),
|
||||
ProjectOwnerBootstrapProjectNotPristineError,
|
||||
);
|
||||
|
||||
const second = await setup(t);
|
||||
const issued = await second.service.issue({
|
||||
projectId: PROJECT_ID,
|
||||
issuer: localConsolePrincipal(),
|
||||
nowMs: NOW,
|
||||
ttlMs: TTL_MS,
|
||||
});
|
||||
await second.database.query(
|
||||
`INSERT INTO "${PROJECT_ROLE_BINDING_TABLE}"
|
||||
(project_id, subject_type, subject_id, version, state, role,
|
||||
mutation_id, changed_by_type, changed_by_id, created_at_ms)
|
||||
VALUES
|
||||
('default', 'user', 'existing', 1, 'active', 'viewer',
|
||||
'existing-binding', 'system', 'owner-bootstrap', 1)`,
|
||||
);
|
||||
await assert.rejects(
|
||||
second.service.claim({
|
||||
projectId: PROJECT_ID,
|
||||
challengeId: issued.challengeId,
|
||||
token: issued.token,
|
||||
principal: principal(),
|
||||
nowMs: NOW + 1,
|
||||
}),
|
||||
ProjectOwnerBootstrapProjectNotPristineError,
|
||||
);
|
||||
});
|
||||
|
||||
test('fails closed for missing and archived Projects', async (t) => {
|
||||
const missing = await setup(t);
|
||||
await assert.rejects(
|
||||
missing.service.issue({
|
||||
projectId: 'missing',
|
||||
issuer: localConsolePrincipal(),
|
||||
nowMs: NOW,
|
||||
ttlMs: TTL_MS,
|
||||
}),
|
||||
ProjectOwnerBootstrapProjectNotFoundError,
|
||||
);
|
||||
|
||||
const archived = await setup(t);
|
||||
const issued = await archived.service.issue({
|
||||
projectId: PROJECT_ID,
|
||||
issuer: localConsolePrincipal(),
|
||||
nowMs: NOW,
|
||||
ttlMs: TTL_MS,
|
||||
});
|
||||
await archived.database.query(
|
||||
`UPDATE "${PROJECT_TABLE}" SET status = 'archived' WHERE id = 'default'`,
|
||||
);
|
||||
await assert.rejects(
|
||||
archived.service.claim({
|
||||
projectId: PROJECT_ID,
|
||||
challengeId: issued.challengeId,
|
||||
token: issued.token,
|
||||
principal: principal(),
|
||||
nowMs: NOW + 1,
|
||||
}),
|
||||
ProjectOwnerBootstrapProjectInactiveError,
|
||||
);
|
||||
});
|
||||
|
||||
test('rolls back challenge consumption when owner binding insertion fails', async (t) => {
|
||||
const { database, service } = await setup(t);
|
||||
const issued = await service.issue({
|
||||
projectId: PROJECT_ID,
|
||||
issuer: localConsolePrincipal(),
|
||||
nowMs: NOW,
|
||||
ttlMs: TTL_MS,
|
||||
});
|
||||
await database.query(
|
||||
`CREATE TRIGGER reject_bootstrap_owner
|
||||
BEFORE INSERT ON "${PROJECT_ROLE_BINDING_TABLE}"
|
||||
WHEN NEW.mutation_id LIKE 'owner-bootstrap:%'
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'forced bootstrap binding failure');
|
||||
END`,
|
||||
);
|
||||
const request = {
|
||||
projectId: PROJECT_ID,
|
||||
challengeId: issued.challengeId,
|
||||
token: issued.token,
|
||||
principal: principal(),
|
||||
nowMs: NOW + 1,
|
||||
};
|
||||
await assert.rejects(
|
||||
service.claim(request),
|
||||
ProjectOwnerBootstrapUnavailableError,
|
||||
);
|
||||
let challenge = (
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.select(null, PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE)
|
||||
)[0];
|
||||
assert.equal(challenge.consumed_at_ms, null);
|
||||
assert.equal(
|
||||
(
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.select(null, PROJECT_ROLE_BINDING_TABLE)
|
||||
).length,
|
||||
0,
|
||||
);
|
||||
|
||||
await database.query('DROP TRIGGER reject_bootstrap_owner');
|
||||
assert.equal((await service.claim(request)).status, 'claimed');
|
||||
challenge = (
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.select(null, PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE)
|
||||
)[0];
|
||||
assert.equal(Number(challenge.consumed_at_ms), NOW + 1);
|
||||
});
|
||||
|
||||
test('rolls back when SQLite silently ignores challenge consumption', async (t) => {
|
||||
const { database, service } = await setup(t);
|
||||
const issued = await service.issue({
|
||||
projectId: PROJECT_ID,
|
||||
issuer: localConsolePrincipal(),
|
||||
nowMs: NOW,
|
||||
ttlMs: TTL_MS,
|
||||
});
|
||||
await database.query(
|
||||
`CREATE TRIGGER ignore_bootstrap_consumption
|
||||
BEFORE UPDATE ON "${PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE}"
|
||||
WHEN NEW.consumed_at_ms IS NOT NULL
|
||||
BEGIN
|
||||
SELECT RAISE(IGNORE);
|
||||
END`,
|
||||
);
|
||||
await assert.rejects(
|
||||
service.claim({
|
||||
projectId: PROJECT_ID,
|
||||
challengeId: issued.challengeId,
|
||||
token: issued.token,
|
||||
principal: principal(),
|
||||
nowMs: NOW + 1,
|
||||
}),
|
||||
ProjectOwnerBootstrapUnavailableError,
|
||||
);
|
||||
const challenge = (
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.select(null, PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE)
|
||||
)[0];
|
||||
assert.equal(challenge.consumed_at_ms, null);
|
||||
assert.equal(
|
||||
(
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.select(null, PROJECT_ROLE_BINDING_TABLE)
|
||||
).length,
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
test('serializes claims from separate SQLite connections so exactly one owner wins', async (t) => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-bootstrap-db-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
const storage = path.join(root, 'database.sqlite');
|
||||
const first = await setup(t, storage);
|
||||
const issued = await first.service.issue({
|
||||
projectId: PROJECT_ID,
|
||||
issuer: localConsolePrincipal(),
|
||||
nowMs: NOW,
|
||||
ttlMs: TTL_MS,
|
||||
});
|
||||
const secondDatabase = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => secondDatabase.close());
|
||||
const secondService = new ProjectOwnerBootstrapService(
|
||||
new LegacySequelizeProjectOwnerBootstrapRepository(secondDatabase),
|
||||
deterministicRandomSource(50),
|
||||
);
|
||||
const claim = (service, ownerId) =>
|
||||
service.claim({
|
||||
projectId: PROJECT_ID,
|
||||
challengeId: issued.challengeId,
|
||||
token: issued.token,
|
||||
principal: principal({ type: 'user', id: ownerId }),
|
||||
nowMs: NOW + 1,
|
||||
});
|
||||
const results = await Promise.allSettled([
|
||||
claim(first.service, 'owner-a'),
|
||||
claim(secondService, 'owner-b'),
|
||||
]);
|
||||
assert.equal(
|
||||
results.filter((result) => result.status === 'fulfilled').length,
|
||||
1,
|
||||
);
|
||||
const rejected = results.find((result) => result.status === 'rejected');
|
||||
assert.ok(rejected.reason instanceof ProjectOwnerBootstrapClaimRejectedError);
|
||||
const bindings = await first.database
|
||||
.getQueryInterface()
|
||||
.select(null, PROJECT_ROLE_BINDING_TABLE);
|
||||
assert.equal(bindings.length, 1);
|
||||
assert.equal(bindings[0].role, 'owner');
|
||||
assert.ok(['owner-a', 'owner-b'].includes(bindings[0].subject_id));
|
||||
});
|
||||
|
||||
test('rejects non-SQLite bootstrap repositories', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
new LegacySequelizeProjectOwnerBootstrapRepository({
|
||||
getDialect() {
|
||||
return 'postgres';
|
||||
},
|
||||
}),
|
||||
/SQLite-only/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,599 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const { Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const {
|
||||
PROJECT_ROLE_BINDING_CURRENT_INDEX,
|
||||
PROJECT_ROLE_BINDING_MUTATION_INDEX,
|
||||
PROJECT_ROLE_BINDING_SUBJECT_INDEX,
|
||||
PROJECT_ROLE_BINDING_TABLE,
|
||||
PROJECT_SLUG_INDEX,
|
||||
PROJECT_TABLE,
|
||||
projectPolicyMigration,
|
||||
} = require('../../back/migrations/0017-project-policy');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeProjectPolicyRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/projectPolicyRepository');
|
||||
const {
|
||||
ProjectPolicyArtifactReadAuthorizer,
|
||||
} = require('../../back/runtime/adapters/policy/projectPolicyArtifactReadAuthorizer');
|
||||
const {
|
||||
ProjectPolicyEngine,
|
||||
} = require('../../back/runtime/application/projectPolicyEngine');
|
||||
const {
|
||||
LocalArtifactReadService,
|
||||
} = require('../../back/runtime/application/localArtifactReadService');
|
||||
const {
|
||||
ProjectPolicyUnavailableError,
|
||||
ProjectRoleBindingMutationConflictError,
|
||||
ProjectRoleBindingVersionConflictError,
|
||||
normalizeProjectPermission,
|
||||
normalizeProjectRoleBindingRecord,
|
||||
} = require('../../back/runtime/domain/projectPolicy');
|
||||
|
||||
const PROJECT_ID = 'default';
|
||||
const CHANGED_BY = Object.freeze({ type: 'user', id: 'local-owner' });
|
||||
|
||||
async function setup(t, storage = ':memory:') {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [projectPolicyMigration],
|
||||
logger: { info() {} },
|
||||
});
|
||||
return {
|
||||
database,
|
||||
repository: new LegacySequelizeProjectPolicyRepository(database),
|
||||
};
|
||||
}
|
||||
|
||||
function binding({
|
||||
subject = { type: 'user', id: 'user-1' },
|
||||
version = 1,
|
||||
role = 'viewer',
|
||||
state = 'active',
|
||||
mutationId = 'mutation-1',
|
||||
createdAtMs = version,
|
||||
} = {}) {
|
||||
return {
|
||||
projectId: PROJECT_ID,
|
||||
subject,
|
||||
version,
|
||||
state,
|
||||
...(state === 'active' ? { role } : {}),
|
||||
mutationId,
|
||||
changedBy: CHANGED_BY,
|
||||
createdAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
async function append(repository, value, expectedCurrentVersion) {
|
||||
return repository.append({
|
||||
expectedCurrentVersion,
|
||||
binding: value,
|
||||
});
|
||||
}
|
||||
|
||||
test('migration creates an ownerless default Project and bounded lookup indexes', async (t) => {
|
||||
const { database, repository } = await setup(t);
|
||||
const projects = await database
|
||||
.getQueryInterface()
|
||||
.select(null, PROJECT_TABLE);
|
||||
assert.deepEqual(projects, [
|
||||
{
|
||||
id: 'default',
|
||||
name: 'Default',
|
||||
slug: 'default',
|
||||
status: 'active',
|
||||
version: 1,
|
||||
created_at_ms: 0,
|
||||
updated_at_ms: 0,
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(
|
||||
await repository.resolve(PROJECT_ID, { type: 'user', id: 'local-owner' }),
|
||||
{
|
||||
project: {
|
||||
id: 'default',
|
||||
name: 'Default',
|
||||
slug: 'default',
|
||||
status: 'active',
|
||||
version: 1,
|
||||
createdAtMs: 0,
|
||||
updatedAtMs: 0,
|
||||
},
|
||||
},
|
||||
);
|
||||
const projectIndexes = new Set(
|
||||
(await database.getQueryInterface().showIndex(PROJECT_TABLE)).map(
|
||||
(index) => index.name,
|
||||
),
|
||||
);
|
||||
const bindingIndexes = new Set(
|
||||
(
|
||||
await database.getQueryInterface().showIndex(PROJECT_ROLE_BINDING_TABLE)
|
||||
).map((index) => index.name),
|
||||
);
|
||||
assert.ok(projectIndexes.has(PROJECT_SLUG_INDEX));
|
||||
assert.ok(bindingIndexes.has(PROJECT_ROLE_BINDING_CURRENT_INDEX));
|
||||
assert.ok(bindingIndexes.has(PROJECT_ROLE_BINDING_MUTATION_INDEX));
|
||||
assert.ok(bindingIndexes.has(PROJECT_ROLE_BINDING_SUBJECT_INDEX));
|
||||
});
|
||||
|
||||
test('normalizes only declared permissions and exact tool identities', () => {
|
||||
assert.equal(normalizeProjectPermission('artifact.read'), 'artifact.read');
|
||||
assert.equal(
|
||||
normalizeProjectPermission('tool.call:github.issue.read'),
|
||||
'tool.call:github.issue.read',
|
||||
);
|
||||
for (const value of [
|
||||
'logs',
|
||||
'artifact.write',
|
||||
'tool.call:*',
|
||||
'tool.call:',
|
||||
'tool.call:bad value',
|
||||
]) {
|
||||
assert.throws(
|
||||
() => normalizeProjectPermission(value),
|
||||
/permission is invalid/,
|
||||
);
|
||||
}
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeProjectRoleBindingRecord({
|
||||
...binding({ state: 'revoked' }),
|
||||
role: 'viewer',
|
||||
}),
|
||||
/shape is invalid/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeProjectRoleBindingRecord({
|
||||
...binding(),
|
||||
permissions: ['artifact.read'],
|
||||
}),
|
||||
/shape is invalid/,
|
||||
);
|
||||
});
|
||||
|
||||
test('appends immutable role versions, replays mutations and resolves latest state', async (t) => {
|
||||
const { repository } = await setup(t);
|
||||
const first = binding();
|
||||
assert.deepEqual(await append(repository, first, 0), {
|
||||
status: 'inserted',
|
||||
binding: first,
|
||||
});
|
||||
assert.deepEqual(await append(repository, first, 0), {
|
||||
status: 'existing',
|
||||
binding: first,
|
||||
});
|
||||
await assert.rejects(
|
||||
append(repository, { ...first, role: 'operator' }, 0),
|
||||
ProjectRoleBindingMutationConflictError,
|
||||
);
|
||||
await assert.rejects(
|
||||
append(repository, binding({ mutationId: 'stale-mutation' }), 0),
|
||||
ProjectRoleBindingVersionConflictError,
|
||||
);
|
||||
|
||||
const second = binding({
|
||||
version: 2,
|
||||
role: 'operator',
|
||||
mutationId: 'mutation-2',
|
||||
});
|
||||
assert.equal((await append(repository, second, 1)).status, 'inserted');
|
||||
assert.deepEqual(
|
||||
(await repository.resolve(PROJECT_ID, first.subject)).binding,
|
||||
second,
|
||||
);
|
||||
|
||||
const revoked = binding({
|
||||
version: 3,
|
||||
state: 'revoked',
|
||||
mutationId: 'mutation-3',
|
||||
});
|
||||
assert.equal((await append(repository, revoked, 2)).status, 'inserted');
|
||||
assert.deepEqual(
|
||||
(await repository.resolve(PROJECT_ID, first.subject)).binding,
|
||||
revoked,
|
||||
);
|
||||
});
|
||||
|
||||
test('serializes concurrent first assignments so only one current version wins', async (t) => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-policy-db-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
const storage = path.join(root, 'database.sqlite');
|
||||
const { repository } = await setup(t, storage);
|
||||
const secondDatabase = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => secondDatabase.close());
|
||||
const secondRepository = new LegacySequelizeProjectPolicyRepository(
|
||||
secondDatabase,
|
||||
);
|
||||
const subject = { type: 'api_app', id: 'app-1' };
|
||||
const results = await Promise.allSettled([
|
||||
append(
|
||||
secondRepository,
|
||||
binding({ subject, role: 'viewer', mutationId: 'concurrent-a' }),
|
||||
0,
|
||||
),
|
||||
append(
|
||||
repository,
|
||||
binding({ subject, role: 'operator', mutationId: 'concurrent-b' }),
|
||||
0,
|
||||
),
|
||||
]);
|
||||
assert.equal(
|
||||
results.filter((result) => result.status === 'fulfilled').length,
|
||||
1,
|
||||
);
|
||||
const rejected = results.find((result) => result.status === 'rejected');
|
||||
assert.ok(rejected.reason instanceof ProjectRoleBindingVersionConflictError);
|
||||
const snapshot = await repository.resolve(PROJECT_ID, subject);
|
||||
assert.equal(snapshot.binding.version, 1);
|
||||
assert.ok(['viewer', 'operator'].includes(snapshot.binding.role));
|
||||
});
|
||||
|
||||
test('defaults to deny and applies viewer, operator, archive and revocation rules', async (t) => {
|
||||
const { database, repository } = await setup(t);
|
||||
const engine = new ProjectPolicyEngine(repository);
|
||||
const viewer = { type: 'user', id: 'viewer-1' };
|
||||
assert.deepEqual(
|
||||
await engine.decide({
|
||||
subject: viewer,
|
||||
projectId: 'missing',
|
||||
permission: 'artifact.read',
|
||||
}),
|
||||
{ effect: 'deny', reasons: ['project_not_found'] },
|
||||
);
|
||||
assert.deepEqual(
|
||||
await engine.decide({
|
||||
subject: viewer,
|
||||
projectId: PROJECT_ID,
|
||||
permission: 'artifact.read',
|
||||
}),
|
||||
{ effect: 'deny', reasons: ['subject_unbound'] },
|
||||
);
|
||||
await append(
|
||||
repository,
|
||||
binding({ subject: viewer, role: 'viewer', mutationId: 'viewer-bind' }),
|
||||
0,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await engine.decide({
|
||||
subject: viewer,
|
||||
projectId: PROJECT_ID,
|
||||
permission: 'artifact.read',
|
||||
})
|
||||
).effect,
|
||||
'allow',
|
||||
);
|
||||
assert.deepEqual(
|
||||
await engine.decide({
|
||||
subject: viewer,
|
||||
projectId: PROJECT_ID,
|
||||
permission: 'run.start',
|
||||
}),
|
||||
{ effect: 'deny', reasons: ['permission_missing'] },
|
||||
);
|
||||
|
||||
const operator = binding({
|
||||
subject: viewer,
|
||||
version: 2,
|
||||
role: 'operator',
|
||||
mutationId: 'viewer-promote',
|
||||
});
|
||||
await append(repository, operator, 1);
|
||||
assert.equal(
|
||||
(
|
||||
await engine.decide({
|
||||
subject: viewer,
|
||||
projectId: PROJECT_ID,
|
||||
permission: 'run.start',
|
||||
})
|
||||
).effect,
|
||||
'allow',
|
||||
);
|
||||
await database
|
||||
.getQueryInterface()
|
||||
.bulkUpdate(
|
||||
PROJECT_TABLE,
|
||||
{ status: 'archived', version: 2, updated_at_ms: 2 },
|
||||
{ id: PROJECT_ID },
|
||||
);
|
||||
assert.deepEqual(
|
||||
await engine.decide({
|
||||
subject: viewer,
|
||||
projectId: PROJECT_ID,
|
||||
permission: 'run.start',
|
||||
}),
|
||||
{ effect: 'deny', reasons: ['project_archived'] },
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await engine.decide({
|
||||
subject: viewer,
|
||||
projectId: PROJECT_ID,
|
||||
permission: 'artifact.read',
|
||||
})
|
||||
).effect,
|
||||
'allow',
|
||||
);
|
||||
|
||||
await append(
|
||||
repository,
|
||||
binding({
|
||||
subject: viewer,
|
||||
version: 3,
|
||||
state: 'revoked',
|
||||
mutationId: 'viewer-revoke',
|
||||
}),
|
||||
2,
|
||||
);
|
||||
assert.deepEqual(
|
||||
await engine.decide({
|
||||
subject: viewer,
|
||||
projectId: PROJECT_ID,
|
||||
permission: 'artifact.read',
|
||||
}),
|
||||
{ effect: 'deny', reasons: ['subject_unbound'] },
|
||||
);
|
||||
});
|
||||
|
||||
test('requires approval for Agent mutations while allowing bound reads', async (t) => {
|
||||
const { repository } = await setup(t);
|
||||
const engine = new ProjectPolicyEngine(repository);
|
||||
const agent = { type: 'agent', id: 'agent-1' };
|
||||
await append(
|
||||
repository,
|
||||
binding({ subject: agent, role: 'operator', mutationId: 'agent-bind' }),
|
||||
0,
|
||||
);
|
||||
assert.deepEqual(
|
||||
await engine.decide({
|
||||
subject: agent,
|
||||
projectId: PROJECT_ID,
|
||||
permission: 'run.start',
|
||||
}),
|
||||
{
|
||||
effect: 'require_approval',
|
||||
reasons: ['agent_action_requires_approval'],
|
||||
},
|
||||
);
|
||||
assert.deepEqual(
|
||||
await engine.decide({
|
||||
subject: agent,
|
||||
projectId: PROJECT_ID,
|
||||
permission: 'tool.call:github.issue.read',
|
||||
}),
|
||||
{
|
||||
effect: 'require_approval',
|
||||
reasons: ['agent_action_requires_approval'],
|
||||
},
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await engine.decide({
|
||||
subject: agent,
|
||||
projectId: PROJECT_ID,
|
||||
permission: 'artifact.read',
|
||||
})
|
||||
).effect,
|
||||
'allow',
|
||||
);
|
||||
});
|
||||
|
||||
test('enforces the owner, admin, operator and viewer permission matrix', async (t) => {
|
||||
const { repository } = await setup(t);
|
||||
const engine = new ProjectPolicyEngine(repository);
|
||||
const cases = [
|
||||
{
|
||||
role: 'owner',
|
||||
allow: [
|
||||
'project.manage',
|
||||
'policy.manage',
|
||||
'task.delete',
|
||||
'approval.recover',
|
||||
],
|
||||
deny: [],
|
||||
},
|
||||
{
|
||||
role: 'admin',
|
||||
allow: [
|
||||
'policy.manage',
|
||||
'task.delete',
|
||||
'secret.manage',
|
||||
'approval.recover',
|
||||
],
|
||||
deny: ['project.manage'],
|
||||
},
|
||||
{
|
||||
role: 'operator',
|
||||
allow: ['task.update', 'run.start', 'secret.use'],
|
||||
deny: [
|
||||
'task.delete',
|
||||
'secret.manage',
|
||||
'policy.manage',
|
||||
'approval.recover',
|
||||
],
|
||||
},
|
||||
{
|
||||
role: 'viewer',
|
||||
allow: ['project.read', 'task.read', 'run.read', 'artifact.read'],
|
||||
deny: ['task.update', 'run.start', 'secret.use', 'approval.recover'],
|
||||
},
|
||||
];
|
||||
for (const [index, item] of cases.entries()) {
|
||||
const subject = { type: 'user', id: `${item.role}-user` };
|
||||
await append(
|
||||
repository,
|
||||
binding({
|
||||
subject,
|
||||
role: item.role,
|
||||
mutationId: `matrix-${index}`,
|
||||
}),
|
||||
0,
|
||||
);
|
||||
for (const permission of item.allow) {
|
||||
assert.equal(
|
||||
(await engine.decide({ subject, projectId: PROJECT_ID, permission }))
|
||||
.effect,
|
||||
'allow',
|
||||
`${item.role} should allow ${permission}`,
|
||||
);
|
||||
}
|
||||
for (const permission of item.deny) {
|
||||
assert.equal(
|
||||
(await engine.decide({ subject, projectId: PROJECT_ID, permission }))
|
||||
.effect,
|
||||
'deny',
|
||||
`${item.role} should deny ${permission}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('Artifact authorizer delegates only the bound subject, Project and permission', async () => {
|
||||
const calls = [];
|
||||
const authorizer = new ProjectPolicyArtifactReadAuthorizer({
|
||||
async decide(value) {
|
||||
calls.push(value);
|
||||
return { effect: 'require_approval', reasons: ['test'] };
|
||||
},
|
||||
});
|
||||
const effect = await authorizer.authorize({
|
||||
action: 'artifact.read',
|
||||
subject: { type: 'api_app', id: 'app-1' },
|
||||
projectId: PROJECT_ID,
|
||||
runId: '019f7600-0000-7000-8000-000000000001',
|
||||
logArtifactId: `local-${'e'.repeat(30)}`,
|
||||
});
|
||||
assert.equal(effect, 'require_approval');
|
||||
assert.deepEqual(calls, [
|
||||
{
|
||||
subject: { type: 'api_app', id: 'app-1' },
|
||||
projectId: PROJECT_ID,
|
||||
permission: 'artifact.read',
|
||||
},
|
||||
]);
|
||||
await assert.rejects(
|
||||
authorizer.authorize({
|
||||
action: 'artifact.delete',
|
||||
subject: { type: 'api_app', id: 'app-1' },
|
||||
projectId: PROJECT_ID,
|
||||
runId: '019f7600-0000-7000-8000-000000000001',
|
||||
logArtifactId: `local-${'e'.repeat(30)}`,
|
||||
}),
|
||||
/action is invalid/,
|
||||
);
|
||||
});
|
||||
|
||||
test('real Policy Core keeps Artifact bytes unreachable until a viewer binding exists', async (t) => {
|
||||
const { repository } = await setup(t);
|
||||
const subject = { type: 'api_app', id: 'artifact-client' };
|
||||
const runId = '019f7600-0000-7000-8000-000000000001';
|
||||
const attemptId = '019f7600-0000-7000-8000-000000000002';
|
||||
const logArtifactId = `local-${'e'.repeat(30)}`;
|
||||
let byteReads = 0;
|
||||
const service = new LocalArtifactReadService(
|
||||
{
|
||||
async find() {
|
||||
return {
|
||||
projectId: PROJECT_ID,
|
||||
runId,
|
||||
attemptId,
|
||||
logArtifactId,
|
||||
};
|
||||
},
|
||||
},
|
||||
new ProjectPolicyArtifactReadAuthorizer(
|
||||
new ProjectPolicyEngine(repository),
|
||||
),
|
||||
{
|
||||
async read() {
|
||||
byteReads += 1;
|
||||
return {
|
||||
status: 'available',
|
||||
content: Buffer.from('log'),
|
||||
start: 0,
|
||||
endExclusive: 3,
|
||||
totalBytes: 3,
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
async read() {
|
||||
return null;
|
||||
},
|
||||
},
|
||||
);
|
||||
const request = {
|
||||
subject,
|
||||
projectId: PROJECT_ID,
|
||||
runId,
|
||||
logArtifactId,
|
||||
range: { offset: 0, length: 1024 },
|
||||
};
|
||||
assert.deepEqual(await service.read(request), {
|
||||
status: 'forbidden',
|
||||
effect: 'deny',
|
||||
});
|
||||
assert.equal(byteReads, 0);
|
||||
|
||||
await append(
|
||||
repository,
|
||||
binding({ subject, role: 'viewer', mutationId: 'artifact-viewer' }),
|
||||
0,
|
||||
);
|
||||
const available = await service.read(request);
|
||||
assert.equal(available.status, 'available');
|
||||
assert.equal(available.content.toString(), 'log');
|
||||
assert.equal(byteReads, 1);
|
||||
});
|
||||
|
||||
test('fails closed for corrupt current bindings and non-SQLite adapters', async (t) => {
|
||||
const { database, repository } = await setup(t);
|
||||
const current = binding();
|
||||
await append(repository, current, 0);
|
||||
await database.getQueryInterface().bulkUpdate(
|
||||
PROJECT_ROLE_BINDING_TABLE,
|
||||
{ role: null },
|
||||
{
|
||||
project_id: current.projectId,
|
||||
subject_type: current.subject.type,
|
||||
subject_id: current.subject.id,
|
||||
version: current.version,
|
||||
},
|
||||
);
|
||||
await assert.rejects(
|
||||
repository.resolve(PROJECT_ID, current.subject),
|
||||
ProjectPolicyUnavailableError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
new LegacySequelizeProjectPolicyRepository({
|
||||
getDialect() {
|
||||
return 'postgres';
|
||||
},
|
||||
}),
|
||||
/SQLite-only/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,264 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
FIXTURE,
|
||||
LIMITATIONS,
|
||||
validateApprovalManagementKubernetesLiveReport,
|
||||
} = require('../../scripts/ql3-approval-management-kubernetes-live-audit.cjs');
|
||||
|
||||
function digest(character) {
|
||||
return 'sha256:' + character.repeat(64);
|
||||
}
|
||||
|
||||
function validReport() {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
fixture: FIXTURE,
|
||||
observedAt: '2026-08-10T12:00:00.000Z',
|
||||
platform: {
|
||||
distribution: 'k3s',
|
||||
kubernetesVersion: 'v1.34.3+k3s1',
|
||||
architecture: 'arm64',
|
||||
kubernetesImageId: digest('1'),
|
||||
managementImageId: digest('2'),
|
||||
cniName: 'flannel',
|
||||
cniDistributionBinding: 'rancher/k3s:v1.34.3-k3s1',
|
||||
controlPlaneNodes: 1,
|
||||
workerNodes: 2,
|
||||
cniReadyNodes: 3,
|
||||
},
|
||||
database: {
|
||||
operator: 'cloudnative-pg',
|
||||
operatorVersion: '1.30.0',
|
||||
postgresVersionNumber: 180004,
|
||||
postgresImageId: digest('3'),
|
||||
instances: 3,
|
||||
readyInstances: 3,
|
||||
managerRole: 'ql3_approval_manager',
|
||||
migrationCount: 54,
|
||||
controlCoreCapability: 53,
|
||||
tlsVerified: true,
|
||||
primaryChangedDuringFailover: true,
|
||||
},
|
||||
deployment: {
|
||||
namespace: 'qinglong3-system',
|
||||
service: 'ql3-approval-management',
|
||||
port: 8447,
|
||||
replicas: 2,
|
||||
readyReplicas: 2,
|
||||
podIdentitySha256: [digest('4'), digest('5')],
|
||||
nodeIdentitySha256: [digest('6'), digest('7')],
|
||||
serviceAccount: 'ql3-approval-management',
|
||||
automountServiceAccountToken: false,
|
||||
requiredPodAntiAffinity: true,
|
||||
podDisruptionBudgetMinAvailable: 1,
|
||||
maxUnavailable: 0,
|
||||
maxConnectionsPerPod: 2,
|
||||
},
|
||||
client: {
|
||||
binary: 'ql3-approval-client',
|
||||
operations: ['approval.inspect', 'approval.decide'],
|
||||
inputKind: 'Secret',
|
||||
inputImmutable: true,
|
||||
callerDrivenJob: true,
|
||||
backoffLimit: 0,
|
||||
serviceAccountTokenMounted: false,
|
||||
rbacGranted: false,
|
||||
transportProtocol: 'TLSv1.3',
|
||||
mutualTls: true,
|
||||
servernameVerified: true,
|
||||
exactPodRequests: 5,
|
||||
inspectStatuses: ['found', 'found', 'found'],
|
||||
decisionStatuses: ['decided', 'existing'],
|
||||
responseRedacted: true,
|
||||
},
|
||||
identityRotation: {
|
||||
overlapOldAssertionAccepted: true,
|
||||
overlapNewAssertionAccepted: true,
|
||||
revokedOldAssertionRejected: true,
|
||||
activeNewAssertionAccepted: true,
|
||||
rollbackSurgeFailedClosed: true,
|
||||
twoReadyReplicasPreserved: true,
|
||||
durableGenerationReachedThree: true,
|
||||
},
|
||||
certificateRotation: {
|
||||
previousSerialSha256: digest('8'),
|
||||
currentSerialSha256: digest('9'),
|
||||
previousBundleSha256: digest('a'),
|
||||
currentBundleSha256: digest('b'),
|
||||
oldClientAcceptedBefore: true,
|
||||
replacementClientAcceptedBefore: true,
|
||||
oldClientRejectedAfter: true,
|
||||
replacementClientAcceptedAfter: true,
|
||||
fullPodReplacement: true,
|
||||
allReplicasReadyThroughout: true,
|
||||
},
|
||||
availability: {
|
||||
databaseFailureWithdrewReadiness: true,
|
||||
databaseFailurePreservedLiveness: true,
|
||||
stalePodsDidNotRecoverInPlace: true,
|
||||
freshPodsRecoveredAfterDatabase: true,
|
||||
bothReplicasServedAfterRecovery: true,
|
||||
},
|
||||
isolation: {
|
||||
labelledClientAllowed: true,
|
||||
unlabelledClientDenied: true,
|
||||
wrongPortDenied: true,
|
||||
kubernetesApiEgressDenied: true,
|
||||
publicInternetEgressDenied: true,
|
||||
cloudNativePgEgressAllowed: true,
|
||||
managerSecretReadDenied: true,
|
||||
managerMutationRbacDenied: true,
|
||||
},
|
||||
durability: {
|
||||
approvalVersion: 2,
|
||||
approvalState: 'approved',
|
||||
decisionIdSha256: digest('c'),
|
||||
allowedAuditCount: 4,
|
||||
deniedAuditCount: 1,
|
||||
duplicateDecisionCount: 0,
|
||||
identityGeneration: 3,
|
||||
survivedCloudNativePgFailover: true,
|
||||
},
|
||||
gates: {
|
||||
realThreeNodeKubernetes: true,
|
||||
realCniPolicy: true,
|
||||
threeInstanceCloudNativePg: true,
|
||||
twoManagerPodsOnDistinctNodes: true,
|
||||
tls13ProductClientAcrossBothPods: true,
|
||||
strongUserDecision: true,
|
||||
identityProjectionRotation: true,
|
||||
certificateRevocationRollout: true,
|
||||
databaseReadinessFence: true,
|
||||
durableFactsSurvivedFailover: true,
|
||||
leastPrivilege: true,
|
||||
passed: true,
|
||||
},
|
||||
limitations: [...LIMITATIONS],
|
||||
};
|
||||
}
|
||||
|
||||
function mutate(change) {
|
||||
const report = structuredClone(validReport());
|
||||
change(report);
|
||||
return validateApprovalManagementKubernetesLiveReport(report);
|
||||
}
|
||||
|
||||
test('accepts the exact content-free Kubernetes approval live report', () => {
|
||||
assert.deepEqual(
|
||||
validateApprovalManagementKubernetesLiveReport(validReport()),
|
||||
{
|
||||
schemaVersion: 1,
|
||||
fixture: FIXTURE,
|
||||
findings: [],
|
||||
compatible: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects topology, migration and approval deployment weakening', () => {
|
||||
for (const [code, change] of [
|
||||
[
|
||||
'QL3_APPROVAL_KUBERNETES_LIVE_PLATFORM',
|
||||
(report) => {
|
||||
report.platform.workerNodes = 1;
|
||||
},
|
||||
],
|
||||
[
|
||||
'QL3_APPROVAL_KUBERNETES_LIVE_DATABASE',
|
||||
(report) => {
|
||||
report.database.migrationCount = 52;
|
||||
},
|
||||
],
|
||||
[
|
||||
'QL3_APPROVAL_KUBERNETES_LIVE_DEPLOYMENT',
|
||||
(report) => {
|
||||
report.deployment.nodeIdentitySha256[1] =
|
||||
report.deployment.nodeIdentitySha256[0];
|
||||
},
|
||||
],
|
||||
]) {
|
||||
assert.ok(mutate(change).findings.some((entry) => entry.code === code));
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects widened client, incomplete rotations and false availability', () => {
|
||||
for (const [code, change] of [
|
||||
[
|
||||
'QL3_APPROVAL_KUBERNETES_LIVE_CLIENT',
|
||||
(report) => {
|
||||
report.client.rbacGranted = true;
|
||||
},
|
||||
],
|
||||
[
|
||||
'QL3_APPROVAL_KUBERNETES_LIVE_IDENTITY_ROTATION',
|
||||
(report) => {
|
||||
report.identityRotation.revokedOldAssertionRejected = false;
|
||||
},
|
||||
],
|
||||
[
|
||||
'QL3_APPROVAL_KUBERNETES_LIVE_CERTIFICATE_ROTATION',
|
||||
(report) => {
|
||||
report.certificateRotation.fullPodReplacement = false;
|
||||
},
|
||||
],
|
||||
[
|
||||
'QL3_APPROVAL_KUBERNETES_LIVE_AVAILABILITY',
|
||||
(report) => {
|
||||
report.availability.stalePodsDidNotRecoverInPlace = false;
|
||||
},
|
||||
],
|
||||
]) {
|
||||
assert.ok(mutate(change).findings.some((entry) => entry.code === code));
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects incomplete isolation and durable approval drift', () => {
|
||||
assert.ok(
|
||||
mutate((report) => {
|
||||
report.isolation.publicInternetEgressDenied = false;
|
||||
}).findings.some(
|
||||
({ code }) => code === 'QL3_APPROVAL_KUBERNETES_LIVE_ISOLATION',
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
mutate((report) => {
|
||||
report.durability.approvalVersion = 1;
|
||||
}).findings.some(
|
||||
({ code }) => code === 'QL3_APPROVAL_KUBERNETES_LIVE_DURABILITY',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects secret material, widened schema, false gates and hidden limitations', () => {
|
||||
for (const [code, change] of [
|
||||
[
|
||||
'QL3_APPROVAL_KUBERNETES_LIVE_SECRET_EXPOSURE',
|
||||
(report) => {
|
||||
report.client.assertion =
|
||||
'eyJhbGciOiJFZERTQSJ9.eyJzdWIiOiJvcGVyYXRvciJ9.signature0123456789';
|
||||
},
|
||||
],
|
||||
[
|
||||
'QL3_APPROVAL_KUBERNETES_LIVE_REPORT_SHAPE',
|
||||
(report) => {
|
||||
report.debug = true;
|
||||
},
|
||||
],
|
||||
[
|
||||
'QL3_APPROVAL_KUBERNETES_LIVE_GATES',
|
||||
(report) => {
|
||||
report.gates.passed = false;
|
||||
},
|
||||
],
|
||||
[
|
||||
'QL3_APPROVAL_KUBERNETES_LIVE_LIMITATIONS',
|
||||
(report) => {
|
||||
report.limitations = [];
|
||||
},
|
||||
],
|
||||
]) {
|
||||
assert.ok(mutate(change).findings.some((entry) => entry.code === code));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,125 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { spawnSync } = require('node:child_process');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
assertion,
|
||||
assertionForSubject,
|
||||
decisionCommand,
|
||||
inspectCommand,
|
||||
keyset,
|
||||
reviewedKey,
|
||||
weakAssertion,
|
||||
} = require('../../scripts/ql3-approval-management-kubernetes-live-contract.cjs');
|
||||
|
||||
test('approval live report path is mandatory before mutation begins', () => {
|
||||
const script = path.resolve(
|
||||
__dirname,
|
||||
'../../scripts/ql3-approval-management-kubernetes-live-contract.cjs',
|
||||
);
|
||||
const result = spawnSync(process.execPath, [script], {
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, QL3_APPROVAL_MANAGEMENT_KUBERNETES_LIVE: '1' },
|
||||
});
|
||||
assert.equal(result.status, 1);
|
||||
assert.match(result.stderr, /--report=\/absolute\/private-report\.json/);
|
||||
assert.doesNotMatch(result.stderr, /Docker\/Kubernetes/);
|
||||
});
|
||||
|
||||
test('approval live can sign a strong but unauthorized User identity', () => {
|
||||
const key = reviewedKey('approval-outsider-test-key');
|
||||
const payload = JSON.parse(
|
||||
Buffer.from(
|
||||
assertionForSubject(key, 'approval-outsider', 'unit-test').split('.')[1],
|
||||
'base64url',
|
||||
),
|
||||
);
|
||||
assert.equal(payload.sub, 'approval-outsider');
|
||||
assert.equal(payload.acr, 'urn:ql3:mfa');
|
||||
assert.deepEqual(payload.amr, ['pwd', 'otp']);
|
||||
});
|
||||
|
||||
test('approval live identity is audience, type, purpose and assurance bound', () => {
|
||||
const key = reviewedKey('approval-live-test-key');
|
||||
const document = keyset(1, [key]);
|
||||
assert.equal(document.generation, 1);
|
||||
assert.equal(document.audience, 'qinglong3-approval-management');
|
||||
assert.deepEqual(document.revokedKids, []);
|
||||
|
||||
const strong = assertion(key, 'strong-unit-test');
|
||||
const [encodedHeader, encodedPayload, signature] = strong.split('.');
|
||||
assert.ok(signature.length > 32);
|
||||
assert.deepEqual(JSON.parse(Buffer.from(encodedHeader, 'base64url')), {
|
||||
alg: 'EdDSA',
|
||||
kid: 'approval-live-test-key',
|
||||
typ: 'ql3-approval-management+jwt',
|
||||
});
|
||||
const payload = JSON.parse(Buffer.from(encodedPayload, 'base64url'));
|
||||
assert.equal(payload.aud, 'qinglong3-approval-management');
|
||||
assert.equal(payload.ql3_purpose, 'approval-management');
|
||||
assert.equal(payload.sub, 'approval-operator');
|
||||
assert.equal(payload.acr, 'urn:ql3:mfa');
|
||||
assert.deepEqual(payload.amr, ['pwd', 'otp']);
|
||||
|
||||
const weakPayload = JSON.parse(
|
||||
Buffer.from(weakAssertion(key, 'weak-unit-test').split('.')[1], 'base64url'),
|
||||
);
|
||||
assert.equal(weakPayload.acr, 'urn:ql3:password');
|
||||
assert.deepEqual(weakPayload.amr, ['pwd']);
|
||||
});
|
||||
|
||||
test('approval live commands bind exact request, action and distinct audits', () => {
|
||||
const inspect = inspectCommand(
|
||||
'project-a',
|
||||
'approval-a',
|
||||
'inspect-a',
|
||||
1,
|
||||
);
|
||||
assert.equal(inspect.operation, 'approval.inspect');
|
||||
assert.notEqual(
|
||||
inspect.request.auditEventId,
|
||||
inspect.request.failureAuditEventId,
|
||||
);
|
||||
|
||||
const decide = decisionCommand(
|
||||
'project-a',
|
||||
'approval-a',
|
||||
'decide-a',
|
||||
'decision-a',
|
||||
2,
|
||||
);
|
||||
assert.equal(decide.operation, 'approval.decide');
|
||||
assert.equal(decide.request.expectedVersion, 1);
|
||||
assert.equal(decide.request.expectedAction.permission, 'run.start');
|
||||
assert.equal(decide.request.decision, 'approved');
|
||||
assert.equal(decide.request.reasonCode, 'reviewed');
|
||||
});
|
||||
|
||||
test('approval live runner remains opt-in, audited and observation backed', () => {
|
||||
const source = fs.readFileSync(
|
||||
path.resolve(
|
||||
__dirname,
|
||||
'../../scripts/ql3-approval-management-kubernetes-live-contract.cjs',
|
||||
),
|
||||
'utf8',
|
||||
);
|
||||
assert.match(
|
||||
source,
|
||||
/QL3_APPROVAL_MANAGEMENT_KUBERNETES_LIVE !== '1'/,
|
||||
);
|
||||
assert.match(source, /reviewedOperatorManifest\(operatorManifestFile\)/);
|
||||
assert.match(source, /validateApprovalManagementKubernetesLiveReport/);
|
||||
assert.match(source, /createManagementClientExecutor/);
|
||||
assert.match(source, /clientTcpProbe/);
|
||||
assert.match(source, /podTcpProbe/);
|
||||
assert.match(source, /weakUserRejected/);
|
||||
assert.match(source, /identity ledger rollback surge failure/);
|
||||
assert.match(source, /CloudNativePG primary promotion/);
|
||||
assert.match(source, /migrationCount: 54/);
|
||||
assert.match(source, /controlCoreCapability: 53/);
|
||||
assert.match(source, /flannel\.alpha\.coreos\.com\/backend-type/);
|
||||
assert.doesNotMatch(source, /kubectl.*logs/);
|
||||
assert.doesNotMatch(source, /ceremony is not complete/);
|
||||
});
|
||||
@@ -0,0 +1,260 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
FIXTURE,
|
||||
LIMITATIONS,
|
||||
validateAutomationManagementKubernetesLiveReport,
|
||||
} = require('../../scripts/ql3-automation-management-kubernetes-live-audit.cjs');
|
||||
|
||||
function digest(character) {
|
||||
return `sha256:${character.repeat(64)}`;
|
||||
}
|
||||
|
||||
function validReport() {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
fixture: FIXTURE,
|
||||
observedAt: '2026-08-01T12:00:00.000Z',
|
||||
platform: {
|
||||
distribution: 'k3s',
|
||||
kubernetesVersion: 'v1.34.3+k3s1',
|
||||
architecture: 'arm64',
|
||||
kubernetesImageId: digest('1'),
|
||||
managementImageId: digest('2'),
|
||||
cniName: 'flannel',
|
||||
cniDistributionBinding: 'rancher/k3s:v1.34.3-k3s1',
|
||||
controlPlaneNodes: 1,
|
||||
workerNodes: 2,
|
||||
cniReadyNodes: 3,
|
||||
},
|
||||
database: {
|
||||
operator: 'cloudnative-pg',
|
||||
operatorVersion: '1.27.1',
|
||||
postgresVersionNumber: 180004,
|
||||
postgresImageId: digest('3'),
|
||||
instances: 3,
|
||||
readyInstances: 3,
|
||||
managerRole: 'ql3_automation_manager',
|
||||
controlCoreCapability: 53,
|
||||
tlsVerified: true,
|
||||
primaryChangedDuringFailover: true,
|
||||
},
|
||||
deployment: {
|
||||
namespace: 'qinglong3-system',
|
||||
service: 'ql3-automation-management',
|
||||
port: 8445,
|
||||
replicas: 2,
|
||||
readyReplicas: 2,
|
||||
podIdentitySha256: [digest('4'), digest('5')],
|
||||
nodeIdentitySha256: [digest('6'), digest('7')],
|
||||
serviceAccount: 'ql3-automation-management',
|
||||
automountServiceAccountToken: false,
|
||||
requiredPodAntiAffinity: true,
|
||||
podDisruptionBudgetMinAvailable: 1,
|
||||
maxUnavailable: 0,
|
||||
maxConnectionsPerPod: 2,
|
||||
},
|
||||
client: {
|
||||
binary: 'ql3-automation-client',
|
||||
operation: 'task.publish',
|
||||
inputKind: 'Secret',
|
||||
inputImmutable: true,
|
||||
callerDrivenJob: true,
|
||||
backoffLimit: 0,
|
||||
serviceAccountTokenMounted: false,
|
||||
rbacGranted: false,
|
||||
transportProtocol: 'TLSv1.3',
|
||||
mutualTls: true,
|
||||
servernameVerified: true,
|
||||
exactPodRequests: 2,
|
||||
resultStatuses: ['created', 'existing'],
|
||||
responseRedacted: true,
|
||||
},
|
||||
identityRotation: {
|
||||
overlapOldAssertionAccepted: true,
|
||||
overlapNewAssertionAccepted: true,
|
||||
revokedOldAssertionRejected: true,
|
||||
activeNewAssertionAccepted: true,
|
||||
rollbackSurgeFailedClosed: true,
|
||||
twoReadyReplicasPreserved: true,
|
||||
durableGenerationReachedThree: true,
|
||||
},
|
||||
certificateRotation: {
|
||||
previousSerialSha256: digest('8'),
|
||||
currentSerialSha256: digest('9'),
|
||||
previousBundleSha256: digest('a'),
|
||||
currentBundleSha256: digest('b'),
|
||||
oldClientAcceptedBefore: true,
|
||||
replacementClientAcceptedBefore: true,
|
||||
oldClientRejectedAfter: true,
|
||||
replacementClientAcceptedAfter: true,
|
||||
fullPodReplacement: true,
|
||||
allReplicasReadyThroughout: true,
|
||||
},
|
||||
availability: {
|
||||
databaseFailureWithdrewReadiness: true,
|
||||
databaseFailurePreservedLiveness: true,
|
||||
stalePodsDidNotRecoverInPlace: true,
|
||||
freshPodsRecoveredAfterDatabase: true,
|
||||
bothReplicasServedAfterRecovery: true,
|
||||
},
|
||||
isolation: {
|
||||
labelledClientAllowed: true,
|
||||
unlabelledClientDenied: true,
|
||||
wrongPortDenied: true,
|
||||
kubernetesApiEgressDenied: true,
|
||||
publicInternetEgressDenied: true,
|
||||
cloudNativePgEgressAllowed: true,
|
||||
managerSecretReadDenied: true,
|
||||
managerMutationRbacDenied: true,
|
||||
},
|
||||
durability: {
|
||||
taskRevisionCount: 4,
|
||||
triggerRevisionCount: 2,
|
||||
allowedAuditCount: 6,
|
||||
replayDuplicateCount: 0,
|
||||
taskCurrentRevision: 4,
|
||||
triggerCurrentRevision: 2,
|
||||
survivedCloudNativePgFailover: true,
|
||||
},
|
||||
gates: {
|
||||
realThreeNodeKubernetes: true,
|
||||
realCniPolicy: true,
|
||||
threeInstanceCloudNativePg: true,
|
||||
twoManagerPodsOnDistinctNodes: true,
|
||||
tls13ProductClientAcrossBothPods: true,
|
||||
identityProjectionRotation: true,
|
||||
certificateRevocationRollout: true,
|
||||
databaseReadinessFence: true,
|
||||
durableFactsSurvivedFailover: true,
|
||||
leastPrivilege: true,
|
||||
passed: true,
|
||||
},
|
||||
limitations: [...LIMITATIONS],
|
||||
};
|
||||
}
|
||||
|
||||
function mutate(change) {
|
||||
const report = structuredClone(validReport());
|
||||
change(report);
|
||||
return validateAutomationManagementKubernetesLiveReport(report);
|
||||
}
|
||||
|
||||
test('accepts the exact low-sensitive Kubernetes automation live report', () => {
|
||||
assert.deepEqual(
|
||||
validateAutomationManagementKubernetesLiveReport(validReport()),
|
||||
{
|
||||
schemaVersion: 1,
|
||||
fixture: FIXTURE,
|
||||
findings: [],
|
||||
compatible: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects topology, CloudNativePG and manager deployment weakening', () => {
|
||||
for (const [code, change] of [
|
||||
[
|
||||
'QL3_AUTOMATION_KUBERNETES_LIVE_PLATFORM',
|
||||
(report) => {
|
||||
report.platform.workerNodes = 1;
|
||||
},
|
||||
],
|
||||
[
|
||||
'QL3_AUTOMATION_KUBERNETES_LIVE_DATABASE',
|
||||
(report) => {
|
||||
report.database.instances = 1;
|
||||
},
|
||||
],
|
||||
[
|
||||
'QL3_AUTOMATION_KUBERNETES_LIVE_DEPLOYMENT',
|
||||
(report) => {
|
||||
report.deployment.nodeIdentitySha256[1] =
|
||||
report.deployment.nodeIdentitySha256[0];
|
||||
},
|
||||
],
|
||||
]) {
|
||||
assert.ok(mutate(change).findings.some((entry) => entry.code === code));
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects widened client, incomplete rotations and false availability', () => {
|
||||
for (const [code, change] of [
|
||||
[
|
||||
'QL3_AUTOMATION_KUBERNETES_LIVE_CLIENT',
|
||||
(report) => {
|
||||
report.client.rbacGranted = true;
|
||||
},
|
||||
],
|
||||
[
|
||||
'QL3_AUTOMATION_KUBERNETES_LIVE_IDENTITY_ROTATION',
|
||||
(report) => {
|
||||
report.identityRotation.revokedOldAssertionRejected = false;
|
||||
},
|
||||
],
|
||||
[
|
||||
'QL3_AUTOMATION_KUBERNETES_LIVE_CERTIFICATE_ROTATION',
|
||||
(report) => {
|
||||
report.certificateRotation.fullPodReplacement = false;
|
||||
},
|
||||
],
|
||||
[
|
||||
'QL3_AUTOMATION_KUBERNETES_LIVE_AVAILABILITY',
|
||||
(report) => {
|
||||
report.availability.stalePodsDidNotRecoverInPlace = false;
|
||||
},
|
||||
],
|
||||
]) {
|
||||
assert.ok(mutate(change).findings.some((entry) => entry.code === code));
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects incomplete CNI isolation and durable fact drift', () => {
|
||||
assert.ok(
|
||||
mutate((report) => {
|
||||
report.isolation.publicInternetEgressDenied = false;
|
||||
}).findings.some(
|
||||
({ code }) => code === 'QL3_AUTOMATION_KUBERNETES_LIVE_ISOLATION',
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
mutate((report) => {
|
||||
report.durability.replayDuplicateCount = 1;
|
||||
}).findings.some(
|
||||
({ code }) => code === 'QL3_AUTOMATION_KUBERNETES_LIVE_DURABILITY',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects secret material, widened schema, false gates and hidden limitations', () => {
|
||||
for (const [code, change] of [
|
||||
[
|
||||
'QL3_AUTOMATION_KUBERNETES_LIVE_SECRET_EXPOSURE',
|
||||
(report) => {
|
||||
report.client.assertion =
|
||||
'eyJhbGciOiJFZERTQSJ9.eyJzdWIiOiJvcGVyYXRvciJ9.signature0123456789';
|
||||
},
|
||||
],
|
||||
[
|
||||
'QL3_AUTOMATION_KUBERNETES_LIVE_REPORT_SHAPE',
|
||||
(report) => {
|
||||
report.debug = true;
|
||||
},
|
||||
],
|
||||
[
|
||||
'QL3_AUTOMATION_KUBERNETES_LIVE_GATES',
|
||||
(report) => {
|
||||
report.gates.passed = false;
|
||||
},
|
||||
],
|
||||
[
|
||||
'QL3_AUTOMATION_KUBERNETES_LIVE_LIMITATIONS',
|
||||
(report) => {
|
||||
report.limitations = [];
|
||||
},
|
||||
],
|
||||
]) {
|
||||
assert.ok(mutate(change).findings.some((entry) => entry.code === code));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,80 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
assertion,
|
||||
envelope,
|
||||
keyset,
|
||||
reviewedKey,
|
||||
taskCommand,
|
||||
triggerCommand,
|
||||
} = require('../../scripts/ql3-automation-management-kubernetes-live-contract.cjs');
|
||||
|
||||
test('automation live identity ceremony is audience, type and purpose bound', () => {
|
||||
const key = reviewedKey('automation-live-test-key');
|
||||
const document = keyset(1, [key]);
|
||||
assert.equal(document.generation, 1);
|
||||
assert.equal(document.audience, 'qinglong3-automation-management');
|
||||
assert.deepEqual(document.revokedKids, []);
|
||||
const token = assertion(key, 'unit-test');
|
||||
const [encodedHeader, encodedPayload, signature] = token.split('.');
|
||||
assert.ok(signature.length > 32);
|
||||
const header = JSON.parse(Buffer.from(encodedHeader, 'base64url'));
|
||||
const payload = JSON.parse(Buffer.from(encodedPayload, 'base64url'));
|
||||
assert.deepEqual(header, {
|
||||
alg: 'EdDSA',
|
||||
kid: 'automation-live-test-key',
|
||||
typ: 'ql3-automation-management+jwt',
|
||||
});
|
||||
assert.equal(payload.aud, 'qinglong3-automation-management');
|
||||
assert.equal(payload.ql3_purpose, 'automation-management');
|
||||
assert.equal(payload.sub, 'automation-operator');
|
||||
});
|
||||
|
||||
test('automation live commands retain exact revision and task pinning', () => {
|
||||
const task = taskCommand('project-a', null, '001', 'v1');
|
||||
const taskEnvelope = envelope('task.publish', 'task-v1', task);
|
||||
assert.equal(taskEnvelope.operation, 'task.publish');
|
||||
assert.equal(taskEnvelope.request.command.expectedRevision, null);
|
||||
const published = {
|
||||
taskId: task.taskId,
|
||||
revision: 1,
|
||||
contentDigest: 'a'.repeat(64),
|
||||
};
|
||||
const trigger = triggerCommand('project-a', null, published, '001');
|
||||
assert.equal(trigger.taskRevision, 1);
|
||||
assert.equal(trigger.taskContentDigest, 'a'.repeat(64));
|
||||
});
|
||||
|
||||
test('automation live runner remains opt-in, audited and observation backed', () => {
|
||||
const source = fs.readFileSync(
|
||||
path.resolve(
|
||||
__dirname,
|
||||
'../../scripts/ql3-automation-management-kubernetes-live-contract.cjs',
|
||||
),
|
||||
'utf8',
|
||||
);
|
||||
assert.match(source, /QL3_AUTOMATION_MANAGEMENT_KUBERNETES_LIVE !== '1'/);
|
||||
assert.match(source, /reviewedOperatorManifest\(operatorManifestFile\)/);
|
||||
assert.match(source, /validateAutomationManagementKubernetesLiveReport/);
|
||||
assert.match(source, /clientTcpProbe\(/);
|
||||
assert.match(source, /podTcpProbe\(/);
|
||||
assert.match(source, /\/dev\/termination-log/);
|
||||
assert.match(source, /umask 077/);
|
||||
assert.match(source, /chmod 600 \/tmp\/client\.json/);
|
||||
assert.match(source, /QL3_PLUGIN_PACKAGE_MANAGEMENT_CLIENT_REQUEST_FAILED/);
|
||||
assert.match(source, /"\$attempt" -ge 60/);
|
||||
assert.doesNotMatch(source, /logs', `job\/\$\{definition\.name\}`/);
|
||||
assert.match(source, /identity ledger rollback surge failure/);
|
||||
assert.match(source, /CloudNativePG primary promotion/);
|
||||
assert.match(source, /flannel\.alpha\.coreos\.com\/backend-type/);
|
||||
assert.match(
|
||||
source,
|
||||
/finalNodes = fixture\.kubectlJson\(\['get', 'nodes'\]\)/,
|
||||
);
|
||||
assert.match(source, /new Set\(cniReadyNodes\.map/);
|
||||
assert.doesNotMatch(source, /app=flannel/);
|
||||
assert.doesNotMatch(source, /ceremony is not complete/);
|
||||
});
|
||||
@@ -0,0 +1,127 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
auditBarmanCloudSupplyChain,
|
||||
} = require('../../scripts/ql3-barman-cloud-supply-chain-audit.cjs');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '../..');
|
||||
const LOCK =
|
||||
'deploy/kubernetes/ql3-cluster/operators/barman-cloud/plugin-lock.json';
|
||||
|
||||
function mutateLock(transform) {
|
||||
const target = path.join(ROOT, LOCK);
|
||||
return (filePath, encoding) => {
|
||||
const source = fs.readFileSync(filePath, encoding);
|
||||
if (path.resolve(filePath) !== target) return source;
|
||||
return JSON.stringify(transform(JSON.parse(source)));
|
||||
};
|
||||
}
|
||||
|
||||
test('accepts the exact Barman candidate lock while preserving release blockers', () => {
|
||||
const report = auditBarmanCloudSupplyChain({ root: ROOT });
|
||||
assert.equal(report.compatible, true, JSON.stringify(report.findings));
|
||||
assert.equal(report.pluginVersion, '0.13.0');
|
||||
assert.equal(report.releaseReady, false);
|
||||
assert.deepEqual(report.releaseBlockers, [
|
||||
'live-object-store-backup-wal-latest-restore-pitr-evidence',
|
||||
]);
|
||||
});
|
||||
|
||||
test('rejects a movable or drifted controller image', () => {
|
||||
const report = auditBarmanCloudSupplyChain({
|
||||
root: ROOT,
|
||||
readFile: mutateLock((lock) => {
|
||||
lock.plugin.controller.image =
|
||||
'ghcr.io/cloudnative-pg/plugin-barman-cloud:v0.13.0';
|
||||
return lock;
|
||||
}),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_BARMAN_CONTROLLER_IMAGE',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a sidecar platform digest drift', () => {
|
||||
const report = auditBarmanCloudSupplyChain({
|
||||
root: ROOT,
|
||||
readFile: mutateLock((lock) => {
|
||||
lock.plugin.sidecar.platforms['linux/arm64'] = `sha256:${'0'.repeat(64)}`;
|
||||
return lock;
|
||||
}),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_BARMAN_SIDECAR_IMAGE',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects an unverified release asset', () => {
|
||||
const report = auditBarmanCloudSupplyChain({
|
||||
root: ROOT,
|
||||
readFile: mutateLock((lock) => {
|
||||
lock.plugin.releaseManifestSha256 = 'unverified';
|
||||
return lock;
|
||||
}),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_BARMAN_RELEASE_ASSET',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects certificate authority supply-chain status drift', () => {
|
||||
const report = auditBarmanCloudSupplyChain({
|
||||
root: ROOT,
|
||||
readFile: mutateLock((lock) => {
|
||||
lock.certificateAuthority.status = 'selected-unverified';
|
||||
return lock;
|
||||
}),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_BARMAN_CERTIFICATE_GATE',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects premature release readiness or an unreviewed installer', () => {
|
||||
const readiness = auditBarmanCloudSupplyChain({
|
||||
root: ROOT,
|
||||
readFile: mutateLock((lock) => {
|
||||
lock.releaseReady = true;
|
||||
lock.releaseBlockers = [];
|
||||
return lock;
|
||||
}),
|
||||
});
|
||||
assert.equal(
|
||||
readiness.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_BARMAN_PREMATURE_RELEASE',
|
||||
),
|
||||
true,
|
||||
);
|
||||
|
||||
const installer = auditBarmanCloudSupplyChain({
|
||||
root: ROOT,
|
||||
readDirectory: () => ['plugin-lock.json', 'manifest.yaml'],
|
||||
});
|
||||
assert.equal(
|
||||
installer.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_BARMAN_INSTALLER_UNREVIEWED',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
createQingLong3PackageClosureBuildPlan,
|
||||
resolveQingLong3Package,
|
||||
} = require('../../scripts/ql3-build-package-closure.cjs');
|
||||
|
||||
function fixture(t, directoryName = 'ql3-example', name = '@qinglong/example') {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-build-closure-'));
|
||||
const packageDirectory = path.join(root, 'packages', directoryName);
|
||||
fs.mkdirSync(packageDirectory, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(packageDirectory, 'package.json'),
|
||||
JSON.stringify({ name, scripts: { build: 'tsc -p tsconfig.json' } }),
|
||||
);
|
||||
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
|
||||
return { packageDirectory, root };
|
||||
}
|
||||
|
||||
test('builds one package and its workspace dependency closure in topology order', (t) => {
|
||||
const { packageDirectory, root } = fixture(t);
|
||||
assert.deepEqual(resolveQingLong3Package(root, packageDirectory), {
|
||||
name: '@qinglong/example',
|
||||
packageDirectory,
|
||||
});
|
||||
const plan = createQingLong3PackageClosureBuildPlan(root, packageDirectory);
|
||||
assert.deepEqual(plan.args, [
|
||||
'-r',
|
||||
'--workspace-concurrency=1',
|
||||
'--filter',
|
||||
'@qinglong/example...',
|
||||
'run',
|
||||
'build',
|
||||
]);
|
||||
assert.equal(plan.cwd, root);
|
||||
});
|
||||
|
||||
test('rejects package escapes, non-QL3 directories and recursive builds', (t) => {
|
||||
const valid = fixture(t);
|
||||
assert.throws(
|
||||
() => resolveQingLong3Package(valid.root, valid.root),
|
||||
/cwd must be one direct packages\/ql3-\* directory/,
|
||||
);
|
||||
|
||||
const legacy = fixture(t, 'legacy-example');
|
||||
assert.throws(
|
||||
() => resolveQingLong3Package(legacy.root, legacy.packageDirectory),
|
||||
/cwd must be one direct packages\/ql3-\* directory/,
|
||||
);
|
||||
|
||||
const recursive = fixture(t, 'ql3-recursive', '@qinglong/recursive');
|
||||
const manifestPath = path.join(recursive.packageDirectory, 'package.json');
|
||||
fs.writeFileSync(
|
||||
manifestPath,
|
||||
JSON.stringify({
|
||||
name: '@qinglong/recursive',
|
||||
scripts: { build: 'pnpm --filter @qinglong/example build && tsc' },
|
||||
}),
|
||||
);
|
||||
assert.throws(
|
||||
() => resolveQingLong3Package(recursive.root, recursive.packageDirectory),
|
||||
/build must compile only itself/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,130 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
auditCertManagerSelection,
|
||||
} = require('../../scripts/ql3-cert-manager-selection-audit.cjs');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '../..');
|
||||
const SELECTION =
|
||||
'deploy/kubernetes/ql3-cluster/operators/cert-manager/selection-lock.json';
|
||||
const BARMAN =
|
||||
'deploy/kubernetes/ql3-cluster/operators/barman-cloud/plugin-lock.json';
|
||||
|
||||
function mutateJson(relativePath, transform) {
|
||||
const target = path.join(ROOT, relativePath);
|
||||
return (filePath, encoding) => {
|
||||
const source = fs.readFileSync(filePath, encoding);
|
||||
if (path.resolve(filePath) !== target) return source;
|
||||
return JSON.stringify(transform(JSON.parse(source)));
|
||||
};
|
||||
}
|
||||
|
||||
test('accepts the supply-chain-verified cert-manager selection with live blockers', () => {
|
||||
const report = auditCertManagerSelection({ root: ROOT });
|
||||
assert.equal(report.compatible, true, JSON.stringify(report.findings));
|
||||
assert.equal(report.certManagerVersion, '1.20.3');
|
||||
assert.equal(report.kubernetesVersion, '1.32.8');
|
||||
assert.equal(report.releaseReady, false);
|
||||
assert.deepEqual(report.releaseBlockers, [
|
||||
'live-cert-manager-api-and-plugin-mtls-rotation-evidence',
|
||||
]);
|
||||
});
|
||||
|
||||
test('rejects selecting a cert-manager minor outside the Kubernetes baseline', () => {
|
||||
const report = auditCertManagerSelection({
|
||||
root: ROOT,
|
||||
readFile: mutateJson(SELECTION, (selection) => {
|
||||
selection.certManager.version = '1.21.0';
|
||||
return selection;
|
||||
}),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_CERT_MANAGER_SELECTION',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects widening or hiding the Kubernetes support boundary', () => {
|
||||
const report = auditCertManagerSelection({
|
||||
root: ROOT,
|
||||
readFile: mutateJson(SELECTION, (selection) => {
|
||||
selection.compatibility.supportedKubernetesMin = '1.33';
|
||||
return selection;
|
||||
}),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) =>
|
||||
candidate.code === 'QL3_CERT_MANAGER_KUBERNETES_COMPATIBILITY',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects pretending the live evidence gate is release-ready', () => {
|
||||
const report = auditCertManagerSelection({
|
||||
root: ROOT,
|
||||
readFile: mutateJson(SELECTION, (selection) => {
|
||||
selection.releaseReady = true;
|
||||
selection.releaseBlockers = [];
|
||||
return selection;
|
||||
}),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_CERT_MANAGER_PREMATURE_RELEASE',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects certificate identity, usage or namespace drift', () => {
|
||||
const report = auditCertManagerSelection({
|
||||
root: ROOT,
|
||||
readFile: mutateJson(SELECTION, (selection) => {
|
||||
selection.pluginTls.certificates[1].dnsNames = ['unreviewed'];
|
||||
return selection;
|
||||
}),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_CERT_MANAGER_PLUGIN_TLS',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects Barman binding drift or an unverified installer', () => {
|
||||
const binding = auditCertManagerSelection({
|
||||
root: ROOT,
|
||||
readFile: mutateJson(BARMAN, (lock) => {
|
||||
lock.certificateAuthority.version = 'latest';
|
||||
return lock;
|
||||
}),
|
||||
});
|
||||
assert.equal(
|
||||
binding.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_CERT_MANAGER_BARMAN_BINDING',
|
||||
),
|
||||
true,
|
||||
);
|
||||
|
||||
const installer = auditCertManagerSelection({
|
||||
root: ROOT,
|
||||
readDirectory: () => ['selection-lock.json', 'manifest.yaml'],
|
||||
});
|
||||
assert.equal(
|
||||
installer.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_CERT_MANAGER_INSTALLER_UNVERIFIED',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,64 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
cleanQingLong3PackageArtifacts,
|
||||
} = require('../../scripts/ql3-clean-package-artifacts.cjs');
|
||||
|
||||
test('removes dist and paired emitted source files only from QL3 importers', (t) => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-clean-artifacts-'));
|
||||
t.after(() => fs.rmSync(root, { force: true, recursive: true }));
|
||||
|
||||
const registeredPackage = path.join(root, 'packages/ql3-runtime-core');
|
||||
const unregisteredPackage = path.join(root, 'packages/ql3-stale-package');
|
||||
const legacyPackage = path.join(root, 'packages/legacy-package');
|
||||
for (const packageDirectory of [
|
||||
registeredPackage,
|
||||
unregisteredPackage,
|
||||
legacyPackage,
|
||||
]) {
|
||||
fs.mkdirSync(path.join(packageDirectory, 'dist'), { recursive: true });
|
||||
fs.writeFileSync(path.join(packageDirectory, 'dist/stale.js'), 'stale');
|
||||
fs.writeFileSync(path.join(packageDirectory, 'source.ts'), 'source');
|
||||
}
|
||||
fs.writeFileSync(path.join(registeredPackage, 'package.json'), '{}');
|
||||
fs.writeFileSync(path.join(legacyPackage, 'package.json'), '{}');
|
||||
fs.mkdirSync(path.join(registeredPackage, 'src/nested'), { recursive: true });
|
||||
fs.writeFileSync(path.join(registeredPackage, 'src/nested/example.ts'), '');
|
||||
for (const suffix of ['js', 'js.map', 'd.ts', 'd.ts.map']) {
|
||||
fs.writeFileSync(
|
||||
path.join(registeredPackage, `src/nested/example.${suffix}`),
|
||||
'emitted',
|
||||
);
|
||||
}
|
||||
fs.writeFileSync(
|
||||
path.join(registeredPackage, 'src/nested/intentional.js'),
|
||||
'module.exports = {};',
|
||||
);
|
||||
|
||||
assert.deepEqual(cleanQingLong3PackageArtifacts(root), [
|
||||
'packages/ql3-runtime-core/dist',
|
||||
]);
|
||||
assert.equal(fs.existsSync(path.join(registeredPackage, 'dist')), false);
|
||||
assert.equal(fs.existsSync(path.join(registeredPackage, 'source.ts')), true);
|
||||
assert.equal(
|
||||
fs.existsSync(path.join(registeredPackage, 'src/nested/example.ts')),
|
||||
true,
|
||||
);
|
||||
for (const suffix of ['js', 'js.map', 'd.ts', 'd.ts.map']) {
|
||||
assert.equal(
|
||||
fs.existsSync(
|
||||
path.join(registeredPackage, `src/nested/example.${suffix}`),
|
||||
),
|
||||
false,
|
||||
);
|
||||
}
|
||||
assert.equal(
|
||||
fs.existsSync(path.join(registeredPackage, 'src/nested/intentional.js')),
|
||||
true,
|
||||
);
|
||||
assert.equal(fs.existsSync(path.join(unregisteredPackage, 'dist')), true);
|
||||
assert.equal(fs.existsSync(path.join(legacyPackage, 'dist')), true);
|
||||
});
|
||||
@@ -0,0 +1,158 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
auditCloudNativePgBackup,
|
||||
} = require('../../scripts/ql3-cloudnativepg-backup-audit.cjs');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '../..');
|
||||
|
||||
function intercept(relativePath, transform) {
|
||||
const target = path.join(ROOT, relativePath);
|
||||
return (filePath, encoding) => {
|
||||
const source = fs.readFileSync(filePath, encoding);
|
||||
return path.resolve(filePath) === target ? transform(source) : source;
|
||||
};
|
||||
}
|
||||
|
||||
test('accepts isolated CNPG-I WAL, backup and restore contracts', () => {
|
||||
const report = auditCloudNativePgBackup({ root: ROOT });
|
||||
assert.equal(report.compatible, true, JSON.stringify(report.findings));
|
||||
assert.equal(report.plugin, 'barman-cloud.cloudnative-pg.io');
|
||||
assert.equal(report.sourceCluster, 'ql3-postgres');
|
||||
assert.equal(report.restoreCluster, 'ql3-postgres-restore');
|
||||
assert.equal(report.retentionPolicy, '30d');
|
||||
});
|
||||
|
||||
test('rejects deprecated in-tree backup or multiple WAL authorities', () => {
|
||||
const report = auditCloudNativePgBackup({
|
||||
root: ROOT,
|
||||
readFile: intercept(
|
||||
'deploy/kubernetes/ql3-cluster/components/barman-cloud-backup/cluster-plugin-patch.yaml',
|
||||
(source) =>
|
||||
`${source.replace(
|
||||
'name: barman-cloud.cloudnative-pg.io',
|
||||
'name: unreviewed.example',
|
||||
)}\n backup:\n retentionPolicy: 30d\n`,
|
||||
),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_CNPG_WAL_ARCHIVER',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects primary-only or implicit backup methods', () => {
|
||||
const report = auditCloudNativePgBackup({
|
||||
root: ROOT,
|
||||
readFile: intercept(
|
||||
'deploy/kubernetes/ql3-cluster/components/barman-cloud-backup/scheduled-backup.yaml',
|
||||
(source) =>
|
||||
source
|
||||
.replace('target: prefer-standby', 'target: primary')
|
||||
.replace('method: plugin', 'method: barmanObjectStore'),
|
||||
),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_CNPG_BASE_BACKUP_SCHEDULE',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects plaintext endpoints, embedded credentials or missing retention', () => {
|
||||
const report = auditCloudNativePgBackup({
|
||||
root: ROOT,
|
||||
readFile: intercept(
|
||||
'deploy/kubernetes/ql3-cluster/components/barman-cloud-backup/object-store.s3.example.yaml',
|
||||
(source) =>
|
||||
source
|
||||
.replace('retentionPolicy: 30d', 'retentionPolicy: 1d')
|
||||
.replace('https://REPLACE_WITH_', 'http://REPLACE_WITH_')
|
||||
.replace(
|
||||
' configuration:',
|
||||
' stringData:\n password: embedded\n configuration:',
|
||||
),
|
||||
),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) =>
|
||||
candidate.code === 'QL3_CNPG_OBJECT_STORE_CONTRACT' ||
|
||||
candidate.code === 'QL3_CNPG_OBJECT_STORE_SECRET_BOUNDARY',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects in-place recovery or empty-WAL-archive bypass', () => {
|
||||
const report = auditCloudNativePgBackup({
|
||||
root: ROOT,
|
||||
readFile: intercept(
|
||||
'deploy/kubernetes/ql3-cluster/operations/cloudnative-pg-restore/restore-cluster.yaml',
|
||||
(source) =>
|
||||
source
|
||||
.replace('name: ql3-postgres-restore', 'name: ql3-postgres')
|
||||
.replace(
|
||||
' labels:',
|
||||
' annotations:\n cnpg.io/skipEmptyWalArchiveCheck: enabled\n labels:',
|
||||
),
|
||||
),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_CNPG_RESTORE_ISOLATION',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects source ObjectStore reuse as restored-cluster WAL destination', () => {
|
||||
const report = auditCloudNativePgBackup({
|
||||
root: ROOT,
|
||||
readFile: intercept(
|
||||
'deploy/kubernetes/ql3-cluster/operations/cloudnative-pg-restore/restore-cluster.yaml',
|
||||
(source) =>
|
||||
source.replace(
|
||||
' postgresql:',
|
||||
' plugins:\n - name: barman-cloud.cloudnative-pg.io\n isWALArchiver: true\n parameters:\n barmanObjectName: ql3-postgres-recovery-source\n postgresql:',
|
||||
),
|
||||
),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_CNPG_RESTORE_ISOLATION',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects legacy serverName authority inside the recovery ObjectStore', () => {
|
||||
const report = auditCloudNativePgBackup({
|
||||
root: ROOT,
|
||||
readFile: intercept(
|
||||
'deploy/kubernetes/ql3-cluster/operations/cloudnative-pg-restore/object-store.s3.example.yaml',
|
||||
(source) =>
|
||||
source.replace(
|
||||
' s3Credentials:',
|
||||
' serverName: ql3-postgres\n s3Credentials:',
|
||||
),
|
||||
),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_CNPG_RECOVERY_SOURCE',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,928 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const crypto = require('node:crypto');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
assertCloudNativePgDrRunnerCapacity,
|
||||
auditedCloudNativePgDrEvidence,
|
||||
backupRuntimeEvidence,
|
||||
certificateRotationEvidence,
|
||||
digestOnlyReference,
|
||||
imageIdDigest,
|
||||
minioFixtureResources,
|
||||
parseEvidenceReportPath,
|
||||
platformDigestFromImageIndex,
|
||||
preflightPrivateEvidenceReportPath,
|
||||
privateDockerDataBindArgs,
|
||||
postgresArchiverEvidence,
|
||||
postgresClusterRuntimeEvidence,
|
||||
postgresDatabaseContractEvidence,
|
||||
postgresMigrationJobResource,
|
||||
postgresMarkerEvidence,
|
||||
postgresRestoreFixtureResources,
|
||||
postgresRestoreApplicationProbeResources,
|
||||
postgresRestoreApplicationRuntimeEvidence,
|
||||
postgresRoleSecretResources,
|
||||
postgresSourceFixtureResources,
|
||||
postgresQueryJson,
|
||||
postgresSql,
|
||||
redactRuntimeText,
|
||||
replaceExactlyOnce,
|
||||
restoreMarkerEvidence,
|
||||
reviewedManifest,
|
||||
webhookConfigurationHasCaBundle,
|
||||
writePrivateEvidenceReport,
|
||||
} = require('../../scripts/ql3-cloudnativepg-barman-live-contract.cjs');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '../..');
|
||||
const POSTGRES_ROLES = [
|
||||
'ql3_admin',
|
||||
'ql3_ai_credential_manager',
|
||||
'ql3_ai_credential_tester',
|
||||
'ql3_ai_maintenance',
|
||||
'ql3_approval_manager',
|
||||
'ql3_automation_manager',
|
||||
'ql3_migration',
|
||||
'ql3_package_executor',
|
||||
'ql3_package_manager',
|
||||
'ql3_runtime',
|
||||
'ql3_worker_credential_executor',
|
||||
'ql3_worker_credential_manager',
|
||||
'ql3_worker_ingress',
|
||||
];
|
||||
|
||||
test('rejects an undersized DR runner before creating temporary cluster state', () => {
|
||||
const gibibyte = 1024n * 1024n * 1024n;
|
||||
assert.deepEqual(
|
||||
assertCloudNativePgDrRunnerCapacity(() => ({
|
||||
bavail: 35n,
|
||||
bsize: gibibyte,
|
||||
})),
|
||||
{
|
||||
minimumBytes: 35n * gibibyte,
|
||||
availableBytes: 35n * gibibyte,
|
||||
},
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
assertCloudNativePgDrRunnerCapacity(() => ({
|
||||
bavail: 34n,
|
||||
bsize: gibibyte,
|
||||
})),
|
||||
/requires at least 35 GiB free; found 36507222016 bytes/,
|
||||
);
|
||||
});
|
||||
|
||||
test('extracts only an exact terminal Kubernetes platform image digest', () => {
|
||||
const digest = `sha256:${'a'.repeat(64)}`;
|
||||
assert.equal(imageIdDigest(`registry.example/image@${digest}`), digest);
|
||||
assert.throws(() => imageIdDigest('registry.example/image:latest'));
|
||||
assert.throws(() => imageIdDigest(`${digest}-suffix`));
|
||||
});
|
||||
|
||||
test('removes a tag without confusing a registry port before Skopeo copy', () => {
|
||||
const digest = `sha256:${'d'.repeat(64)}`;
|
||||
assert.equal(
|
||||
digestOnlyReference(`registry.example:5443/team/image:v1@${digest}`),
|
||||
`registry.example:5443/team/image@${digest}`,
|
||||
);
|
||||
assert.equal(
|
||||
digestOnlyReference(`registry.example:5443/team/image@${digest}`),
|
||||
`registry.example:5443/team/image@${digest}`,
|
||||
);
|
||||
assert.throws(() => digestOnlyReference('registry.example/team/image:v1'));
|
||||
});
|
||||
|
||||
test('redacts every runtime secret occurrence from failure diagnostics', () => {
|
||||
assert.equal(
|
||||
redactRuntimeText('token=secret; repeated=secret', ['secret']),
|
||||
'token=[REDACTED]; repeated=[REDACTED]',
|
||||
);
|
||||
assert.equal(redactRuntimeText('safe', ['', undefined]), 'safe');
|
||||
});
|
||||
|
||||
test('resolves exactly one reviewed platform child from an OCI image index', () => {
|
||||
const amd64 = `sha256:${'a'.repeat(64)}`;
|
||||
const arm64 = `sha256:${'b'.repeat(64)}`;
|
||||
const index = {
|
||||
manifests: [
|
||||
{ digest: amd64, platform: { os: 'linux', architecture: 'amd64' } },
|
||||
{ digest: arm64, platform: { os: 'linux', architecture: 'arm64' } },
|
||||
{
|
||||
digest: `sha256:${'c'.repeat(64)}`,
|
||||
platform: { os: 'unknown', architecture: 'unknown' },
|
||||
},
|
||||
],
|
||||
};
|
||||
assert.equal(platformDigestFromImageIndex(index, 'arm64'), arm64);
|
||||
assert.throws(() => platformDigestFromImageIndex(index, 's390x'));
|
||||
assert.throws(() =>
|
||||
platformDigestFromImageIndex(
|
||||
{ manifests: [...index.manifests, index.manifests[1]] },
|
||||
'arm64',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('rewrites one reviewed release reference and rejects ambiguity', () => {
|
||||
assert.equal(
|
||||
replaceExactlyOnce(
|
||||
'image: product:v1',
|
||||
'product:v1',
|
||||
'product@sha256:locked',
|
||||
),
|
||||
'image: product@sha256:locked',
|
||||
);
|
||||
assert.throws(() =>
|
||||
replaceExactlyOnce(
|
||||
'product:v1 product:v1',
|
||||
'product:v1',
|
||||
'product@sha256:locked',
|
||||
),
|
||||
);
|
||||
assert.throws(() =>
|
||||
replaceExactlyOnce(
|
||||
'image: product:v2',
|
||||
'product:v1',
|
||||
'product@sha256:locked',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts only a checksum-bound regular manifest before pinning images', () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-barman-unit-'));
|
||||
const source = path.join(directory, 'source.yaml');
|
||||
const target = path.join(directory, 'pinned.yaml');
|
||||
try {
|
||||
const manifest = `${'x'.repeat(1024)}\nimage: product:v1\n`;
|
||||
fs.writeFileSync(source, manifest, { mode: 0o600, flag: 'wx' });
|
||||
const digest = crypto.createHash('sha256').update(manifest).digest('hex');
|
||||
const result = reviewedManifest(source, target, digest, [
|
||||
['product:v1', `product:v1@sha256:${'b'.repeat(64)}`],
|
||||
]);
|
||||
assert.equal(result.sourceSha256, digest);
|
||||
assert.match(fs.readFileSync(target, 'utf8'), /product:v1@sha256:b{64}/);
|
||||
|
||||
assert.throws(() =>
|
||||
reviewedManifest(
|
||||
source,
|
||||
path.join(directory, 'rejected.yaml'),
|
||||
'0'.repeat(64),
|
||||
[],
|
||||
),
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('publishes a private evidence report atomically without overwriting history', () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-dr-report-'));
|
||||
const target = path.join(directory, 'report.json');
|
||||
const report = { schemaVersion: 1, fixture: 'test/evidence@v1' };
|
||||
try {
|
||||
assert.equal(
|
||||
parseEvidenceReportPath([`--report=${target}`]),
|
||||
path.normalize(target),
|
||||
);
|
||||
assert.equal(parseEvidenceReportPath([]), undefined);
|
||||
assert.throws(() => parseEvidenceReportPath(['--report=relative.json']));
|
||||
assert.throws(() =>
|
||||
parseEvidenceReportPath([`--report=${target}`, '--unexpected']),
|
||||
);
|
||||
assert.equal(
|
||||
preflightPrivateEvidenceReportPath(target),
|
||||
path.join(fs.realpathSync(directory), 'report.json'),
|
||||
);
|
||||
const published = writePrivateEvidenceReport(target, report);
|
||||
assert.equal(published.path, fs.realpathSync(target));
|
||||
assert.match(published.sha256, /^sha256:[a-f0-9]{64}$/);
|
||||
assert.deepEqual(JSON.parse(fs.readFileSync(target, 'utf8')), report);
|
||||
assert.throws(
|
||||
() => preflightPrivateEvidenceReportPath(target),
|
||||
/refusing to overwrite/,
|
||||
);
|
||||
assert.equal(fs.statSync(target).mode & 0o077, 0);
|
||||
assert.equal(
|
||||
fs.readdirSync(directory).filter((name) => name.endsWith('.tmp')).length,
|
||||
0,
|
||||
);
|
||||
assert.throws(
|
||||
() => writePrivateEvidenceReport(target, { replaced: true }),
|
||||
{
|
||||
code: 'EEXIST',
|
||||
},
|
||||
);
|
||||
assert.deepEqual(JSON.parse(fs.readFileSync(target, 'utf8')), report);
|
||||
} finally {
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('binds every image-declared data directory to private ephemeral storage', () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-dr-data-'));
|
||||
try {
|
||||
const targets = [
|
||||
'/var/lib/cni',
|
||||
'/var/lib/kubelet',
|
||||
'/var/lib/rancher/k3s',
|
||||
'/var/log',
|
||||
];
|
||||
const args = privateDockerDataBindArgs(directory, 'ql3-test-node', targets);
|
||||
assert.equal(args.length, targets.length * 2);
|
||||
for (let index = 0; index < targets.length; index += 1) {
|
||||
assert.equal(args[index * 2], '--mount');
|
||||
const expectedDirectory = path.join(
|
||||
directory,
|
||||
'ql3-test-node',
|
||||
`${String(index).padStart(2, '0')}-${path.basename(targets[index])}`,
|
||||
);
|
||||
assert.equal(
|
||||
args[index * 2 + 1],
|
||||
`type=bind,src=${expectedDirectory},dst=${targets[index]}`,
|
||||
);
|
||||
assert.equal(fs.statSync(expectedDirectory).mode & 0o777, 0o700);
|
||||
}
|
||||
const registryArgs = privateDockerDataBindArgs(
|
||||
directory,
|
||||
'ql3-test-registry',
|
||||
['/var/lib/registry'],
|
||||
);
|
||||
assert.match(
|
||||
registryArgs[1],
|
||||
/type=bind,src=.*\/ql3-test-registry\/00-registry,dst=\/var\/lib\/registry$/,
|
||||
);
|
||||
assert.throws(() =>
|
||||
privateDockerDataBindArgs('relative', 'ql3-test-node', targets),
|
||||
);
|
||||
assert.throws(() =>
|
||||
privateDockerDataBindArgs(directory, '../escape', targets),
|
||||
);
|
||||
assert.throws(() =>
|
||||
privateDockerDataBindArgs(directory, 'ql3-test-node', [
|
||||
'/var/lib/unreviewed',
|
||||
]),
|
||||
);
|
||||
assert.throws(() =>
|
||||
privateDockerDataBindArgs(directory, 'ql3-test-node', [
|
||||
'/var/log',
|
||||
'/var/log',
|
||||
]),
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('pins an explicitly reviewed repeated manifest reference count', () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-barman-count-'));
|
||||
const source = path.join(directory, 'source.yaml');
|
||||
const target = path.join(directory, 'pinned.yaml');
|
||||
try {
|
||||
const manifest = `${'x'.repeat(
|
||||
1024,
|
||||
)}\nimage: product:v1\nenv: product:v1\n`;
|
||||
fs.writeFileSync(source, manifest, { mode: 0o600, flag: 'wx' });
|
||||
const digest = crypto.createHash('sha256').update(manifest).digest('hex');
|
||||
reviewedManifest(source, target, digest, [
|
||||
['product:v1', `product:v1@sha256:${'c'.repeat(64)}`, 2],
|
||||
]);
|
||||
assert.equal(
|
||||
fs.readFileSync(target, 'utf8').match(/product:v1@sha256:c{64}/g)?.length,
|
||||
2,
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('accepts cert-manager webhook readiness only after every CA bundle exists', () => {
|
||||
const ready = {
|
||||
webhooks: [
|
||||
{ clientConfig: { caBundle: Buffer.from('ca-one').toString('base64') } },
|
||||
{ clientConfig: { caBundle: Buffer.from('ca-two').toString('base64') } },
|
||||
],
|
||||
};
|
||||
assert.equal(webhookConfigurationHasCaBundle(ready), true);
|
||||
assert.equal(
|
||||
webhookConfigurationHasCaBundle({
|
||||
webhooks: [ready.webhooks[0], { clientConfig: {} }],
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assert.equal(webhookConfigurationHasCaBundle({ webhooks: [] }), false);
|
||||
});
|
||||
|
||||
test('builds a TLS object store fixture with separate writer and read-only recovery authority', () => {
|
||||
const digest = `sha256:${'e'.repeat(64)}`;
|
||||
const credentials = {
|
||||
root: { accessKey: 'QL3ROOTTEST', secretKey: 'root-secret-value' },
|
||||
writer: { accessKey: 'QL3WRITERTEST', secretKey: 'writer-secret-value' },
|
||||
recovery: {
|
||||
accessKey: 'QL3RECOVERYTEST',
|
||||
secretKey: 'recovery-secret-value',
|
||||
},
|
||||
};
|
||||
const fixture = minioFixtureResources({
|
||||
minioImage: `registry:5000/ql3/minio@${digest}`,
|
||||
clientImage: `registry:5000/ql3/minio-client@${digest}`,
|
||||
credentials,
|
||||
});
|
||||
const serialized = JSON.stringify(fixture);
|
||||
assert.doesNotMatch(serialized, /root-secret-value|writer-secret-value/);
|
||||
assert.doesNotMatch(serialized, /recovery-secret-value/);
|
||||
|
||||
const byKindAndName = (kind, name) =>
|
||||
fixture.core.items.find(
|
||||
(item) => item.kind === kind && item.metadata?.name === name,
|
||||
);
|
||||
const writerSecret = byKindAndName('Secret', 'ql3-object-store-writer');
|
||||
const recoverySecret = byKindAndName('Secret', 'ql3-object-store-recovery');
|
||||
assert.notDeepEqual(writerSecret.data, recoverySecret.data);
|
||||
|
||||
const writerStore = byKindAndName('ObjectStore', 'ql3-postgres-backup');
|
||||
const recoveryStore = byKindAndName(
|
||||
'ObjectStore',
|
||||
'ql3-postgres-recovery-source',
|
||||
);
|
||||
assert.equal(writerStore.spec.retentionPolicy, '30d');
|
||||
assert.match(writerStore.spec.configuration.endpointURL, /^https:\/\//);
|
||||
assert.equal(writerStore.spec.configuration.endpointCA.name, 'ql3-minio-ca');
|
||||
assert.equal(
|
||||
writerStore.spec.configuration.s3Credentials.accessKeyId.name,
|
||||
'ql3-object-store-writer',
|
||||
);
|
||||
assert.equal(
|
||||
recoveryStore.spec.configuration.s3Credentials.accessKeyId.name,
|
||||
'ql3-object-store-recovery',
|
||||
);
|
||||
assert.equal(recoveryStore.spec.configuration.serverName, undefined);
|
||||
|
||||
const deployment = byKindAndName('Deployment', 'ql3-minio');
|
||||
assert.deepEqual(
|
||||
deployment.spec.template.spec.containers[0].resources.limits,
|
||||
{ cpu: '500m', memory: '512Mi' },
|
||||
);
|
||||
assert.equal(
|
||||
byKindAndName('Certificate', 'ql3-minio-server').spec.renewBefore,
|
||||
'1h',
|
||||
);
|
||||
assert.doesNotMatch(serialized, /renewalBefore/);
|
||||
const bootstrap = byKindAndName('Job', 'ql3-minio-bootstrap');
|
||||
const bootstrapScript = bootstrap.spec.template.spec.containers[0].command[2];
|
||||
assert.match(bootstrapScript, /mb --with-lock/);
|
||||
assert.match(bootstrapScript, /retention set --default governance 30d/);
|
||||
assert.match(bootstrapScript, /ilm rule add --expire-days 45/);
|
||||
const verifierScript =
|
||||
fixture.verifier.items[0].spec.template.spec.containers[0].command[2];
|
||||
assert.match(verifierScript, /if mc cp/);
|
||||
assert.match(verifierScript, /if mc rm/);
|
||||
});
|
||||
|
||||
test('builds a constrained three-instance source cluster with one durable WAL authority', () => {
|
||||
const digest = `sha256:${'f'.repeat(64)}`;
|
||||
const fixture = postgresSourceFixtureResources({
|
||||
postgresImage: `registry:5000/ql3/postgresql@${digest}`,
|
||||
});
|
||||
const { cluster, backup } = fixture;
|
||||
assert.equal(cluster.spec.instances, 3);
|
||||
assert.equal(cluster.spec.enableSuperuserAccess, false);
|
||||
assert.equal(cluster.spec.imagePullPolicy, 'IfNotPresent');
|
||||
assert.equal(cluster.spec.plugins.length, 1);
|
||||
assert.deepEqual(cluster.spec.plugins[0], {
|
||||
name: 'barman-cloud.cloudnative-pg.io',
|
||||
isWALArchiver: true,
|
||||
parameters: { barmanObjectName: 'ql3-postgres-backup' },
|
||||
});
|
||||
assert.equal(cluster.spec.backup, undefined);
|
||||
assert.equal(
|
||||
cluster.spec.postgresql.parameters.synchronous_commit,
|
||||
'remote_apply',
|
||||
);
|
||||
assert.deepEqual(cluster.spec.postgresql.synchronous, {
|
||||
method: 'any',
|
||||
number: 1,
|
||||
dataDurability: 'required',
|
||||
failoverQuorum: true,
|
||||
});
|
||||
assert.deepEqual(cluster.spec.affinity, {
|
||||
enablePodAntiAffinity: true,
|
||||
podAntiAffinityType: 'required',
|
||||
topologyKey: 'kubernetes.io/hostname',
|
||||
});
|
||||
assert.deepEqual(cluster.spec.resources.limits, {
|
||||
cpu: '1',
|
||||
memory: '512Mi',
|
||||
});
|
||||
assert.equal(cluster.spec.storage.size, '1Gi');
|
||||
assert.equal(cluster.spec.walStorage.size, '512Mi');
|
||||
assert.equal(backup.spec.method, 'plugin');
|
||||
assert.equal(backup.spec.target, 'prefer-standby');
|
||||
assert.equal(
|
||||
backup.spec.pluginConfiguration.name,
|
||||
'barman-cloud.cloudnative-pg.io',
|
||||
);
|
||||
const serialized = JSON.stringify(fixture);
|
||||
assert.doesNotMatch(serialized, /"kind":"Secret"/);
|
||||
assert.doesNotMatch(serialized, /"password":|secretKeyRef|secretAccessKey/);
|
||||
assert.throws(() =>
|
||||
postgresSourceFixtureResources({ postgresImage: 'postgres:latest' }),
|
||||
);
|
||||
});
|
||||
|
||||
test('builds all production DatabaseRole credentials without plaintext serialization', () => {
|
||||
const credentials = Object.fromEntries(
|
||||
POSTGRES_ROLES.map((role, index) => [
|
||||
role,
|
||||
`${String(index).padStart(2, '0')}${'A'.repeat(32)}`,
|
||||
]),
|
||||
);
|
||||
const resources = postgresRoleSecretResources(credentials);
|
||||
assert.equal(resources.kind, 'List');
|
||||
assert.equal(resources.items.length, POSTGRES_ROLES.length);
|
||||
assert.deepEqual(
|
||||
resources.items.map((secret) =>
|
||||
Buffer.from(secret.data.username, 'base64').toString('utf8'),
|
||||
),
|
||||
POSTGRES_ROLES,
|
||||
);
|
||||
assert.ok(
|
||||
resources.items.every(
|
||||
(secret) =>
|
||||
secret.metadata.namespace === 'ql3-dr' &&
|
||||
secret.type === 'kubernetes.io/basic-auth',
|
||||
),
|
||||
);
|
||||
const serialized = JSON.stringify(resources);
|
||||
for (const password of Object.values(credentials)) {
|
||||
assert.doesNotMatch(serialized, new RegExp(password));
|
||||
}
|
||||
assert.throws(() =>
|
||||
postgresRoleSecretResources({ ...credentials, unexpected: 'A'.repeat(32) }),
|
||||
);
|
||||
});
|
||||
|
||||
test('builds a digest-bound non-root migration Job using the production CLI', () => {
|
||||
const digest = `sha256:${'7'.repeat(64)}`;
|
||||
const image = `ql3-barman-dr-123-abcdef-registry:5000/ql3/cluster-control@${digest}`;
|
||||
const job = postgresMigrationJobResource({ controlImage: image });
|
||||
const pod = job.spec.template.spec;
|
||||
const container = pod.containers[0];
|
||||
assert.equal(job.metadata.namespace, 'ql3-dr');
|
||||
assert.equal(job.spec.backoffLimit, 0);
|
||||
assert.equal(pod.automountServiceAccountToken, false);
|
||||
assert.equal(pod.securityContext.runAsNonRoot, true);
|
||||
assert.equal(container.image, image);
|
||||
assert.equal(container.imagePullPolicy, 'IfNotPresent');
|
||||
assert.deepEqual(container.command, [
|
||||
'node',
|
||||
'/opt/qinglong/node_modules/@qinglong/cluster-postgres/dist/migration/migrationCli.js',
|
||||
]);
|
||||
assert.equal(container.securityContext.readOnlyRootFilesystem, true);
|
||||
assert.deepEqual(container.securityContext.capabilities.drop, ['ALL']);
|
||||
assert.equal(
|
||||
container.env.find(({ name }) => name === 'QL3_POSTGRES_TLS_MODE').value,
|
||||
'verify-full',
|
||||
);
|
||||
assert.equal(
|
||||
container.env.find(({ name }) => name === 'QL3_POSTGRES_MIGRATION_PASSWORD')
|
||||
.valueFrom.secretKeyRef.name,
|
||||
'ql3-postgres-migration-auth',
|
||||
);
|
||||
assert.doesNotMatch(JSON.stringify(job), /ql3_migration_test|postgres:\/\//);
|
||||
assert.throws(() =>
|
||||
postgresMigrationJobResource({ controlImage: 'cluster-control:latest' }),
|
||||
);
|
||||
});
|
||||
|
||||
test('builds a production application readiness probe for each isolated restore', () => {
|
||||
const digest = `sha256:${'6'.repeat(64)}`;
|
||||
const image = `ql3-barman-dr-123-abcdef-registry:5000/ql3/cluster-control@${digest}`;
|
||||
const pepper = Buffer.alloc(32, 5).toString('base64url');
|
||||
const resources = postgresRestoreApplicationProbeResources({
|
||||
clusterName: 'ql3-postgres-restore-latest',
|
||||
controlImage: image,
|
||||
apiCredentialPepper: pepper,
|
||||
});
|
||||
const secret = resources.items.find(({ kind }) => kind === 'Secret');
|
||||
const deployment = resources.items.find(({ kind }) => kind === 'Deployment');
|
||||
const pod = deployment.spec.template.spec;
|
||||
const container = pod.containers[0];
|
||||
assert.equal(secret.metadata.name, 'ql3-dr-application-latest-security');
|
||||
assert.doesNotMatch(JSON.stringify(resources), new RegExp(pepper));
|
||||
assert.equal(deployment.spec.replicas, 1);
|
||||
assert.equal(pod.automountServiceAccountToken, false);
|
||||
assert.equal(container.image, image);
|
||||
assert.equal(container.securityContext.readOnlyRootFilesystem, true);
|
||||
assert.equal(
|
||||
container.env.find(({ name }) => name === 'QL3_WORKER_INGRESS_ENABLED')
|
||||
.value,
|
||||
'false',
|
||||
);
|
||||
assert.equal(
|
||||
container.env.find(({ name }) => name === 'QL3_POSTGRES_RUNTIME_HOST')
|
||||
.value,
|
||||
'ql3-postgres-restore-latest-rw.ql3-dr.svc',
|
||||
);
|
||||
assert.equal(
|
||||
pod.volumes.find(({ name }) => name === 'postgres-ca').secret.secretName,
|
||||
'ql3-postgres-restore-latest-ca',
|
||||
);
|
||||
assert.equal(container.readinessProbe.httpGet.path, '/readyz');
|
||||
assert.throws(() =>
|
||||
postgresRestoreApplicationProbeResources({
|
||||
clusterName: 'ql3-postgres',
|
||||
controlImage: image,
|
||||
apiCredentialPepper: pepper,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
test('requires the restored production application Pod to pass its real readiness probe', () => {
|
||||
const clusterName = 'ql3-postgres-restore-pitr';
|
||||
const name = 'ql3-dr-application-pitr';
|
||||
const deployment = {
|
||||
metadata: { name },
|
||||
spec: { replicas: 1 },
|
||||
status: { availableReplicas: 1, readyReplicas: 1 },
|
||||
};
|
||||
const pod = {
|
||||
metadata: {
|
||||
name: `${name}-abc`,
|
||||
labels: {
|
||||
'app.kubernetes.io/name': name,
|
||||
'ql3.cloud/restore-cluster': clusterName,
|
||||
},
|
||||
},
|
||||
status: { conditions: [{ type: 'Ready', status: 'True' }] },
|
||||
};
|
||||
assert.equal(
|
||||
postgresRestoreApplicationRuntimeEvidence(deployment, [pod], clusterName)
|
||||
.ready,
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
postgresRestoreApplicationRuntimeEvidence(
|
||||
{ ...deployment, status: { availableReplicas: 0, readyReplicas: 0 } },
|
||||
[pod],
|
||||
clusterName,
|
||||
).ready,
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts only the complete production schema owner and non-elevated role catalog', () => {
|
||||
const roles = POSTGRES_ROLES.map((name) => ({
|
||||
name,
|
||||
login: true,
|
||||
superuser: false,
|
||||
createdb: false,
|
||||
createrole: false,
|
||||
replication: false,
|
||||
bypassrls: false,
|
||||
}));
|
||||
const input = {
|
||||
migrationCount: '52',
|
||||
controlCoreCapability: '51',
|
||||
databaseOwner: 'ql3_migration',
|
||||
postgresVersionNumber: '180004',
|
||||
roles,
|
||||
};
|
||||
const evidence = postgresDatabaseContractEvidence(input);
|
||||
assert.equal(evidence.migrationCount, 52);
|
||||
assert.equal(evidence.controlCoreCapability, 51);
|
||||
assert.equal(evidence.databaseOwner, 'ql3_migration');
|
||||
assert.equal(evidence.postgresVersionNumber, 180004);
|
||||
assert.deepEqual(
|
||||
evidence.roles.map(({ name }) => name),
|
||||
POSTGRES_ROLES,
|
||||
);
|
||||
assert.equal(Object.hasOwn(evidence.roles[0], 'login'), false);
|
||||
assert.throws(() =>
|
||||
postgresDatabaseContractEvidence({ ...input, migrationCount: 51 }),
|
||||
);
|
||||
assert.throws(() =>
|
||||
postgresDatabaseContractEvidence({
|
||||
...input,
|
||||
roles: roles.map((role, index) =>
|
||||
index === 0 ? { ...role, superuser: true } : role,
|
||||
),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
test('builds isolated latest and PITR restores from one read-only source authority', () => {
|
||||
const digest = `sha256:${'9'.repeat(64)}`;
|
||||
const postgresImage = `registry:5000/ql3/postgresql@${digest}`;
|
||||
const latest = postgresRestoreFixtureResources({
|
||||
postgresImage,
|
||||
clusterName: 'ql3-postgres-restore-latest',
|
||||
});
|
||||
const targetTime = '2026-08-04T00:00:00.123Z';
|
||||
const pitr = postgresRestoreFixtureResources({
|
||||
postgresImage,
|
||||
clusterName: 'ql3-postgres-restore-pitr',
|
||||
targetTime,
|
||||
});
|
||||
for (const restore of [latest, pitr]) {
|
||||
assert.equal(restore.spec.instances, 3);
|
||||
assert.equal(restore.spec.enableSuperuserAccess, false);
|
||||
assert.equal(restore.spec.plugins, undefined);
|
||||
assert.equal(
|
||||
restore.spec.postgresql.parameters.synchronous_commit,
|
||||
'remote_apply',
|
||||
);
|
||||
assert.equal(restore.spec.postgresql.synchronous.number, 1);
|
||||
assert.equal(restore.spec.affinity.podAntiAffinityType, 'required');
|
||||
assert.equal(restore.spec.externalClusters.length, 1);
|
||||
const plugin = restore.spec.externalClusters[0].plugin;
|
||||
assert.equal(plugin.name, 'barman-cloud.cloudnative-pg.io');
|
||||
assert.deepEqual(plugin.parameters, {
|
||||
barmanObjectName: 'ql3-postgres-recovery-source',
|
||||
serverName: 'ql3-postgres',
|
||||
});
|
||||
}
|
||||
assert.deepEqual(latest.spec.bootstrap.recovery, {
|
||||
source: 'ql3-postgres-origin',
|
||||
});
|
||||
assert.deepEqual(pitr.spec.bootstrap.recovery, {
|
||||
source: 'ql3-postgres-origin',
|
||||
recoveryTarget: { targetTime },
|
||||
});
|
||||
assert.throws(() =>
|
||||
postgresRestoreFixtureResources({
|
||||
postgresImage,
|
||||
clusterName: 'ql3-postgres',
|
||||
}),
|
||||
);
|
||||
assert.throws(() =>
|
||||
postgresRestoreFixtureResources({
|
||||
postgresImage,
|
||||
clusterName: 'ql3-postgres-restore-pitr',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts only a three-node ready source cluster with a live primary', () => {
|
||||
const pod = (ordinal, node, ready = true) => ({
|
||||
metadata: {
|
||||
name: `ql3-postgres-${ordinal}`,
|
||||
labels: { 'cnpg.io/cluster': 'ql3-postgres' },
|
||||
},
|
||||
spec: { nodeName: node },
|
||||
status: {
|
||||
conditions: [{ type: 'Ready', status: ready ? 'True' : 'False' }],
|
||||
},
|
||||
});
|
||||
const cluster = {
|
||||
metadata: { name: 'ql3-postgres' },
|
||||
spec: { instances: 3 },
|
||||
status: { currentPrimary: 'ql3-postgres-1', readyInstances: 3 },
|
||||
};
|
||||
const pods = [pod(1, 'node-a'), pod(2, 'node-b'), pod(3, 'node-c')];
|
||||
const ready = postgresClusterRuntimeEvidence(cluster, pods);
|
||||
assert.equal(ready.ready, true);
|
||||
assert.deepEqual(ready.value.nodes, ['node-a', 'node-b', 'node-c']);
|
||||
assert.equal(
|
||||
postgresClusterRuntimeEvidence(cluster, [
|
||||
pods[0],
|
||||
pods[1],
|
||||
pod(3, 'node-b'),
|
||||
]).ready,
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
postgresClusterRuntimeEvidence(
|
||||
{ ...cluster, status: { ...cluster.status, readyInstances: 2 } },
|
||||
pods,
|
||||
).ready,
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts only a completed plugin backup with bounded WAL evidence', () => {
|
||||
const completed = {
|
||||
status: {
|
||||
method: 'plugin',
|
||||
phase: 'completed',
|
||||
instanceID: { podName: 'ql3-postgres-2' },
|
||||
backupId: '20260804T010203',
|
||||
beginWal: '00000001000000000000000A',
|
||||
endWal: '00000001000000000000000B',
|
||||
startedAt: '2026-08-04T01:02:03Z',
|
||||
stoppedAt: '2026-08-04T01:02:05Z',
|
||||
},
|
||||
};
|
||||
const result = backupRuntimeEvidence(completed);
|
||||
assert.equal(result.ready, true);
|
||||
assert.equal(result.value.method, 'plugin');
|
||||
assert.equal(
|
||||
backupRuntimeEvidence({
|
||||
status: { ...completed.status, method: 'barmanObjectStore' },
|
||||
}).ready,
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
backupRuntimeEvidence({
|
||||
status: { ...completed.status, phase: 'failed' },
|
||||
}).ready,
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts certificate rotation only after serial, Secret and revision advance', () => {
|
||||
const previous = {
|
||||
serialSha256: `sha256:${'1'.repeat(64)}`,
|
||||
resourceVersion: '10',
|
||||
};
|
||||
const current = {
|
||||
serialSha256: `sha256:${'2'.repeat(64)}`,
|
||||
resourceVersion: '12',
|
||||
};
|
||||
assert.deepEqual(certificateRotationEvidence(previous, current, 1, 2), {
|
||||
previousSerialSha256: previous.serialSha256,
|
||||
currentSerialSha256: current.serialSha256,
|
||||
previousSecretResourceVersion: '10',
|
||||
currentSecretResourceVersion: '12',
|
||||
});
|
||||
assert.throws(() => certificateRotationEvidence(previous, previous, 1, 2));
|
||||
assert.throws(() => certificateRotationEvidence(previous, current, 2, 2));
|
||||
});
|
||||
|
||||
test('accepts WAL archiving only after a successful archived segment', () => {
|
||||
assert.deepEqual(
|
||||
postgresArchiverEvidence({
|
||||
archivedCount: '2',
|
||||
failedCount: '0',
|
||||
lastArchivedWal: '00000001000000000000000C',
|
||||
lastArchivedTime: '2026-08-04T01:02:06Z',
|
||||
}),
|
||||
{
|
||||
archivedCount: 2,
|
||||
failedCount: 0,
|
||||
lastArchivedWal: '00000001000000000000000C',
|
||||
lastArchivedTime: '2026-08-04T01:02:06Z',
|
||||
},
|
||||
);
|
||||
assert.throws(() =>
|
||||
postgresArchiverEvidence({
|
||||
archivedCount: '0',
|
||||
failedCount: '0',
|
||||
lastArchivedWal: '',
|
||||
lastArchivedTime: '',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
test('executes PostgreSQL evidence queries without a network credential', () => {
|
||||
const calls = [];
|
||||
const kubectl = (args, options) => {
|
||||
calls.push({ args, options });
|
||||
return { stdout: '{"ok":true}' };
|
||||
};
|
||||
assert.deepEqual(
|
||||
postgresQueryJson(
|
||||
kubectl,
|
||||
'ql3-dr',
|
||||
'ql3-postgres-1',
|
||||
`SELECT '{"ok":true}'`,
|
||||
),
|
||||
{ ok: true },
|
||||
);
|
||||
assert.equal(
|
||||
postgresSql(kubectl, 'ql3-dr', 'ql3-postgres-1', 'SELECT 1'),
|
||||
'{"ok":true}',
|
||||
);
|
||||
assert.equal(calls.length, 2);
|
||||
assert.ok(calls[0].args.includes('--username'));
|
||||
assert.ok(calls[0].args.includes('postgres'));
|
||||
assert.equal(calls[0].args.includes('--password'), false);
|
||||
assert.deepEqual(calls[0].options, { capture: true, quiet: true });
|
||||
assert.throws(() =>
|
||||
postgresSql(kubectl, 'ql3-dr', 'unexpected-pod', 'SELECT 1'),
|
||||
);
|
||||
});
|
||||
|
||||
test('distinguishes latest and PITR marker boundaries exactly', () => {
|
||||
assert.deepEqual(
|
||||
restoreMarkerEvidence(
|
||||
{ beforeMarkerPresent: true, afterMarkerPresent: true },
|
||||
true,
|
||||
),
|
||||
{ beforeMarkerPresent: true, afterMarkerPresent: true },
|
||||
);
|
||||
assert.deepEqual(
|
||||
restoreMarkerEvidence(
|
||||
{ beforeMarkerPresent: true, afterMarkerPresent: false },
|
||||
false,
|
||||
),
|
||||
{ beforeMarkerPresent: true, afterMarkerPresent: false },
|
||||
);
|
||||
assert.throws(() =>
|
||||
restoreMarkerEvidence(
|
||||
{ beforeMarkerPresent: false, afterMarkerPresent: false },
|
||||
false,
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts only content-free UUID, timestamp and WAL marker evidence', () => {
|
||||
assert.deepEqual(
|
||||
postgresMarkerEvidence({
|
||||
id: '123e4567-e89b-42d3-a456-426614174001',
|
||||
createdAt: '2026-08-04T01:02:03.123456Z',
|
||||
wal: '00000001000000000000000D',
|
||||
}),
|
||||
{
|
||||
id: '123e4567-e89b-42d3-a456-426614174001',
|
||||
createdAt: '2026-08-04T01:02:03.123456Z',
|
||||
wal: '00000001000000000000000D',
|
||||
},
|
||||
);
|
||||
assert.throws(() =>
|
||||
postgresMarkerEvidence({
|
||||
id: 'before-base-backup',
|
||||
createdAt: '2026-08-04T01:02:03Z',
|
||||
wal: 'not-wal',
|
||||
}),
|
||||
);
|
||||
assert.throws(() => auditedCloudNativePgDrEvidence({}));
|
||||
});
|
||||
|
||||
test('keeps the destructive live path opt-in and isolated by prefix', () => {
|
||||
const source = fs.readFileSync(
|
||||
path.join(ROOT, 'scripts/ql3-cloudnativepg-barman-live-contract.cjs'),
|
||||
'utf8',
|
||||
);
|
||||
assert.match(source, /QL3_CLOUDNATIVEPG_BARMAN_LIVE !== '1'/);
|
||||
assert.ok(
|
||||
source.indexOf(' assertCloudNativePgDrRunnerCapacity();') <
|
||||
source.indexOf(' const temporary = fs.mkdtempSync('),
|
||||
);
|
||||
assert.match(source, /ql3-barman-dr-/);
|
||||
assert.doesNotMatch(source, /ql3-cnpg-evidence-control-plane/);
|
||||
assert.doesNotMatch(source, /apiservice\/v1\.webhook\.cert-manager\.io/);
|
||||
assert.match(
|
||||
source,
|
||||
/imagePullPolicy: Always'[\s\S]{0,80}imagePullPolicy: IfNotPresent'/,
|
||||
);
|
||||
assert.match(source, /registry:2@sha256:[a-f0-9]{64}/);
|
||||
assert.match(source, /skopeo\/stable:v1\.20\.0@sha256:[a-f0-9]{64}/);
|
||||
assert.match(source, /Docker-Content-Digest|inspection\.Digest/);
|
||||
assert.doesNotMatch(source, /imagePullPolicy: Never/);
|
||||
assert.doesNotMatch(source, /'images',\s*'import'/);
|
||||
assert.doesNotMatch(source, /reviewed-images\.tar/);
|
||||
assert.match(source, /run\(docker, \['rm', '-f', '-v', container\], \{/);
|
||||
assert.doesNotMatch(source, /run\(docker, \['rm', '-f', container\], \{/);
|
||||
assert.match(source, /REGISTRY_DATA_TARGETS = Object\.freeze/);
|
||||
assert.match(source, /K3S_DATA_TARGETS = Object\.freeze/);
|
||||
assert.match(
|
||||
source,
|
||||
/io\.qinglong\.ql3\.live=cloudnativepg-barman-disaster-recovery/,
|
||||
);
|
||||
assert.match(source, /io\.qinglong\.ql3\.run/);
|
||||
assert.match(source, /\['network', 'create', \.\.\.dockerLabels, network\]/);
|
||||
assert.match(
|
||||
source,
|
||||
/privateDockerDataBindArgs\(\s*dockerDataRoot,\s*registry,\s*REGISTRY_DATA_TARGETS/,
|
||||
);
|
||||
assert.match(
|
||||
source,
|
||||
/privateDockerDataBindArgs\(dockerDataRoot, node, K3S_DATA_TARGETS\)/,
|
||||
);
|
||||
assert.match(source, /pg_stat_archiver/);
|
||||
assert.match(source, /SELECT pg_switch_wal\(\)/);
|
||||
assert.match(source, /backupRuntimeEvidence/);
|
||||
assert.match(source, /plugin-barman-cloud/);
|
||||
assert.match(source, /prefer-standby backup unexpectedly ran on the primary/);
|
||||
assert.match(source, /privateKey: \{ rotationPolicy: 'Always' \}/);
|
||||
assert.match(source, /Barman mutual TLS certificate rotation/);
|
||||
assert.match(source, /post-rotation plugin base backup completion/);
|
||||
assert.match(source, /database-roles\.yaml/);
|
||||
assert.match(
|
||||
source,
|
||||
/@qinglong\/cluster-postgres\/dist\/migration\/migrationCli\.js/,
|
||||
);
|
||||
assert.match(source, /"event":"migration_completed"/);
|
||||
assert.match(source, /ALTER DATABASE qinglong OWNER TO ql3_migration/);
|
||||
assert.match(source, /postgresDatabaseContractEvidence/);
|
||||
assert.match(source, /production application readiness/);
|
||||
assert.match(source, /latestApplicationRtoSeconds/);
|
||||
assert.match(source, /pitrApplicationRtoSeconds/);
|
||||
assert.match(source, /ql3-postgres-restore-latest/);
|
||||
assert.match(source, /ql3-postgres-restore-pitr/);
|
||||
assert.match(source, /sourceClusterAfterRestores\.metadata\.uid/);
|
||||
assert.match(source, /cloudnativepg-disaster-recovery@v1/);
|
||||
assert.match(source, /auditedCloudNativePgDrEvidence/);
|
||||
assert.match(source, /schemaAndRoles: true/);
|
||||
assert.equal(
|
||||
source.match(/`eviction-hard=\$\{K3S_EVICTION_HARD\}`/g)?.length,
|
||||
2,
|
||||
);
|
||||
assert.match(
|
||||
source,
|
||||
/memory\.available<100Mi,nodefs\.available<64Mi,imagefs\.available<64Mi,nodefs\.inodesFree<1%/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '../..');
|
||||
const WORKFLOW = fs.readFileSync(
|
||||
path.join(ROOT, '.github/workflows/ql3-cloudnativepg-dr-live.yml'),
|
||||
'utf8',
|
||||
);
|
||||
const RUNNER = fs.readFileSync(
|
||||
path.join(ROOT, 'scripts/ql3-cloudnativepg-barman-live-contract.cjs'),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
test('keeps the expensive CloudNativePG DR evidence gate manual and exact', () => {
|
||||
assert.match(WORKFLOW, /^on:\n workflow_dispatch:\s*$/m);
|
||||
assert.doesNotMatch(WORKFLOW, /^ (?:push|pull_request|schedule):/m);
|
||||
assert.match(WORKFLOW, /runs-on: ubuntu-24\.04/);
|
||||
assert.match(WORKFLOW, /timeout-minutes: 120/);
|
||||
assert.match(WORKFLOW, /at least 35 GiB free/);
|
||||
assert.match(WORKFLOW, /node-version: '24\.18\.0'/);
|
||||
assert.match(WORKFLOW, /version: '8\.3\.1'/);
|
||||
assert.match(WORKFLOW, /kubectl v1\.32\.8/);
|
||||
assert.match(
|
||||
WORKFLOW,
|
||||
/cert-manager\/cert-manager\/releases\/download\/v1\.20\.3/,
|
||||
);
|
||||
assert.match(WORKFLOW, /plugin-barman-cloud\/releases\/download\/v0\.13\.0/);
|
||||
assert.match(WORKFLOW, /cloudnative-pg\/releases\/download\/v1\.30\.0/);
|
||||
assert.match(WORKFLOW, /QL3_CLOUDNATIVEPG_BARMAN_LIVE: '1'/);
|
||||
assert.match(WORKFLOW, /QL3_SOURCE_REVISION: \$\{\{ github\.sha \}\}/);
|
||||
assert.match(WORKFLOW, /"--report=\$\{QL3_DR_REPORT\}"/);
|
||||
assert.match(WORKFLOW, /stat -c '%a'.*= '600'/);
|
||||
assert.match(WORKFLOW, /audit:cloudnativepg-dr-evidence:ql3/);
|
||||
assert.match(
|
||||
WORKFLOW,
|
||||
/test\/back\/ql3CloudNativePgBarmanWorkflow\.test\.cjs/,
|
||||
);
|
||||
assert.match(
|
||||
WORKFLOW,
|
||||
/actions\/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a/,
|
||||
);
|
||||
assert.match(WORKFLOW, /if-no-files-found: error/);
|
||||
assert.match(WORKFLOW, /retention-days: 14/);
|
||||
assert.match(WORKFLOW, /overwrite: false/);
|
||||
});
|
||||
|
||||
test('fails closed on runner resource leaks without pruning shared Docker state', () => {
|
||||
assert.match(WORKFLOW, /ql3-dangling-volumes\.before/);
|
||||
assert.match(WORKFLOW, /ql3-dangling-volumes\.after/);
|
||||
assert.match(WORKFLOW, /diff --unified/);
|
||||
assert.match(WORKFLOW, /docker ps -aq --filter name=ql3-barman-dr-/);
|
||||
assert.match(WORKFLOW, /docker network ls -q --filter name=ql3-barman-dr-/);
|
||||
assert.match(
|
||||
WORKFLOW,
|
||||
/docker ps -aq --filter label=io\.qinglong\.ql3\.live=cloudnativepg-barman-disaster-recovery/,
|
||||
);
|
||||
assert.match(
|
||||
WORKFLOW,
|
||||
/docker network ls -q --filter label=io\.qinglong\.ql3\.live=cloudnativepg-barman-disaster-recovery/,
|
||||
);
|
||||
assert.doesNotMatch(WORKFLOW, /(?:system|builder|volume) prune/);
|
||||
assert.doesNotMatch(WORKFLOW, /continue-on-error/);
|
||||
assert.doesNotMatch(WORKFLOW, /ql3-cnpg-evidence-control-plane/);
|
||||
assert.match(
|
||||
RUNNER,
|
||||
/run\(docker, \['rm', '-f', '-v', container\], \{[\s\S]*?allowFailure: true/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,197 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
auditCloudNativePgDeployment,
|
||||
} = require('../../scripts/ql3-cloudnativepg-deployment-audit.cjs');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '../..');
|
||||
|
||||
function intercept(relativePath, transform) {
|
||||
const target = path.join(ROOT, relativePath);
|
||||
return (filePath, encoding) => {
|
||||
const source = fs.readFileSync(filePath, encoding);
|
||||
return path.resolve(filePath) === target ? transform(source) : source;
|
||||
};
|
||||
}
|
||||
|
||||
test('accepts the locked CloudNativePG HA and authority profile', () => {
|
||||
const report = auditCloudNativePgDeployment({ root: ROOT });
|
||||
assert.equal(report.compatible, true, JSON.stringify(report.findings));
|
||||
assert.equal(report.operatorVersion, '1.30.0');
|
||||
assert.equal(report.postgresqlVersion, '18.4');
|
||||
assert.equal(report.instances, 3);
|
||||
assert.deepEqual(report.roles, [
|
||||
'ql3_admin',
|
||||
'ql3_ai_credential_manager',
|
||||
'ql3_ai_credential_tester',
|
||||
'ql3_ai_maintenance',
|
||||
'ql3_approval_manager',
|
||||
'ql3_automation_manager',
|
||||
'ql3_migration',
|
||||
'ql3_package_executor',
|
||||
'ql3_package_manager',
|
||||
'ql3_runtime',
|
||||
'ql3_worker_credential_executor',
|
||||
'ql3_worker_credential_manager',
|
||||
'ql3_worker_ingress',
|
||||
]);
|
||||
});
|
||||
|
||||
test('rejects a missing or rewritten operator release manifest digest', () => {
|
||||
for (const transform of [
|
||||
(source) => source.replace(/\s+"releaseManifestSha256": "[^"]+",/, ''),
|
||||
(source) =>
|
||||
source.replace(
|
||||
/"releaseManifestSha256": "[^"]+"/,
|
||||
`"releaseManifestSha256": "sha256:${'0'.repeat(64)}"`,
|
||||
),
|
||||
]) {
|
||||
const report = auditCloudNativePgDeployment({
|
||||
root: ROOT,
|
||||
readFile: intercept(
|
||||
'deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/operator-lock.json',
|
||||
transform,
|
||||
),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.ok(
|
||||
report.findings.some(({ code }) => code === 'QL3_CNPG_SUPPLY_CHAIN_LOCK'),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects a single instance or unpinned PostgreSQL operand', () => {
|
||||
const report = auditCloudNativePgDeployment({
|
||||
root: ROOT,
|
||||
readFile: intercept(
|
||||
'deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/cluster.yaml',
|
||||
(source) =>
|
||||
source
|
||||
.replace('instances: 3', 'instances: 1')
|
||||
.replace(
|
||||
/imageName: .+/,
|
||||
'imageName: ghcr.io/cloudnative-pg/postgresql:18.4-minimal-trixie',
|
||||
),
|
||||
),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_CNPG_CLUSTER_BASELINE',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a privileged database role', () => {
|
||||
const report = auditCloudNativePgDeployment({
|
||||
root: ROOT,
|
||||
readFile: intercept(
|
||||
'deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/database-roles.yaml',
|
||||
(source) => source.replace('superuser: false', 'superuser: true'),
|
||||
),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_CNPG_DATABASE_ROLE',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects runtime DSN authority or a non-primary endpoint', () => {
|
||||
const report = auditCloudNativePgDeployment({
|
||||
root: ROOT,
|
||||
readFile: intercept(
|
||||
'deploy/kubernetes/ql3-cluster/overlays/cloudnative-pg/postgres-runtime-patch.yaml',
|
||||
(source) =>
|
||||
source
|
||||
.replace('$patch: delete', 'value: postgres://embedded-secret')
|
||||
.replace(
|
||||
'ql3-postgres-rw.qinglong3-system.svc',
|
||||
'ql3-postgres-ro.qinglong3-system.svc',
|
||||
),
|
||||
),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_CNPG_RUNTIME_BINDING',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects migration credentials or CA from the runtime domain', () => {
|
||||
const report = auditCloudNativePgDeployment({
|
||||
root: ROOT,
|
||||
readFile: intercept(
|
||||
'deploy/kubernetes/ql3-cluster/operations/cloudnative-pg/migrate-job-patch.yaml',
|
||||
(source) =>
|
||||
source
|
||||
.replaceAll(
|
||||
'ql3-postgres-migration-auth',
|
||||
'ql3-postgres-runtime-auth',
|
||||
)
|
||||
.replace('value: ql3-postgres-ca', 'value: runtime-ca')
|
||||
.replace('value: ca.crt', 'value: postgres-ca.crt'),
|
||||
),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_CNPG_MIGRATION_BINDING',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a deployable tag or missing image transform in either application path', () => {
|
||||
for (const [relativePath, transform, findingCode] of [
|
||||
[
|
||||
'deploy/kubernetes/ql3-cluster/overlays/cloudnative-pg/kustomization.yaml',
|
||||
(source) =>
|
||||
source.replace(
|
||||
/\s+digest: sha256:[0-9a-f]{64}/,
|
||||
'\n newTag: latest',
|
||||
),
|
||||
'QL3_CNPG_RUNTIME_BINDING',
|
||||
],
|
||||
[
|
||||
'deploy/kubernetes/ql3-cluster/operations/cloudnative-pg/kustomization.yaml',
|
||||
(source) => source.replace(/\nimages:[\s\S]*?(?=\npatches:)/, '\n'),
|
||||
'QL3_CNPG_MIGRATION_BINDING',
|
||||
],
|
||||
]) {
|
||||
const report = auditCloudNativePgDeployment({
|
||||
root: ROOT,
|
||||
readFile: intercept(relativePath, transform),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.ok(report.findings.some(({ code }) => code === findingCode));
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects applying credential examples through kustomize', () => {
|
||||
const report = auditCloudNativePgDeployment({
|
||||
root: ROOT,
|
||||
readFile: intercept(
|
||||
'deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/kustomization.yaml',
|
||||
(source) =>
|
||||
source.replace(
|
||||
' - database.yaml',
|
||||
' - database.yaml\n - credentials.example.yaml',
|
||||
),
|
||||
),
|
||||
});
|
||||
assert.equal(report.compatible, false);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_CNPG_SECRET_APPLICATION_BOUNDARY',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,412 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { execFileSync } = require('node:child_process');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
validateCloudNativePgDrEvidence,
|
||||
validateCloudNativePgDrReleaseEvidence,
|
||||
} = require('../../scripts/ql3-cloudnativepg-dr-evidence-audit.cjs');
|
||||
const {
|
||||
parseArguments: parseReleaseGateArguments,
|
||||
} = require('../../scripts/ql3-cloudnativepg-dr-release-gate.cjs');
|
||||
|
||||
const SOURCE_REVISION = 'a'.repeat(40);
|
||||
|
||||
function hash(character) {
|
||||
return `sha256:${character.repeat(64)}`;
|
||||
}
|
||||
|
||||
function roles() {
|
||||
return [
|
||||
'ql3_admin',
|
||||
'ql3_ai_credential_manager',
|
||||
'ql3_ai_credential_tester',
|
||||
'ql3_ai_maintenance',
|
||||
'ql3_approval_manager',
|
||||
'ql3_automation_manager',
|
||||
'ql3_migration',
|
||||
'ql3_package_executor',
|
||||
'ql3_package_manager',
|
||||
'ql3_runtime',
|
||||
'ql3_worker_credential_executor',
|
||||
'ql3_worker_credential_manager',
|
||||
'ql3_worker_ingress',
|
||||
].map((name) => ({
|
||||
name,
|
||||
superuser: false,
|
||||
createdb: false,
|
||||
createrole: false,
|
||||
replication: false,
|
||||
bypassrls: false,
|
||||
}));
|
||||
}
|
||||
|
||||
function restore(cluster, afterMarkerPresent) {
|
||||
return {
|
||||
cluster,
|
||||
sourceObjectStore: 'ql3-postgres-recovery-source',
|
||||
sourceServerName: 'ql3-postgres',
|
||||
sourceClusterUnmodified: true,
|
||||
targetWalArchiver: false,
|
||||
instances: 3,
|
||||
ready: true,
|
||||
migrationCount: 54,
|
||||
controlCoreCapability: 53,
|
||||
databaseOwner: 'ql3_migration',
|
||||
synchronousCommit: 'remote_apply',
|
||||
synchronousStandbys: 1,
|
||||
roles: roles(),
|
||||
beforeMarkerPresent: true,
|
||||
afterMarkerPresent,
|
||||
};
|
||||
}
|
||||
|
||||
function validReport() {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
fixture: 'qinglong/cloudnativepg-disaster-recovery@v1',
|
||||
observedAt: '2026-07-24T12:30:00.000Z',
|
||||
sourceRevision: SOURCE_REVISION,
|
||||
platform: {
|
||||
kubernetesVersion: '1.32.8',
|
||||
architecture: 'amd64',
|
||||
cloudNativePgVersion: '1.30.0',
|
||||
cloudNativePgImageId: hash('1'),
|
||||
postgresVersionNumber: 180004,
|
||||
postgresImageId: hash('2'),
|
||||
barmanVersion: '0.13.0',
|
||||
barmanControllerImageId:
|
||||
'sha256:417449fe4f6f0a56acdeb30e4131930815f2b46b9afeb808059b57aa8b4c2ef5',
|
||||
barmanSidecarImageIds: [
|
||||
'sha256:15cb1a01e7c5235eedac2061cab8208e5f7c39dbda292f9c2d4ddaa0c1f211e6',
|
||||
'sha256:15cb1a01e7c5235eedac2061cab8208e5f7c39dbda292f9c2d4ddaa0c1f211e6',
|
||||
'sha256:15cb1a01e7c5235eedac2061cab8208e5f7c39dbda292f9c2d4ddaa0c1f211e6',
|
||||
],
|
||||
certManagerVersion: '1.20.3',
|
||||
certManagerImageIds: [
|
||||
'sha256:1e4af57beb469cc3bb0fb48b9201caea2723819b9ffd3c3ea98568f55b4dd38b',
|
||||
'sha256:a2b12d27950d1603d2c8168c3ccd95d07b93ce6ec4b530316196a31db592a9c0',
|
||||
'sha256:953a97df613f7da7eda8ce4b1c8d8e6b50963db0800fab595d040db6eb5cb060',
|
||||
],
|
||||
},
|
||||
source: {
|
||||
cluster: 'ql3-postgres',
|
||||
backup: {
|
||||
name: 'ql3-dr-backup-20260724',
|
||||
phase: 'completed',
|
||||
startedAt: '2026-07-24T12:00:00.000Z',
|
||||
completedAt: '2026-07-24T12:05:00.000Z',
|
||||
beginWal: '000000010000000000000001',
|
||||
endWal: '000000010000000000000002',
|
||||
},
|
||||
markers: {
|
||||
before: {
|
||||
id: '123e4567-e89b-42d3-a456-426614174001',
|
||||
createdAt: '2026-07-24T12:01:00.000Z',
|
||||
wal: '000000010000000000000001',
|
||||
},
|
||||
after: {
|
||||
id: '123e4567-e89b-42d3-a456-426614174002',
|
||||
createdAt: '2026-07-24T12:10:00.000Z',
|
||||
wal: '000000010000000000000003',
|
||||
},
|
||||
},
|
||||
wal: {
|
||||
archiveHealthy: true,
|
||||
continuous: true,
|
||||
noGaps: true,
|
||||
lastArchivedWal: '000000010000000000000004',
|
||||
},
|
||||
},
|
||||
latestRestore: restore('ql3-postgres-restore-latest', true),
|
||||
pitrRestore: {
|
||||
...restore('ql3-postgres-restore-pitr', false),
|
||||
targetTime: '2026-07-24T12:06:00.000Z',
|
||||
},
|
||||
certificateRotation: {
|
||||
client: {
|
||||
previousSerialSha256: hash('7'),
|
||||
currentSerialSha256: hash('8'),
|
||||
previousSecretResourceVersion: '101',
|
||||
currentSecretResourceVersion: '102',
|
||||
},
|
||||
server: {
|
||||
previousSerialSha256: hash('9'),
|
||||
currentSerialSha256: hash('a'),
|
||||
previousSecretResourceVersion: '201',
|
||||
currentSecretResourceVersion: '202',
|
||||
},
|
||||
walArchivedDuringRotation: true,
|
||||
backupCompletedAfterRotation: true,
|
||||
latestRestoreCompletedAfterRotation: true,
|
||||
pitrCompletedAfterRotation: true,
|
||||
maxObservedInterruptionSeconds: 2.4,
|
||||
},
|
||||
objectStoreAuthority: {
|
||||
sourceObjectStore: 'ql3-postgres-backup',
|
||||
recoveryObjectStore: 'ql3-postgres-recovery-source',
|
||||
sourceWriterIdentitySha256: hash('b'),
|
||||
recoveryReaderIdentitySha256: hash('c'),
|
||||
recoveryReadOnly: true,
|
||||
versioning: true,
|
||||
immutability: true,
|
||||
lifecycleDays: 30,
|
||||
},
|
||||
serviceLevels: {
|
||||
targetMaxRpoSeconds: 60,
|
||||
observedRpoSeconds: 5,
|
||||
targetMaxDatabaseRtoSeconds: 1200,
|
||||
latestDatabaseRtoSeconds: 410,
|
||||
pitrDatabaseRtoSeconds: 470,
|
||||
targetMaxApplicationRtoSeconds: 1800,
|
||||
latestApplicationRtoSeconds: 520,
|
||||
pitrApplicationRtoSeconds: 590,
|
||||
},
|
||||
gates: {
|
||||
latestRestore: true,
|
||||
pointInTimeRestore: true,
|
||||
schemaAndRoles: true,
|
||||
sourceIsolation: true,
|
||||
certificateRotation: true,
|
||||
serviceLevels: true,
|
||||
passed: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('accepts complete latest, PITR, rotation and service-level evidence', () => {
|
||||
const report = validateCloudNativePgDrEvidence(validReport());
|
||||
assert.equal(report.compatible, true, JSON.stringify(report.findings));
|
||||
});
|
||||
|
||||
test('accepts fresh disaster-recovery evidence bound to the release source', () => {
|
||||
const report = validateCloudNativePgDrReleaseEvidence(validReport(), {
|
||||
sourceCommit: SOURCE_REVISION,
|
||||
releaseVersion: '3.0.0-rc.1',
|
||||
nowMs: Date.parse('2026-07-24T13:00:00.000Z'),
|
||||
});
|
||||
assert.equal(report.compatible, true, JSON.stringify(report.findings));
|
||||
assert.equal(report.maximumAgeSeconds, 86_400);
|
||||
});
|
||||
|
||||
test('rejects stale or source-detached disaster-recovery release evidence', () => {
|
||||
const report = validateCloudNativePgDrReleaseEvidence(validReport(), {
|
||||
sourceCommit: 'b'.repeat(40),
|
||||
releaseVersion: '3.0.0',
|
||||
nowMs: Date.parse('2026-07-26T12:30:01.000Z'),
|
||||
});
|
||||
assert.equal(
|
||||
report.findings.some(({ code }) => code === 'QL3_DR_RELEASE_SOURCE'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
report.findings.some(({ code }) => code === 'QL3_DR_RELEASE_FRESHNESS'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('release gate accepts only one exact private-report argument set', () => {
|
||||
assert.deepEqual(
|
||||
parseReleaseGateArguments([
|
||||
'--report=/run/qinglong3-release-evidence/a/report.json',
|
||||
`--source-commit=${SOURCE_REVISION}`,
|
||||
'--release-version=3.0.0',
|
||||
]),
|
||||
{
|
||||
reportPath: '/run/qinglong3-release-evidence/a/report.json',
|
||||
sourceCommit: SOURCE_REVISION,
|
||||
releaseVersion: '3.0.0',
|
||||
},
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
parseReleaseGateArguments([
|
||||
'--report=relative.json',
|
||||
`--source-commit=${SOURCE_REVISION}`,
|
||||
'--release-version=3.0.0',
|
||||
]),
|
||||
/must be absolute/,
|
||||
);
|
||||
});
|
||||
|
||||
test('release gate CLI accepts a fresh private regular report', (t) => {
|
||||
const directory = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), 'ql3-dr-release-gate-'),
|
||||
);
|
||||
t.after(() => fs.rmSync(directory, { recursive: true, force: true }));
|
||||
fs.chmodSync(directory, 0o700);
|
||||
|
||||
const report = validReport();
|
||||
report.observedAt = new Date().toISOString();
|
||||
const reportPath = path.join(directory, 'cloudnativepg-dr-evidence.json');
|
||||
fs.writeFileSync(reportPath, `${JSON.stringify(report)}\n`, { mode: 0o600 });
|
||||
|
||||
const output = execFileSync(
|
||||
process.execPath,
|
||||
[
|
||||
path.resolve(
|
||||
__dirname,
|
||||
'../../scripts/ql3-cloudnativepg-dr-release-gate.cjs',
|
||||
),
|
||||
`--report=${reportPath}`,
|
||||
`--source-commit=${SOURCE_REVISION}`,
|
||||
'--release-version=3.0.0-rc.1',
|
||||
],
|
||||
{ encoding: 'utf8' },
|
||||
);
|
||||
assert.equal(JSON.parse(output).compatible, true);
|
||||
|
||||
fs.chmodSync(reportPath, 0o700);
|
||||
assert.throws(
|
||||
() =>
|
||||
execFileSync(
|
||||
process.execPath,
|
||||
[
|
||||
path.resolve(
|
||||
__dirname,
|
||||
'../../scripts/ql3-cloudnativepg-dr-release-gate.cjs',
|
||||
),
|
||||
`--report=${reportPath}`,
|
||||
`--source-commit=${SOURCE_REVISION}`,
|
||||
'--release-version=3.0.0-rc.1',
|
||||
],
|
||||
{ encoding: 'utf8' },
|
||||
),
|
||||
/mode-0600 regular file/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a restore report pinned to the obsolete schema and role set', () => {
|
||||
const input = validReport();
|
||||
input.latestRestore.migrationCount = 17;
|
||||
input.latestRestore.controlCoreCapability = 16;
|
||||
input.latestRestore.roles = input.latestRestore.roles.filter(
|
||||
({ name }) =>
|
||||
!name.startsWith('ql3_worker_credential_') &&
|
||||
name !== 'ql3_automation_manager',
|
||||
);
|
||||
const report = validateCloudNativePgDrEvidence(input);
|
||||
assert.equal(
|
||||
report.findings.some(({ code }) => code === 'QL3_DR_LATEST_RESTORE'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects credential or private material in the report', () => {
|
||||
const input = validReport();
|
||||
input.objectStoreAuthority.password = 'must-not-appear';
|
||||
const report = validateCloudNativePgDrEvidence(input);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_DR_SECRET_EXPOSURE',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects latest restore without both durable markers', () => {
|
||||
const input = validReport();
|
||||
input.latestRestore.afterMarkerPresent = false;
|
||||
const report = validateCloudNativePgDrEvidence(input);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_DR_LATEST_RESTORE',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects PITR outside the marker window or containing the later marker', () => {
|
||||
const input = validReport();
|
||||
input.pitrRestore.targetTime = '2026-07-24T12:11:00.000Z';
|
||||
input.pitrRestore.afterMarkerPresent = true;
|
||||
const report = validateCloudNativePgDrEvidence(input);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_DR_PITR_RESTORE',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects schema, role or restored-cluster write-authority drift', () => {
|
||||
const input = validReport();
|
||||
input.latestRestore.controlCoreCapability = 15;
|
||||
input.latestRestore.roles[0].superuser = true;
|
||||
input.latestRestore.targetWalArchiver = true;
|
||||
const report = validateCloudNativePgDrEvidence(input);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_DR_LATEST_RESTORE',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects shared object-store identities or weakened retention', () => {
|
||||
const input = validReport();
|
||||
input.objectStoreAuthority.recoveryReaderIdentitySha256 =
|
||||
input.objectStoreAuthority.sourceWriterIdentitySha256;
|
||||
input.objectStoreAuthority.lifecycleDays = 7;
|
||||
const report = validateCloudNativePgDrEvidence(input);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_DR_OBJECT_STORE_AUTHORITY',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects certificate rotation without new identities and continued recovery', () => {
|
||||
const input = validReport();
|
||||
input.certificateRotation.client.currentSerialSha256 =
|
||||
input.certificateRotation.client.previousSerialSha256;
|
||||
input.certificateRotation.pitrCompletedAfterRotation = false;
|
||||
const report = validateCloudNativePgDrEvidence(input);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_DR_CERTIFICATE_ROTATION',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects observed RPO or RTO above deployment targets', () => {
|
||||
const input = validReport();
|
||||
input.serviceLevels.observedRpoSeconds = 61;
|
||||
input.serviceLevels.pitrApplicationRtoSeconds = 1801;
|
||||
const report = validateCloudNativePgDrEvidence(input);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_DR_SERVICE_LEVELS',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a summary that hides an independently failed gate', () => {
|
||||
const input = validReport();
|
||||
input.gates.pointInTimeRestore = false;
|
||||
const report = validateCloudNativePgDrEvidence(input);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_DR_GATE_SUMMARY',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects cert-manager image count, order or digest drift', () => {
|
||||
const input = validReport();
|
||||
input.platform.certManagerImageIds[0] = hash('d');
|
||||
input.platform.certManagerImageIds.push(hash('e'));
|
||||
const report = validateCloudNativePgDrEvidence(input);
|
||||
assert.equal(
|
||||
report.findings.some(
|
||||
(candidate) => candidate.code === 'QL3_DR_PLATFORM_PROVENANCE',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,208 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
imageDigest,
|
||||
imageTag,
|
||||
localApplicationManifest,
|
||||
manifestSha256,
|
||||
reviewedOperatorManifest,
|
||||
verifyImageIds,
|
||||
} = require('../../scripts/ql3-cloudnativepg-live-contract.cjs');
|
||||
|
||||
const INDEX = `sha256:${'a'.repeat(64)}`;
|
||||
const PLATFORM = `sha256:${'b'.repeat(64)}`;
|
||||
|
||||
function pods(...imageIds) {
|
||||
return imageIds.map((imageID) => ({
|
||||
status: { containerStatuses: [{ imageID }] },
|
||||
}));
|
||||
}
|
||||
|
||||
test('extracts only one exact digest-pinned image reference', () => {
|
||||
assert.equal(
|
||||
imageDigest(`registry.example/operand:18.4@${INDEX}`),
|
||||
INDEX,
|
||||
);
|
||||
assert.throws(() => imageDigest('registry.example/operand:18.4'));
|
||||
assert.throws(() => imageDigest(`registry.example/operand@${INDEX}:tag`));
|
||||
});
|
||||
|
||||
test('derives a normal tagged preload reference from a reviewed image', () => {
|
||||
assert.equal(
|
||||
imageTag(`registry.example:5000/operand:18.4@${INDEX}`),
|
||||
'registry.example:5000/operand:18.4',
|
||||
);
|
||||
assert.throws(() => imageTag(`registry.example/operand@${INDEX}`));
|
||||
assert.throws(() => imageTag('registry.example/operand:18.4'));
|
||||
});
|
||||
|
||||
test('replaces exactly one fail-closed application image only in live rendering', () => {
|
||||
const placeholder = `registry.example.com/qinglong/qinglong3-cluster-control@sha256:${'0'.repeat(64)}`;
|
||||
const rendered = `kind: Deployment\nspec:\n image: ${placeholder}\n`;
|
||||
assert.equal(
|
||||
localApplicationManifest(rendered),
|
||||
'kind: Deployment\nspec:\n image: registry.example.com/qinglong/qinglong3-cluster-control:3.0.0-alpha.0\n',
|
||||
);
|
||||
assert.throws(() => localApplicationManifest('kind: Deployment\n'));
|
||||
assert.throws(() =>
|
||||
localApplicationManifest(`${rendered}---\n${rendered}`),
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts uniform runtime reporting of the reviewed index or platform digest', () => {
|
||||
assert.deepEqual(
|
||||
verifyImageIds(
|
||||
pods(`registry.example/operand@${INDEX}`),
|
||||
[INDEX, PLATFORM],
|
||||
'operand',
|
||||
),
|
||||
[`registry.example/operand@${INDEX}`],
|
||||
);
|
||||
assert.deepEqual(
|
||||
verifyImageIds(
|
||||
pods(`registry.example/operand@${PLATFORM}`),
|
||||
[INDEX, PLATFORM],
|
||||
'operand',
|
||||
),
|
||||
[`registry.example/operand@${PLATFORM}`],
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects tags, unknown digests, missing status and widened reviewed sets', () => {
|
||||
for (const invoke of [
|
||||
() => verifyImageIds(pods('registry.example/operand:18.4'), [INDEX], 'operand'),
|
||||
() =>
|
||||
verifyImageIds(
|
||||
pods(`registry.example/operand@sha256:${'c'.repeat(64)}`),
|
||||
[INDEX, PLATFORM],
|
||||
'operand',
|
||||
),
|
||||
() => verifyImageIds([{ status: {} }], [INDEX], 'operand'),
|
||||
() => verifyImageIds(pods(`registry.example/operand@${INDEX}`), ['*'], 'operand'),
|
||||
]) {
|
||||
assert.throws(invoke);
|
||||
}
|
||||
});
|
||||
|
||||
test('creates the namespaced control identity before the migration Job', () => {
|
||||
const source = fs.readFileSync(
|
||||
path.resolve(__dirname, '../../scripts/ql3-cloudnativepg-live-contract.cjs'),
|
||||
'utf8',
|
||||
);
|
||||
const namespace = source.indexOf(
|
||||
"'deploy/kubernetes/ql3-cluster/base/namespace.yaml'",
|
||||
);
|
||||
const serviceAccount = source.indexOf(
|
||||
"'deploy/kubernetes/ql3-cluster/base/service-account.yaml'",
|
||||
);
|
||||
const migration = source.indexOf(
|
||||
"'deploy/kubernetes/ql3-cluster/operations/cloudnative-pg'",
|
||||
);
|
||||
assert.ok(namespace >= 0);
|
||||
assert.ok(serviceAccount > namespace);
|
||||
assert.ok(migration > serviceAccount);
|
||||
assert.match(
|
||||
source.slice(namespace, serviceAccount),
|
||||
/kubectl\(\[/,
|
||||
);
|
||||
assert.match(
|
||||
source.slice(namespace, migration),
|
||||
/'-n',\s*NAMESPACE,\s*'apply',\s*'-f',\s*'deploy\/kubernetes\/ql3-cluster\/base\/service-account\.yaml'/,
|
||||
);
|
||||
});
|
||||
|
||||
test('provisions the fail-closed worker ingress identity and derives all role evidence from one set', () => {
|
||||
const source = fs.readFileSync(
|
||||
path.resolve(__dirname, '../../scripts/ql3-cloudnativepg-live-contract.cjs'),
|
||||
'utf8',
|
||||
);
|
||||
assert.match(source, /const roleList = ROLE_NAMES\.map/);
|
||||
assert.match(source, /WHERE rolname IN \(\$\{roleList\}\)/);
|
||||
assert.match(source, /assert\.deepEqual\(schema, \['53', '52'\]\)/);
|
||||
assert.match(source, /migrationCount: 54/);
|
||||
assert.match(source, /contractVersion: 53/);
|
||||
assert.match(source, /createWorkerIngressTls\(tempDirectory\)/);
|
||||
for (const key of [
|
||||
'worker-credential-pepper',
|
||||
'artifact-s3-bucket',
|
||||
'artifact-s3-region',
|
||||
'artifact-s3-encryption',
|
||||
'tls.key',
|
||||
'tls.crt',
|
||||
'client-ca.crt',
|
||||
]) {
|
||||
assert.ok(source.includes(`'${key}'`));
|
||||
}
|
||||
assert.match(source, /basicConstraints=critical,CA:TRUE/);
|
||||
assert.match(source, /extendedKeyUsage=serverAuth/);
|
||||
assert.match(source, /subjectAltName=DNS:ql3-cluster-control/);
|
||||
});
|
||||
|
||||
test('preloads both lock-owned images before applying the operator manifest', () => {
|
||||
const source = fs.readFileSync(
|
||||
path.resolve(__dirname, '../../scripts/ql3-cloudnativepg-live-contract.cjs'),
|
||||
'utf8',
|
||||
);
|
||||
const preload = source.indexOf(
|
||||
'for (const reviewedImage of [OPERATOR_IMAGE, POSTGRES_IMAGE])',
|
||||
);
|
||||
const manifest = source.indexOf(
|
||||
"'download official CloudNativePG 1.30.0 release manifest'",
|
||||
);
|
||||
assert.ok(preload >= 0);
|
||||
assert.ok(manifest > preload);
|
||||
const contract = source.slice(preload, manifest);
|
||||
assert.match(contract, /docker\(\['pull', reviewedImage\]\)/);
|
||||
assert.match(contract, /imageDigest\(reviewedImage\)/);
|
||||
assert.match(
|
||||
contract,
|
||||
/const preloadTag = imageTag\(reviewedImage\)/,
|
||||
);
|
||||
assert.match(
|
||||
contract,
|
||||
/docker\(\['tag', reviewedImage, preloadTag\]\)/,
|
||||
);
|
||||
assert.match(
|
||||
contract,
|
||||
/kind\(\['load', 'docker-image', preloadTag, '--name', clusterName\]\)/,
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
contract,
|
||||
/kind\(\['load', 'docker-image', reviewedImage/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a canonical but checksum-unreviewed operator manifest', () => {
|
||||
const directory = fs.realpathSync(
|
||||
fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-cnpg-manifest-test-')),
|
||||
);
|
||||
const candidate = path.join(directory, 'operator.yaml');
|
||||
try {
|
||||
fs.writeFileSync(candidate, 'x'.repeat(2048), { mode: 0o600 });
|
||||
assert.match(manifestSha256(candidate), /^sha256:[a-f0-9]{64}$/);
|
||||
assert.throws(
|
||||
() => reviewedOperatorManifest(candidate),
|
||||
/reviewed lock digest/,
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('bounds remote manifest retries and removes disposable temporary state', () => {
|
||||
const source = fs.readFileSync(
|
||||
path.resolve(__dirname, '../../scripts/ql3-cloudnativepg-live-contract.cjs'),
|
||||
'utf8',
|
||||
);
|
||||
assert.match(source, /'--http1\.1'/);
|
||||
assert.match(source, /'--retry-max-time',\s*'300'/);
|
||||
assert.match(source, /reviewedOperatorManifest\(downloadedOperatorManifest\)/);
|
||||
assert.match(
|
||||
source,
|
||||
/fs\.rmSync\(tempDirectory, \{ recursive: true, force: true \}\)/,
|
||||
);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,575 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const test = require('node:test');
|
||||
const {
|
||||
auditClusterImageCiWorkflow,
|
||||
auditClusterImageRelease,
|
||||
auditReleaseWorkflow,
|
||||
} = require('../../scripts/ql3-cluster-image-release-audit.cjs');
|
||||
|
||||
const root = path.resolve(__dirname, '../..');
|
||||
const ciSource = fs.readFileSync(
|
||||
path.join(root, '.github/workflows/ql3-ci.yml'),
|
||||
'utf8',
|
||||
);
|
||||
const releaseSource = fs.readFileSync(
|
||||
path.join(root, '.github/workflows/ql3-image-release.yml'),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
test('accepts the reviewed native CI and digest release contracts', () => {
|
||||
assert.deepEqual(auditClusterImageRelease(root), {
|
||||
ci: {
|
||||
images: ['control', 'control-ai', 'admin', 'local'],
|
||||
nativeArchitectures: ['amd64', 'arm64'],
|
||||
runtimeInventory: true,
|
||||
ociAttestations: true,
|
||||
osVulnerabilityScan: {
|
||||
scanner: 'trivy@0.70.0',
|
||||
severities: ['HIGH', 'CRITICAL'],
|
||||
packageTypes: ['os'],
|
||||
ignoreUnfixed: false,
|
||||
},
|
||||
},
|
||||
release: {
|
||||
trigger: 'explicit protected v3 tag dispatch',
|
||||
workerManagementEvidence: {
|
||||
sourceAware: true,
|
||||
privateEphemeralRunner: true,
|
||||
maximumAgeSeconds: 86400,
|
||||
artifactUpload: false,
|
||||
},
|
||||
cloudNativePgDisasterRecoveryEvidence: {
|
||||
sourceAware: true,
|
||||
privateEphemeralRunner: true,
|
||||
maximumAgeSeconds: 86400,
|
||||
artifactUpload: false,
|
||||
staticLocksReaudited: true,
|
||||
},
|
||||
osVulnerabilityScan: {
|
||||
scanner: 'trivy@0.70.0',
|
||||
actionCommit: 'ed142fd0673e97e23eac54620cfb913e5ce36c25',
|
||||
platforms: ['linux/amd64', 'linux/arm64'],
|
||||
severities: ['HIGH', 'CRITICAL'],
|
||||
packageTypes: ['os'],
|
||||
ignoreUnfixed: false,
|
||||
maximumExceptionDays: 30,
|
||||
buildOnce: true,
|
||||
immutableArtifactRetentionDays: 1,
|
||||
attestedToPublishedDigest: true,
|
||||
},
|
||||
images: ['control', 'control-ai', 'admin', 'local'],
|
||||
platforms: ['linux/amd64', 'linux/arm64'],
|
||||
keylessSignature: true,
|
||||
buildkitAttestations: ['sbom', 'provenance'],
|
||||
githubAttestations: ['provenance', 'sbom', 'os-vulnerability'],
|
||||
publication: {
|
||||
copier: 'regctl@0.11.5',
|
||||
copierSha256:
|
||||
'c93aa7638749f5aaac1a8e01787321889c78f0101809bb2880343478d0ba0467',
|
||||
rebuildAfterScan: false,
|
||||
tagAfterVerification: true,
|
||||
},
|
||||
localRolloutPreflight: true,
|
||||
localRolloutApply: true,
|
||||
postPublishVerification: [
|
||||
'manifest',
|
||||
'cosign',
|
||||
'provenance',
|
||||
'cyclonedx',
|
||||
'os-vulnerability',
|
||||
'release-tags',
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test('rejects removal of the native arm64 image gate', () => {
|
||||
const mutated = ciSource.replace(
|
||||
'runner: ubuntu-24.04-arm\n node_arch: arm64\n image_arch: arm64\n image: control',
|
||||
'runner: none\n node_arch: arm64\n image_arch: arm64\n image: control',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditClusterImageCiWorkflow(mutated),
|
||||
/matrices must contain only exact/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects removal of the native cluster-admin image gate', () => {
|
||||
const mutated = ciSource.replace(
|
||||
'image_arch: arm64\n image: admin',
|
||||
'image_arch: arm64\n image: disabled',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditClusterImageCiWorkflow(mutated),
|
||||
/matrices must contain only exact/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects an additional unreviewed CI image authority', () => {
|
||||
const mutated = ciSource.replace(
|
||||
' target: runtime\n steps:',
|
||||
' target: runtime\n - runner: ubuntu-24.04\n node_arch: x64\n image_arch: amd64\n image: unreviewed\n dockerfile: unreviewed/Dockerfile\n target: runtime\n steps:',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditClusterImageCiWorkflow(mutated),
|
||||
/matrices must contain only exact/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects removal of the attested OCI evidence job', () => {
|
||||
const mutated = ciSource.replace(' image-oci:', ' image-oci-disabled:');
|
||||
assert.throws(
|
||||
() => auditClusterImageCiWorkflow(mutated),
|
||||
/matrices must contain only exact/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a movable Trivy action in native image CI', () => {
|
||||
const mutated = ciSource.replace(
|
||||
'aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0',
|
||||
'aquasecurity/trivy-action@v0.36.0',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditClusterImageCiWorkflow(mutated),
|
||||
/exact pinned OS-only Trivy failure gate/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects native image CI that hides unfixed OS vulnerabilities', () => {
|
||||
const mutated = ciSource.replace(
|
||||
" ignore-unfixed: 'false'",
|
||||
" ignore-unfixed: 'true'",
|
||||
);
|
||||
assert.throws(
|
||||
() => auditClusterImageCiWorkflow(mutated),
|
||||
/exact pinned OS-only Trivy failure gate/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a release missing an architecture', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'--arm64-layout=${RUNNER_TEMP}/native/arm64/layout',
|
||||
'--arm64-layout=${RUNNER_TEMP}/native/amd64/layout',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/merge and re-audit the two scanned layouts/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects an automatic tag push that bypasses private evidence review', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'on:\n workflow_dispatch:',
|
||||
"on:\n push:\n tags:\n - 'v3.*'\n workflow_dispatch:",
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/explicit protected-tag dispatch/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects release publication without the private evidence dependency', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
' - worker-management-release-evidence',
|
||||
' - worker-management-release-evidence-disabled',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/depend on both protected ephemeral private evidence jobs/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects release publication without the OS vulnerability dependency', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
' - os-vulnerability',
|
||||
' - os-vulnerability-disabled',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/depend on both protected ephemeral private evidence jobs/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects release publication without current disaster-recovery evidence', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
' - cluster-dr-release-evidence',
|
||||
' - cluster-dr-release-evidence-disabled',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/depend on both protected ephemeral private evidence jobs/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects disaster-recovery evidence detached from the release source', () => {
|
||||
const marker = '--source-commit="${GITHUB_SHA}"';
|
||||
const first = releaseSource.indexOf(marker);
|
||||
const second = releaseSource.indexOf(marker, first + marker.length);
|
||||
assert.notEqual(first, -1);
|
||||
assert.notEqual(second, -1);
|
||||
const mutated = `${releaseSource.slice(
|
||||
0,
|
||||
second,
|
||||
)}--source-commit="detached"${releaseSource.slice(second + marker.length)}`;
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/bind the exact report to the release identity/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects an incomplete native OS vulnerability architecture matrix', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
' - image: local\n runner: ubuntu-24.04-arm\n node_arch: arm64\n image_arch: arm64\n dockerfile: deploy/containers/ql3-local-application/Dockerfile',
|
||||
' - image: local\n runner: ubuntu-24.04-arm\n node_arch: arm64\n image_arch: disabled\n dockerfile: deploy/containers/ql3-local-application/Dockerfile',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/OS vulnerability matrix must scan exact/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a movable Trivy action after the upstream supply-chain incident', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0',
|
||||
'aquasecurity/trivy-action@v0.36.0',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/build once, scan that exact OCI layout/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects hiding unfixed high or critical OS vulnerabilities', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
" ignore-unfixed: 'false'",
|
||||
" ignore-unfixed: 'true'",
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/build once, scan that exact OCI layout/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects widening the OS exception policy to application libraries', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
" vuln-type: 'os'",
|
||||
" vuln-type: 'os,library'",
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/build once, scan that exact OCI layout/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects persistent scanner cache in the privileged release workflow', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
" cache: 'false'",
|
||||
" cache: 'true'",
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/build once, scan that exact OCI layout/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a reusable private evidence runner', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'ql3-release-evidence-ephemeral',
|
||||
'ql3-release-evidence-persistent',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/protected ephemeral private evidence job/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects private evidence uploaded as a workflow artifact', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
' - name: Re-audit commit-scoped private production evidence',
|
||||
' - uses: actions/upload-artifact@v4\n - name: Re-audit commit-scoped private production evidence',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/only the reviewed|never persist private evidence/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a private evidence path not scoped to the release commit', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'/run/qinglong3-release-evidence/${GITHUB_SHA}',
|
||||
'/run/qinglong3-release-evidence/current',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/commit-scoped runner mount/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects write authority in the private evidence job', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
' timeout-minutes: 10\n permissions:\n contents: read',
|
||||
' timeout-minutes: 10\n permissions:\n packages: write',
|
||||
);
|
||||
assert.throws(() => auditReleaseWorkflow(mutated), /keep evidence read-only/);
|
||||
});
|
||||
|
||||
test('rejects a release missing the standalone digest signature', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'cosign sign --yes "${IMAGE}@${DIGEST}"',
|
||||
'cosign version',
|
||||
);
|
||||
assert.throws(() => auditReleaseWorkflow(mutated), /keylessly sign/);
|
||||
});
|
||||
|
||||
test('rejects a release missing application SBOM attestation', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'sbom-path: ${{ runner.temp }}/${{ matrix.repository }}.cdx.json',
|
||||
'show-summary: true',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/reviewed application SBOM/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a release missing the independent admin image', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'- image: admin\n repository: qinglong3-cluster-admin\n runtime_root: deploy/containers/ql3-cluster-admin/runtime-dependencies',
|
||||
'- image: admin-disabled\n repository: qinglong3-cluster-admin\n runtime_root: deploy/containers/ql3-cluster-admin/runtime-dependencies',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/matrix must contain only exact/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a release missing the AI-excluded local image', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'- image: local\n repository: qinglong3-local-application\n runtime_root: deploy/containers/ql3-local-application/runtime-dependencies',
|
||||
'- image: local-disabled\n repository: qinglong3-local-application\n runtime_root: deploy/containers/ql3-local-application/runtime-dependencies',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/matrix must contain only exact/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects an additional repository in the privileged release matrix', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
' runtime_root: deploy/containers/ql3-local-application/runtime-dependencies\n steps:',
|
||||
' runtime_root: deploy/containers/ql3-local-application/runtime-dependencies\n - image: unreviewed\n repository: unreviewed\n dockerfile: Dockerfile\n runtime_root: unreviewed\n steps:',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/matrix must contain only exact/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects release publication without the vulnerability gate', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
' --audit-level=high',
|
||||
' --audit-level=none',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/high or critical production dependency advisories/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a release that does not select the image-specific SBOM', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'node scripts/ql3-cluster-image-sbom.cjs\n --image=${{ matrix.image }}',
|
||||
'node scripts/ql3-cluster-image-sbom.cjs\n --image=control',
|
||||
);
|
||||
assert.throws(() => auditReleaseWorkflow(mutated), /selected image SBOM/);
|
||||
});
|
||||
|
||||
test('rejects a release with reduced OIDC authority', () => {
|
||||
const mutated = releaseSource.replace('id-token: write', 'id-token: read');
|
||||
assert.throws(() => auditReleaseWorkflow(mutated), /grant writes only/);
|
||||
});
|
||||
|
||||
test('rejects a movable action tag in the privileged release job', () => {
|
||||
const pinned =
|
||||
'actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6';
|
||||
const offset = releaseSource.lastIndexOf(pinned);
|
||||
const mutated = `${releaseSource.slice(
|
||||
0,
|
||||
offset,
|
||||
)}actions/checkout@v6${releaseSource.slice(offset + pinned.length)}`;
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/privileged publisher|immutable checkout action/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects removal of the published manifest audit', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'node scripts/ql3-cluster-remote-manifest-audit.cjs',
|
||||
'node --version',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/published digest manifest/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a local release without both live rollout Profiles', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
' --profile=standalone',
|
||||
' --profile=edge',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/apply and stop both Profiles/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects signature verification without exact certificate identity', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'--certificate-identity "${certificate_identity}"',
|
||||
'--certificate-identity-regexp ".*"',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/exact keyless workflow identity/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a GitHub attestation verification without source binding', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'--source-digest "${GITHUB_SHA}"',
|
||||
'--source-digest "movable"',
|
||||
);
|
||||
assert.throws(() => auditReleaseWorkflow(mutated), /bind the source commit/);
|
||||
});
|
||||
|
||||
test('rejects CycloneDX verification without the exact predicate type', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'--predicate-type "https://cyclonedx.org/bom"',
|
||||
'--predicate-type "https://example.invalid/sbom"',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/CycloneDX predicate type/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects scanning a daemon tag instead of the exact OCI tar', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
' input: ${{ runner.temp }}/ql3-native/image.oci.tar',
|
||||
' image-ref: movable-candidate:latest',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/build once, scan that exact OCI layout tar/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects retaining scanned native release artifacts for more than one day', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
' retention-days: 1',
|
||||
' retention-days: 2',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/upload only bound immutable evidence/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects overwrite authority on a scanned native artifact', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
' overwrite: false',
|
||||
' overwrite: true',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/upload only bound immutable evidence/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a movable upload-artifact action', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1',
|
||||
'actions/upload-artifact@v7.0.1',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/upload only bound immutable evidence/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects downloading an artifact not bound to the same run attempt', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'ql3-release-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.image }}-amd64',
|
||||
'ql3-release-${{ github.run_id }}-${{ matrix.image }}-amd64',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/same-run scanned native artifacts/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects any privileged rebuild after the native scan', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
' node scripts/ql3-image-release-bundle.cjs \\\n --mode=merge',
|
||||
' docker build .\n node scripts/ql3-image-release-bundle.cjs \\\n --mode=merge',
|
||||
);
|
||||
assert.throws(() => auditReleaseWorkflow(mutated), /without any rebuild/);
|
||||
});
|
||||
|
||||
test('rejects a checksum drift in the exact OCI copier', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'c93aa7638749f5aaac1a8e01787321889c78f0101809bb2880343478d0ba0467',
|
||||
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/checksum-pin the exact regctl OCI copier/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects importing the scanned graph through an unverified tag', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'image import "${IMAGE}@${DIGEST}" "${ARCHIVE}"',
|
||||
'image import "${IMAGE}:candidate" "${ARCHIVE}"',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/import the audited OCI graph by digest/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects an image tag created before digest verification completes', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
' "${REGCTL}" image import "${IMAGE}@${DIGEST}" "${ARCHIVE}"',
|
||||
' "${REGCTL}" image copy "${IMAGE}@${DIGEST}" "${IMAGE}:early"\n "${REGCTL}" image import "${IMAGE}@${DIGEST}" "${ARCHIVE}"',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/before final tag promotion/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects any publisher step after immutable tag promotion', () => {
|
||||
const mutated = `${releaseSource}\n - name: Post-promotion mutation\n run: echo unsafe\n`;
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/before final tag promotion/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects removal of the digest-bound OS vulnerability attestation', () => {
|
||||
const mutated = releaseSource.replace(
|
||||
'predicate-type: https://qinglong.dev/attestations/image-os-vulnerability/v1',
|
||||
'predicate-type: https://example.invalid/not-os-evidence',
|
||||
);
|
||||
assert.throws(
|
||||
() => auditReleaseWorkflow(mutated),
|
||||
/digest-bound OS vulnerability evidence/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,247 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const test = require('node:test');
|
||||
const {
|
||||
auditClusterImageSbom,
|
||||
componentRef,
|
||||
createClusterImageSbom,
|
||||
} = require('../../scripts/ql3-cluster-image-sbom.cjs');
|
||||
|
||||
const root = path.resolve(__dirname, '../..');
|
||||
|
||||
function clone(value) {
|
||||
return JSON.parse(JSON.stringify(value));
|
||||
}
|
||||
|
||||
test('generates the exact reviewed cluster image runtime closure', () => {
|
||||
const document = createClusterImageSbom({ root });
|
||||
const report = auditClusterImageSbom(document, { root });
|
||||
|
||||
assert.deepEqual(report, {
|
||||
image: 'control',
|
||||
root: 'pkg:npm/%40qinglong/cluster-control-image-dependencies@3.0.0-alpha.0',
|
||||
components: 46,
|
||||
externalComponents: 43,
|
||||
internalComponents: 3,
|
||||
dependencyNodes: 47,
|
||||
inventoryVerified: false,
|
||||
});
|
||||
assert.equal(
|
||||
document.components.some((component) => component.name === 'pg-protocol'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
document.components.some((component) => component.name === 'typescript'),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('generates the optional Cluster AI image runtime closure', () => {
|
||||
const document = createClusterImageSbom({ root, image: 'control-ai' });
|
||||
const report = auditClusterImageSbom(document, {
|
||||
root,
|
||||
image: 'control-ai',
|
||||
});
|
||||
|
||||
assert.deepEqual(report, {
|
||||
image: 'control-ai',
|
||||
root: 'pkg:npm/%40qinglong/cluster-control-image-dependencies@3.0.0-alpha.0',
|
||||
components: 47,
|
||||
externalComponents: 43,
|
||||
internalComponents: 4,
|
||||
dependencyNodes: 48,
|
||||
inventoryVerified: false,
|
||||
});
|
||||
assert.equal(
|
||||
document.components.some((component) => component.name === '@qinglong/ai'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('generates the independent reviewed cluster-admin image closure', () => {
|
||||
const document = createClusterImageSbom({ root, image: 'admin' });
|
||||
const report = auditClusterImageSbom(document, {
|
||||
root,
|
||||
image: 'admin',
|
||||
});
|
||||
|
||||
assert.deepEqual(report, {
|
||||
image: 'admin',
|
||||
root: 'pkg:npm/%40qinglong/cluster-admin-image-dependencies@3.0.0-alpha.0',
|
||||
components: 88,
|
||||
externalComponents: 84,
|
||||
internalComponents: 4,
|
||||
dependencyNodes: 89,
|
||||
inventoryVerified: false,
|
||||
});
|
||||
assert.equal(
|
||||
document.components.some(
|
||||
(component) => component.name === '@kubernetes/client-node',
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
document.components.some(
|
||||
(component) => component.name === '@aws-sdk/client-s3',
|
||||
),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('generates the AI-excluded local application image closure', () => {
|
||||
const document = createClusterImageSbom({ root, image: 'local' });
|
||||
const report = auditClusterImageSbom(document, {
|
||||
root,
|
||||
image: 'local',
|
||||
});
|
||||
|
||||
assert.deepEqual(report, {
|
||||
image: 'local',
|
||||
root: 'pkg:npm/%40qinglong/local-application-image@3.0.0-alpha.0',
|
||||
components: 10,
|
||||
externalComponents: 2,
|
||||
internalComponents: 8,
|
||||
dependencyNodes: 11,
|
||||
inventoryVerified: false,
|
||||
});
|
||||
assert.deepEqual(
|
||||
document.components
|
||||
.filter((component) =>
|
||||
['@qinglong/ai', 'drizzle-orm', 'typescript'].includes(component.name),
|
||||
)
|
||||
.map((component) => component.name),
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a control SBOM presented as cluster-admin evidence', () => {
|
||||
const document = createClusterImageSbom({ root });
|
||||
assert.throws(
|
||||
() =>
|
||||
auditClusterImageSbom(document, {
|
||||
root,
|
||||
image: 'admin',
|
||||
}),
|
||||
/selected image profile|image manifest/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects widened metadata and root component drift', () => {
|
||||
const widened = createClusterImageSbom({ root });
|
||||
widened.metadata.generatedBy = 'unreviewed';
|
||||
assert.throws(
|
||||
() => auditClusterImageSbom(widened, { root }),
|
||||
/root component/,
|
||||
);
|
||||
|
||||
const drifted = createClusterImageSbom({ root });
|
||||
drifted.metadata.component.purl = 'pkg:npm/unrelated@1.0.0';
|
||||
assert.throws(
|
||||
() => auditClusterImageSbom(drifted, { root }),
|
||||
/root component/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a missing internal dependency edge', () => {
|
||||
const document = createClusterImageSbom({ root });
|
||||
const controlRef = componentRef('@qinglong/cluster-control', '3.0.0-alpha.0');
|
||||
const edge = document.dependencies.find((entry) => entry.ref === controlRef);
|
||||
edge.dependsOn = edge.dependsOn.slice(1);
|
||||
|
||||
assert.throws(
|
||||
() => auditClusterImageSbom(document, { root }),
|
||||
/dependency edges.*reviewed runtime closure/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects an unexpected development component', () => {
|
||||
const document = createClusterImageSbom({ root });
|
||||
document.components.push({
|
||||
type: 'library',
|
||||
'bom-ref': componentRef('typescript', '5.9.3'),
|
||||
name: 'typescript',
|
||||
version: '5.9.3',
|
||||
purl: componentRef('typescript', '5.9.3'),
|
||||
});
|
||||
|
||||
assert.throws(
|
||||
() => auditClusterImageSbom(document, { root }),
|
||||
/development component leaked/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects tampered locked component metadata', () => {
|
||||
const document = createClusterImageSbom({ root });
|
||||
const pg = document.components.find((component) => component.name === 'pg');
|
||||
pg.version = '8.21.0';
|
||||
|
||||
assert.throws(
|
||||
() => auditClusterImageSbom(document, { root }),
|
||||
/component metadata differs/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a missing or unreviewed runtime license', () => {
|
||||
const missing = createClusterImageSbom({ root, image: 'local' });
|
||||
delete missing.components[0].licenses;
|
||||
assert.throws(
|
||||
() => auditClusterImageSbom(missing, { root, image: 'local' }),
|
||||
/unreviewed license/,
|
||||
);
|
||||
|
||||
const unreviewed = createClusterImageSbom({ root, image: 'local' });
|
||||
unreviewed.components[0].licenses = [{ license: { id: 'GPL-3.0-only' } }];
|
||||
assert.throws(
|
||||
() => auditClusterImageSbom(unreviewed, { root, image: 'local' }),
|
||||
/unreviewed license/,
|
||||
);
|
||||
});
|
||||
|
||||
test('verifies a bounded package inventory against the SBOM', (t) => {
|
||||
const document = createClusterImageSbom({ root });
|
||||
const temporaryRoot = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), 'ql3-image-inventory-'),
|
||||
);
|
||||
t.after(() => fs.rmSync(temporaryRoot, { recursive: true, force: true }));
|
||||
|
||||
for (const component of document.components) {
|
||||
const packageDirectory = component.name.startsWith('@')
|
||||
? path.join(temporaryRoot, ...component.name.split('/'))
|
||||
: path.join(temporaryRoot, component.name);
|
||||
fs.mkdirSync(packageDirectory, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(packageDirectory, 'package.json'),
|
||||
JSON.stringify({
|
||||
name: component.name,
|
||||
version: component.version,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
assert.equal(
|
||||
auditClusterImageSbom(document, {
|
||||
root,
|
||||
inventoryRoot: temporaryRoot,
|
||||
}).inventoryVerified,
|
||||
true,
|
||||
);
|
||||
|
||||
const unexpected = path.join(temporaryRoot, 'unexpected');
|
||||
fs.mkdirSync(unexpected);
|
||||
fs.writeFileSync(
|
||||
path.join(unexpected, 'package.json'),
|
||||
JSON.stringify({ name: 'unexpected', version: '1.0.0' }),
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
auditClusterImageSbom(clone(document), {
|
||||
root,
|
||||
inventoryRoot: temporaryRoot,
|
||||
}),
|
||||
/runtime image package inventory differs/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,550 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const crypto = require('node:crypto');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const test = require('node:test');
|
||||
const {
|
||||
auditClusterOciLayout,
|
||||
} = require('../../scripts/ql3-cluster-oci-layout-audit.cjs');
|
||||
const {
|
||||
mergeNativeLayouts,
|
||||
nativeEvidenceRecord,
|
||||
} = require('../../scripts/ql3-image-release-bundle.cjs');
|
||||
const {
|
||||
createClusterImageSbom,
|
||||
} = require('../../scripts/ql3-cluster-image-sbom.cjs');
|
||||
|
||||
const root = path.resolve(__dirname, '../..');
|
||||
const revision = 'fixture-revision';
|
||||
|
||||
function createFixture(t, options = {}) {
|
||||
const image = options.image || 'control';
|
||||
const isControl = image === 'control' || image === 'control-ai';
|
||||
const isControlAi = image === 'control-ai';
|
||||
const isLocal = image === 'local';
|
||||
const layoutRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-oci-layout-'));
|
||||
t.after(() => fs.rmSync(layoutRoot, { recursive: true, force: true }));
|
||||
const blobDirectory = path.join(layoutRoot, 'blobs', 'sha256');
|
||||
fs.mkdirSync(blobDirectory, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(layoutRoot, 'oci-layout'),
|
||||
JSON.stringify({ imageLayoutVersion: '1.0.0' }),
|
||||
);
|
||||
|
||||
function blob(value, mediaType) {
|
||||
const content = Buffer.isBuffer(value)
|
||||
? value
|
||||
: Buffer.from(JSON.stringify(value));
|
||||
const digest = `sha256:${crypto
|
||||
.createHash('sha256')
|
||||
.update(content)
|
||||
.digest('hex')}`;
|
||||
fs.writeFileSync(
|
||||
path.join(blobDirectory, digest.slice('sha256:'.length)),
|
||||
content,
|
||||
);
|
||||
return { mediaType, digest, size: content.length };
|
||||
}
|
||||
|
||||
const expectedComponents = createClusterImageSbom({
|
||||
root,
|
||||
image,
|
||||
}).components;
|
||||
const applicationPackages = expectedComponents.map((component) => ({
|
||||
name: component.name,
|
||||
versionInfo: component.version,
|
||||
sourceInfo: `acquired package info from installed node module manifest file: /opt/qinglong/node_modules/${component.name}/package.json`,
|
||||
externalRefs: [
|
||||
{
|
||||
referenceCategory: 'PACKAGE-MANAGER',
|
||||
referenceType: 'purl',
|
||||
referenceLocator: component.purl,
|
||||
},
|
||||
],
|
||||
}));
|
||||
if (options.addDevelopmentPackage) {
|
||||
applicationPackages.push({
|
||||
name: '@types/node',
|
||||
versionInfo: '24.13.3',
|
||||
sourceInfo:
|
||||
'acquired package info from installed node module manifest file: /opt/qinglong/node_modules/@types/node/package.json',
|
||||
externalRefs: [
|
||||
{
|
||||
referenceCategory: 'PACKAGE-MANAGER',
|
||||
referenceType: 'purl',
|
||||
referenceLocator: 'pkg:npm/%40types/node@24.13.3',
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
const descriptors = [];
|
||||
const imageDescriptors = [];
|
||||
for (const architecture of options.onlyArchitecture
|
||||
? [options.onlyArchitecture]
|
||||
: options.omitArm64
|
||||
? ['amd64']
|
||||
: ['amd64', 'arm64']) {
|
||||
const config = blob(
|
||||
{
|
||||
architecture,
|
||||
os: 'linux',
|
||||
config: {
|
||||
User:
|
||||
options.rootArm64 && architecture === 'arm64'
|
||||
? '0:0'
|
||||
: isLocal
|
||||
? '65532:65532'
|
||||
: '10001:10001',
|
||||
...(isControl ? { ExposedPorts: { '5800/tcp': {} } } : {}),
|
||||
Env: [
|
||||
'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin',
|
||||
'NODE_VERSION=24.18.0',
|
||||
'YARN_VERSION=1.22.22',
|
||||
'NODE_ENV=production',
|
||||
],
|
||||
Entrypoint: [
|
||||
'node',
|
||||
isLocal
|
||||
? '/opt/qinglong/node_modules/@qinglong/local-application/dist/cli.js'
|
||||
: isControl
|
||||
? isControlAi
|
||||
? '/opt/qinglong/node_modules/@qinglong/cluster-control/dist/aiCli.js'
|
||||
: '/opt/qinglong/node_modules/@qinglong/cluster-control/dist/cli.js'
|
||||
: '/opt/qinglong/node_modules/@qinglong/cluster-admin/dist/plugin-package/recovery/pluginPackageRecoveryCli.js',
|
||||
],
|
||||
WorkingDir: '/opt/qinglong',
|
||||
Labels: {
|
||||
...(isLocal
|
||||
? {
|
||||
'io.qinglong.ai': 'excluded',
|
||||
'io.qinglong.local.application-config': '2',
|
||||
'io.qinglong.local.compose-selection': '1',
|
||||
'io.qinglong.local.sqlite-contract-max': '41',
|
||||
'io.qinglong.local.sqlite-contract-min': '41',
|
||||
'io.qinglong.local.sqlite-write-contract': '41',
|
||||
'io.qinglong.profile': 'edge,standalone',
|
||||
}
|
||||
: {}),
|
||||
'org.opencontainers.image.description': isLocal
|
||||
? 'QingLong 3.0 AI-excluded Edge and Standalone runtime'
|
||||
: isControl
|
||||
? isControlAi
|
||||
? 'Optional QingLong 3.0 AI-enabled cluster control plane'
|
||||
: 'QingLong 3.0 PostgreSQL-backed cluster control plane'
|
||||
: 'QingLong 3.0 short-lived cluster administration jobs',
|
||||
'org.opencontainers.image.licenses': 'Apache-2.0',
|
||||
'org.opencontainers.image.revision': revision,
|
||||
'org.opencontainers.image.source':
|
||||
'https://github.com/whyour/qinglong',
|
||||
'org.opencontainers.image.title': isLocal
|
||||
? 'QingLong 3.0 Local Application'
|
||||
: isControl
|
||||
? isControlAi
|
||||
? 'QingLong 3.0 Cluster Control AI'
|
||||
: 'QingLong 3.0 Cluster Control'
|
||||
: 'QingLong 3.0 Cluster Admin',
|
||||
...(isLocal
|
||||
? {
|
||||
'org.opencontainers.image.version': '3.0.0-alpha.0',
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
},
|
||||
rootfs: {
|
||||
type: 'layers',
|
||||
diff_ids: [
|
||||
`sha256:${(architecture === 'amd64' ? 'a' : 'b').repeat(64)}`,
|
||||
],
|
||||
},
|
||||
},
|
||||
'application/vnd.oci.image.config.v1+json',
|
||||
);
|
||||
const layer = blob(
|
||||
Buffer.from(`fixture-${architecture}`),
|
||||
'application/vnd.oci.image.layer.v1.tar+gzip',
|
||||
);
|
||||
const manifest = blob(
|
||||
{
|
||||
schemaVersion: 2,
|
||||
mediaType: 'application/vnd.oci.image.manifest.v1+json',
|
||||
config,
|
||||
layers: [layer],
|
||||
},
|
||||
'application/vnd.oci.image.manifest.v1+json',
|
||||
);
|
||||
const descriptor = {
|
||||
...manifest,
|
||||
platform: { architecture, os: 'linux' },
|
||||
};
|
||||
imageDescriptors.push(descriptor);
|
||||
descriptors.push(descriptor);
|
||||
}
|
||||
|
||||
for (const imageDescriptor of imageDescriptors) {
|
||||
const spdx = blob(
|
||||
{
|
||||
_type: 'https://in-toto.io/Statement/v1',
|
||||
subject: [],
|
||||
predicateType: 'https://spdx.dev/Document',
|
||||
predicate: {
|
||||
spdxVersion: 'SPDX-2.3',
|
||||
packages: applicationPackages,
|
||||
},
|
||||
},
|
||||
'application/vnd.in-toto+json',
|
||||
);
|
||||
spdx.annotations = {
|
||||
'in-toto.io/predicate-type': 'https://spdx.dev/Document',
|
||||
};
|
||||
const provenance = blob(
|
||||
{
|
||||
_type: 'https://in-toto.io/Statement/v1',
|
||||
subject: [],
|
||||
predicateType: 'https://slsa.dev/provenance/v1',
|
||||
predicate: {
|
||||
buildDefinition: {
|
||||
buildType:
|
||||
'https://github.com/moby/buildkit/blob/master/docs/attestations/slsa-definitions.md',
|
||||
externalParameters: {
|
||||
request: {
|
||||
frontend: 'dockerfile.v0',
|
||||
args: {
|
||||
'build-arg:SOURCE_REVISION': revision,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
runDetails: { builder: { id: '' } },
|
||||
},
|
||||
},
|
||||
'application/vnd.in-toto+json',
|
||||
);
|
||||
provenance.annotations = {
|
||||
'in-toto.io/predicate-type': 'https://slsa.dev/provenance/v1',
|
||||
};
|
||||
const attestationLayers = options.omitProvenance
|
||||
? [spdx]
|
||||
: [spdx, provenance];
|
||||
const attestationConfig = blob(
|
||||
{
|
||||
architecture: 'unknown',
|
||||
os: 'unknown',
|
||||
config: {},
|
||||
rootfs: {
|
||||
type: 'layers',
|
||||
diff_ids: attestationLayers.map((layer) => layer.digest),
|
||||
},
|
||||
},
|
||||
'application/vnd.oci.image.config.v1+json',
|
||||
);
|
||||
const attestationManifest = blob(
|
||||
{
|
||||
schemaVersion: 2,
|
||||
mediaType: 'application/vnd.oci.image.manifest.v1+json',
|
||||
config: attestationConfig,
|
||||
layers: attestationLayers,
|
||||
},
|
||||
'application/vnd.oci.image.manifest.v1+json',
|
||||
);
|
||||
descriptors.push({
|
||||
...attestationManifest,
|
||||
annotations: {
|
||||
'vnd.docker.reference.digest': options.unboundAttestation
|
||||
? `sha256:${'f'.repeat(64)}`
|
||||
: imageDescriptor.digest,
|
||||
'vnd.docker.reference.type': 'attestation-manifest',
|
||||
},
|
||||
platform: { architecture: 'unknown', os: 'unknown' },
|
||||
});
|
||||
}
|
||||
|
||||
if (!options.onlyArchitecture) {
|
||||
while (descriptors.length < 4) {
|
||||
descriptors.push({
|
||||
...descriptors[descriptors.length - 1],
|
||||
digest: `sha256:${'0'.repeat(64)}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
const imageIndex = blob(
|
||||
{
|
||||
schemaVersion: 2,
|
||||
mediaType: 'application/vnd.oci.image.index.v1+json',
|
||||
manifests: descriptors,
|
||||
},
|
||||
'application/vnd.oci.image.index.v1+json',
|
||||
);
|
||||
fs.writeFileSync(
|
||||
path.join(layoutRoot, 'index.json'),
|
||||
JSON.stringify({
|
||||
schemaVersion: 2,
|
||||
mediaType: 'application/vnd.oci.image.index.v1+json',
|
||||
manifests: [imageIndex],
|
||||
}),
|
||||
);
|
||||
return layoutRoot;
|
||||
}
|
||||
|
||||
test('accepts two exact images with bound SBOM and provenance', (t) => {
|
||||
const report = auditClusterOciLayout({
|
||||
root,
|
||||
layoutRoot: createFixture(t),
|
||||
expectedRevision: revision,
|
||||
});
|
||||
assert.equal(report.platforms.length, 2);
|
||||
assert.deepEqual(
|
||||
report.platforms.map((entry) => entry.platform),
|
||||
['linux/amd64', 'linux/arm64'],
|
||||
);
|
||||
assert.deepEqual(
|
||||
report.platforms.map((entry) => entry.spdxApplicationPackages),
|
||||
[46, 46],
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts the independent cluster-admin image and attestation closure', (t) => {
|
||||
const report = auditClusterOciLayout({
|
||||
root,
|
||||
layoutRoot: createFixture(t, { image: 'admin' }),
|
||||
expectedRevision: revision,
|
||||
image: 'admin',
|
||||
});
|
||||
assert.equal(report.image, 'admin');
|
||||
assert.deepEqual(
|
||||
report.platforms.map((entry) => entry.spdxApplicationPackages),
|
||||
[88, 88],
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts the optional Cluster AI image and attestation closure', (t) => {
|
||||
const report = auditClusterOciLayout({
|
||||
root,
|
||||
layoutRoot: createFixture(t, { image: 'control-ai' }),
|
||||
expectedRevision: revision,
|
||||
image: 'control-ai',
|
||||
});
|
||||
assert.equal(report.image, 'control-ai');
|
||||
assert.deepEqual(
|
||||
report.platforms.map((entry) => entry.spdxApplicationPackages),
|
||||
[47, 47],
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts the AI-excluded local image and attestation closure', (t) => {
|
||||
const report = auditClusterOciLayout({
|
||||
root,
|
||||
layoutRoot: createFixture(t, { image: 'local' }),
|
||||
expectedRevision: revision,
|
||||
image: 'local',
|
||||
});
|
||||
assert.equal(report.image, 'local');
|
||||
assert.equal(report.maximumPlatformBytes, 128 * 1024 * 1024);
|
||||
assert.deepEqual(
|
||||
report.platforms.map((entry) => entry.spdxApplicationPackages),
|
||||
[10, 10],
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects cluster-control config presented as cluster-admin evidence', (t) => {
|
||||
assert.throws(
|
||||
() =>
|
||||
auditClusterOciLayout({
|
||||
root,
|
||||
layoutRoot: createFixture(t),
|
||||
expectedRevision: revision,
|
||||
image: 'admin',
|
||||
}),
|
||||
/SPDX application closure differs|image config differs/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects an absent arm64 image', (t) => {
|
||||
assert.throws(
|
||||
() =>
|
||||
auditClusterOciLayout({
|
||||
root,
|
||||
layoutRoot: createFixture(t, { omitArm64: true }),
|
||||
expectedRevision: revision,
|
||||
}),
|
||||
/platform set/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a root runtime config on either architecture', (t) => {
|
||||
assert.throws(
|
||||
() =>
|
||||
auditClusterOciLayout({
|
||||
root,
|
||||
layoutRoot: createFixture(t, { rootArm64: true }),
|
||||
expectedRevision: revision,
|
||||
}),
|
||||
/image config differs/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects an attestation that is not bound to its image digest', (t) => {
|
||||
assert.throws(
|
||||
() =>
|
||||
auditClusterOciLayout({
|
||||
root,
|
||||
layoutRoot: createFixture(t, { unboundAttestation: true }),
|
||||
expectedRevision: revision,
|
||||
}),
|
||||
/one bound attestation/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects an incomplete predicate set', (t) => {
|
||||
assert.throws(
|
||||
() =>
|
||||
auditClusterOciLayout({
|
||||
root,
|
||||
layoutRoot: createFixture(t, { omitProvenance: true }),
|
||||
expectedRevision: revision,
|
||||
}),
|
||||
/exactly SBOM and provenance/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a development package in the image SBOM closure', (t) => {
|
||||
assert.throws(
|
||||
() =>
|
||||
auditClusterOciLayout({
|
||||
root,
|
||||
layoutRoot: createFixture(t, { addDevelopmentPackage: true }),
|
||||
expectedRevision: revision,
|
||||
}),
|
||||
/SPDX application closure differs/,
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts one native layout with one bound attestation', (t) => {
|
||||
const report = auditClusterOciLayout({
|
||||
root,
|
||||
layoutRoot: createFixture(t, { onlyArchitecture: 'arm64' }),
|
||||
expectedRevision: revision,
|
||||
expectedPlatforms: ['linux/arm64'],
|
||||
});
|
||||
assert.deepEqual(
|
||||
report.platforms.map((entry) => entry.platform),
|
||||
['linux/arm64'],
|
||||
);
|
||||
});
|
||||
|
||||
test('merges two scanned native layouts into the exact audited multiarch digest', (t) => {
|
||||
const amd64Layout = createFixture(t, { onlyArchitecture: 'amd64' });
|
||||
const arm64Layout = createFixture(t, { onlyArchitecture: 'arm64' });
|
||||
const outputParent = fs.realpathSync(
|
||||
fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-release-bundle-')),
|
||||
);
|
||||
t.after(() => fs.rmSync(outputParent, { recursive: true, force: true }));
|
||||
const amd64Evidence = path.join(outputParent, 'amd64-evidence.json');
|
||||
const arm64Evidence = path.join(outputParent, 'arm64-evidence.json');
|
||||
fs.writeFileSync(
|
||||
amd64Evidence,
|
||||
JSON.stringify(
|
||||
nativeEvidenceRecord({
|
||||
root,
|
||||
layoutRoot: amd64Layout,
|
||||
expectedRevision: revision,
|
||||
image: 'control',
|
||||
platform: 'linux/amd64',
|
||||
}),
|
||||
),
|
||||
);
|
||||
fs.writeFileSync(
|
||||
arm64Evidence,
|
||||
JSON.stringify(
|
||||
nativeEvidenceRecord({
|
||||
root,
|
||||
layoutRoot: arm64Layout,
|
||||
expectedRevision: revision,
|
||||
image: 'control',
|
||||
platform: 'linux/arm64',
|
||||
}),
|
||||
),
|
||||
);
|
||||
const outputRoot = path.join(outputParent, 'merged');
|
||||
const predicatePath = path.join(outputParent, 'predicate.json');
|
||||
const reportPath = path.join(outputParent, 'report.json');
|
||||
const report = mergeNativeLayouts({
|
||||
root,
|
||||
image: 'control',
|
||||
expectedRevision: revision,
|
||||
amd64Layout,
|
||||
amd64Evidence,
|
||||
arm64Layout,
|
||||
arm64Evidence,
|
||||
outputRoot,
|
||||
predicatePath,
|
||||
reportPath,
|
||||
});
|
||||
assert.match(report.rootIndexDigest, /^sha256:[0-9a-f]{64}$/);
|
||||
assert.equal(
|
||||
JSON.parse(fs.readFileSync(predicatePath, 'utf8')).subjectDigest,
|
||||
report.rootIndexDigest,
|
||||
);
|
||||
assert.equal(
|
||||
auditClusterOciLayout({
|
||||
root,
|
||||
layoutRoot: outputRoot,
|
||||
expectedRevision: revision,
|
||||
image: 'control',
|
||||
}).rootIndexDigest,
|
||||
report.rootIndexDigest,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects native evidence that does not exactly describe its OCI layout', (t) => {
|
||||
const amd64Layout = createFixture(t, { onlyArchitecture: 'amd64' });
|
||||
const arm64Layout = createFixture(t, { onlyArchitecture: 'arm64' });
|
||||
const outputParent = fs.realpathSync(
|
||||
fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-release-bundle-mismatch-')),
|
||||
);
|
||||
t.after(() => fs.rmSync(outputParent, { recursive: true, force: true }));
|
||||
const amd64Evidence = path.join(outputParent, 'amd64-evidence.json');
|
||||
const arm64Evidence = path.join(outputParent, 'arm64-evidence.json');
|
||||
const amd64Record = nativeEvidenceRecord({
|
||||
root,
|
||||
layoutRoot: amd64Layout,
|
||||
expectedRevision: revision,
|
||||
image: 'control',
|
||||
platform: 'linux/amd64',
|
||||
});
|
||||
fs.writeFileSync(
|
||||
amd64Evidence,
|
||||
JSON.stringify({ ...amd64Record, sourceRevision: 'different' }),
|
||||
);
|
||||
fs.writeFileSync(
|
||||
arm64Evidence,
|
||||
JSON.stringify(
|
||||
nativeEvidenceRecord({
|
||||
root,
|
||||
layoutRoot: arm64Layout,
|
||||
expectedRevision: revision,
|
||||
image: 'control',
|
||||
platform: 'linux/arm64',
|
||||
}),
|
||||
),
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
mergeNativeLayouts({
|
||||
root,
|
||||
image: 'control',
|
||||
expectedRevision: revision,
|
||||
amd64Layout,
|
||||
amd64Evidence,
|
||||
arm64Layout,
|
||||
arm64Evidence,
|
||||
outputRoot: path.join(outputParent, 'merged'),
|
||||
predicatePath: path.join(outputParent, 'predicate.json'),
|
||||
reportPath: path.join(outputParent, 'report.json'),
|
||||
}),
|
||||
/native vulnerability evidence differs for linux\/amd64/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,140 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const test = require('node:test');
|
||||
const {
|
||||
auditPublishedManifest,
|
||||
} = require('../../scripts/ql3-cluster-remote-manifest-audit.cjs');
|
||||
|
||||
const image = 'ghcr.io/whyour/qinglong3-cluster-control';
|
||||
const digest = `sha256:${'f'.repeat(64)}`;
|
||||
const descriptorDigest = (character) => `sha256:${character.repeat(64)}`;
|
||||
|
||||
function createManifest() {
|
||||
const amd64Digest = descriptorDigest('a');
|
||||
const arm64Digest = descriptorDigest('b');
|
||||
return {
|
||||
schemaVersion: 2,
|
||||
mediaType: 'application/vnd.oci.image.index.v1+json',
|
||||
manifests: [
|
||||
{
|
||||
mediaType: 'application/vnd.oci.image.manifest.v1+json',
|
||||
digest: amd64Digest,
|
||||
size: 2048,
|
||||
platform: {
|
||||
architecture: 'amd64',
|
||||
os: 'linux',
|
||||
},
|
||||
},
|
||||
{
|
||||
mediaType: 'application/vnd.oci.image.manifest.v1+json',
|
||||
digest: descriptorDigest('c'),
|
||||
size: 1024,
|
||||
annotations: {
|
||||
'vnd.docker.reference.digest': amd64Digest,
|
||||
'vnd.docker.reference.type': 'attestation-manifest',
|
||||
},
|
||||
platform: {
|
||||
architecture: 'unknown',
|
||||
os: 'unknown',
|
||||
},
|
||||
},
|
||||
{
|
||||
mediaType: 'application/vnd.oci.image.manifest.v1+json',
|
||||
digest: arm64Digest,
|
||||
size: 2048,
|
||||
platform: {
|
||||
architecture: 'arm64',
|
||||
os: 'linux',
|
||||
},
|
||||
},
|
||||
{
|
||||
mediaType: 'application/vnd.oci.image.manifest.v1+json',
|
||||
digest: descriptorDigest('d'),
|
||||
size: 1024,
|
||||
annotations: {
|
||||
'vnd.docker.reference.digest': arm64Digest,
|
||||
'vnd.docker.reference.type': 'attestation-manifest',
|
||||
},
|
||||
platform: {
|
||||
architecture: 'unknown',
|
||||
os: 'unknown',
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
test('accepts the exact published dual-architecture manifest contract', () => {
|
||||
assert.deepEqual(
|
||||
auditPublishedManifest(createManifest(), {
|
||||
expectedImage: image,
|
||||
expectedDigest: digest,
|
||||
}),
|
||||
{
|
||||
reference: `${image}@${digest}`,
|
||||
platforms: [
|
||||
{
|
||||
platform: 'linux/amd64',
|
||||
digest: descriptorDigest('a'),
|
||||
},
|
||||
{
|
||||
platform: 'linux/arm64',
|
||||
digest: descriptorDigest('b'),
|
||||
},
|
||||
],
|
||||
attestationBindings: [descriptorDigest('a'), descriptorDigest('b')],
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a published manifest without arm64', () => {
|
||||
const manifest = createManifest();
|
||||
manifest.manifests[2].platform.architecture = 'amd64';
|
||||
assert.throws(
|
||||
() =>
|
||||
auditPublishedManifest(manifest, {
|
||||
expectedImage: image,
|
||||
expectedDigest: digest,
|
||||
}),
|
||||
/unexpected published runnable platform/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects an unreviewed runnable platform', () => {
|
||||
const manifest = createManifest();
|
||||
manifest.manifests[2].platform.architecture = 's390x';
|
||||
assert.throws(
|
||||
() =>
|
||||
auditPublishedManifest(manifest, {
|
||||
expectedImage: image,
|
||||
expectedDigest: digest,
|
||||
}),
|
||||
/unexpected published runnable platform/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects an attestation bound to the wrong image', () => {
|
||||
const manifest = createManifest();
|
||||
manifest.manifests[3].annotations['vnd.docker.reference.digest'] =
|
||||
descriptorDigest('a');
|
||||
assert.throws(
|
||||
() =>
|
||||
auditPublishedManifest(manifest, {
|
||||
expectedImage: image,
|
||||
expectedDigest: digest,
|
||||
}),
|
||||
/not bound one-to-one/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a tag reference in place of an immutable digest', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
auditPublishedManifest(createManifest(), {
|
||||
expectedImage: image,
|
||||
expectedDigest: 'latest',
|
||||
}),
|
||||
/immutable SHA-256/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,168 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { createRequire } = require('node:module');
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const ownerCliRequire = createRequire(
|
||||
require.resolve('../../packages/ql3-local-owner-cli/package.json'),
|
||||
);
|
||||
const localSqliteRequire = createRequire(
|
||||
require.resolve('../../packages/ql3-local-sqlite/package.json'),
|
||||
);
|
||||
const localExecutionRequire = createRequire(
|
||||
require.resolve('../../packages/ql3-local-execution/package.json'),
|
||||
);
|
||||
const { runLocalTaskDefinitionCommandFile } = ownerCliRequire(
|
||||
'@qinglong/local-owner-cli/task-definition-command',
|
||||
);
|
||||
const { runLocalTriggerCommandFile } = ownerCliRequire(
|
||||
'@qinglong/local-owner-cli/trigger-command',
|
||||
);
|
||||
const { openLocalSqliteRuntimeDatabase } = localSqliteRequire(
|
||||
'@qinglong/local-sqlite/runtime',
|
||||
);
|
||||
const { LocalSchedulerCoordinator } = localExecutionRequire(
|
||||
'@qinglong/local-execution/scheduler',
|
||||
);
|
||||
const {
|
||||
localManagementFixture,
|
||||
taskPutRequest,
|
||||
writeCommand,
|
||||
} = require('../../packages/ql3-local-owner-cli/test/localManagementFixture.cjs');
|
||||
|
||||
function triggerRequest(value, task, overrides = {}) {
|
||||
return {
|
||||
projectId: 'default',
|
||||
triggerId: 'fresh-product-trigger',
|
||||
expectedRevision: null,
|
||||
mutationId: 'a1000000-0000-4000-8000-000000000001',
|
||||
requestId: 'fresh-trigger-put-1',
|
||||
failureAuditEventId: 'a2000000-0000-4000-8000-000000000001',
|
||||
taskId: task.taskId,
|
||||
taskRevision: task.revision,
|
||||
taskContentDigest: task.contentDigest,
|
||||
spec: {
|
||||
schema: 'qinglong/cron@v1',
|
||||
config: {
|
||||
expression: '* * * * *',
|
||||
timezone: 'UTC',
|
||||
misfirePolicy: 'fire_once',
|
||||
},
|
||||
},
|
||||
enabled: true,
|
||||
occurredAtMs: value.now + 1,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('fresh product commands create Task then Trigger then one scheduled Run', async (t) => {
|
||||
const value = await localManagementFixture(t);
|
||||
const taskCreate = taskPutRequest(value, '1', {
|
||||
taskId: 'fresh-product-task',
|
||||
});
|
||||
const task = (
|
||||
await runLocalTaskDefinitionCommandFile(
|
||||
writeCommand(value, 'task.put', taskCreate, 'fresh-task-create'),
|
||||
)
|
||||
).task;
|
||||
const triggerCreate = triggerRequest(value, task);
|
||||
const trigger = (
|
||||
await runLocalTriggerCommandFile(
|
||||
writeCommand(value, 'trigger.put', triggerCreate, 'fresh-trigger-create'),
|
||||
)
|
||||
).trigger;
|
||||
assert.equal(trigger.taskContentDigest, task.contentDigest);
|
||||
|
||||
const runtime = await openLocalSqliteRuntimeDatabase({
|
||||
databasePath: value.databasePath,
|
||||
profile: 'edge',
|
||||
});
|
||||
t.after(() => runtime.close());
|
||||
const clockValues = [value.now + 1, value.now + 2_000, value.now + 4_000];
|
||||
const ids = [
|
||||
'a3000000-0000-4000-8000-000000000001',
|
||||
'a3000000-0000-4000-8000-000000000002',
|
||||
'a3000000-0000-4000-8000-000000000003',
|
||||
'a3000000-0000-4000-8000-000000000004',
|
||||
];
|
||||
const scheduler = new LocalSchedulerCoordinator(runtime.schedules, {
|
||||
pageSize: 4,
|
||||
misfireGraceMs: 5_000,
|
||||
clock: () => clockValues.shift(),
|
||||
createId: () => ids.shift(),
|
||||
nextOccurrence: (_schedule, afterMs) => afterMs + 1_000,
|
||||
});
|
||||
assert.equal((await scheduler.scheduleOnce()).initialized, 1);
|
||||
const admitted = await scheduler.scheduleOnce();
|
||||
assert.equal(admitted.admitted, 1);
|
||||
|
||||
const database = new DatabaseSync(value.databasePath, { readOnly: true });
|
||||
try {
|
||||
const run = database
|
||||
.prepare(
|
||||
`SELECT id, task_id AS "taskId", trigger_id AS "triggerId", status
|
||||
FROM "Runs"`,
|
||||
)
|
||||
.get();
|
||||
assert.deepEqual(
|
||||
{ ...run },
|
||||
{
|
||||
id: 'a3000000-0000-4000-8000-000000000001',
|
||||
taskId: task.taskId,
|
||||
triggerId: trigger.triggerId,
|
||||
status: 'queued',
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
|
||||
await runLocalTaskDefinitionCommandFile(
|
||||
writeCommand(
|
||||
value,
|
||||
'task.put',
|
||||
taskPutRequest(value, '2', {
|
||||
taskId: task.taskId,
|
||||
expectedRevision: 1,
|
||||
mutationId: 'a4000000-0000-4000-8000-000000000001',
|
||||
requestId: 'fresh-task-disable-1',
|
||||
failureAuditEventId: 'a5000000-0000-4000-8000-000000000001',
|
||||
enabled: false,
|
||||
occurredAtMs: value.now + 3_000,
|
||||
}),
|
||||
'fresh-task-disable',
|
||||
),
|
||||
);
|
||||
const afterTaskDisable = await scheduler.scheduleOnce();
|
||||
assert.equal(afterTaskDisable.scanned, 0);
|
||||
|
||||
const disabledTrigger = await runLocalTriggerCommandFile(
|
||||
writeCommand(
|
||||
value,
|
||||
'trigger.put',
|
||||
triggerRequest(value, task, {
|
||||
expectedRevision: 1,
|
||||
mutationId: 'a6000000-0000-4000-8000-000000000001',
|
||||
requestId: 'fresh-trigger-disable-1',
|
||||
failureAuditEventId: 'a7000000-0000-4000-8000-000000000001',
|
||||
enabled: false,
|
||||
occurredAtMs: value.now + 4_000,
|
||||
}),
|
||||
'fresh-trigger-disable',
|
||||
),
|
||||
);
|
||||
assert.equal(disabledTrigger.trigger.enabled, false);
|
||||
assert.equal(disabledTrigger.trigger.revision, 2);
|
||||
|
||||
const finalDatabase = new DatabaseSync(value.databasePath, {
|
||||
readOnly: true,
|
||||
});
|
||||
try {
|
||||
assert.equal(
|
||||
finalDatabase.prepare('SELECT COUNT(*) AS count FROM "Runs"').get().count,
|
||||
1,
|
||||
);
|
||||
} finally {
|
||||
finalDatabase.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,195 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { afterEach, test } = require('node:test');
|
||||
|
||||
const {
|
||||
ImageOsVulnerabilityPolicyError,
|
||||
auditImageOsVulnerabilityPolicy,
|
||||
parseArguments,
|
||||
renderTrivyIgnore,
|
||||
runCli,
|
||||
} = require('../../scripts/ql3-image-os-vulnerability-policy.cjs');
|
||||
|
||||
const NOW = Date.parse('2026-08-01T12:00:00.000Z');
|
||||
const temporaryDirectories = [];
|
||||
|
||||
function exception(overrides = {}) {
|
||||
return {
|
||||
id: 'CVE-2026-12345',
|
||||
images: ['admin', 'control'],
|
||||
purls: ['pkg:deb/debian/libssl3@3.0.0-1'],
|
||||
owner: 'security/platform',
|
||||
ticket: 'QLSEC-123',
|
||||
expiresOn: '2026-08-15',
|
||||
rationale: 'Temporary exposure accepted while the fixed base image is qualified.',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function policy(exceptions = []) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
fixture: 'qinglong/image-os-vulnerability-exceptions@v1',
|
||||
exceptions,
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const directory of temporaryDirectories.splice(0)) {
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('accepts the empty fail-closed production exception policy', () => {
|
||||
const audit = auditImageOsVulnerabilityPolicy(policy(), { now: () => NOW });
|
||||
assert.deepEqual(audit, {
|
||||
compatible: true,
|
||||
findings: [],
|
||||
exceptionCount: 0,
|
||||
imageExceptionCounts: {
|
||||
admin: 0,
|
||||
control: 0,
|
||||
'control-ai': 0,
|
||||
local: 0,
|
||||
},
|
||||
});
|
||||
assert.equal(
|
||||
renderTrivyIgnore(policy(), 'local', { now: () => NOW }),
|
||||
'vulnerabilities:\n []\n',
|
||||
);
|
||||
});
|
||||
|
||||
test('renders only one image scoped active exception with lifecycle metadata', () => {
|
||||
const document = policy([exception()]);
|
||||
const admin = renderTrivyIgnore(document, 'admin', { now: () => NOW });
|
||||
assert.match(admin, /CVE-2026-12345/);
|
||||
assert.match(admin, /pkg:deb\/debian\/libssl3@3\.0\.0-1/);
|
||||
assert.match(admin, /expired_at: 2026-08-15/);
|
||||
assert.match(admin, /owner=security\/platform; ticket=QLSEC-123/);
|
||||
assert.equal(
|
||||
renderTrivyIgnore(document, 'local', { now: () => NOW }),
|
||||
'vulnerabilities:\n []\n',
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects expired, same-day and overlong exceptions', () => {
|
||||
for (const expiresOn of ['2026-07-31', '2026-08-01', '2026-09-01']) {
|
||||
const audit = auditImageOsVulnerabilityPolicy(
|
||||
policy([exception({ expiresOn })]),
|
||||
{ now: () => NOW },
|
||||
);
|
||||
assert.equal(audit.compatible, false);
|
||||
assert.equal(
|
||||
audit.findings.some(
|
||||
(finding) =>
|
||||
finding.code === 'QL3_IMAGE_OS_VULNERABILITY_EXCEPTION_EXPIRY',
|
||||
),
|
||||
true,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects missing ownership, ticket and meaningful rationale', () => {
|
||||
const audit = auditImageOsVulnerabilityPolicy(
|
||||
policy([
|
||||
exception({ owner: 'UPPER', ticket: 'none', rationale: 'temporary' }),
|
||||
]),
|
||||
{ now: () => NOW },
|
||||
);
|
||||
assert.deepEqual(
|
||||
audit.findings.map((finding) => finding.code),
|
||||
[
|
||||
'QL3_IMAGE_OS_VULNERABILITY_EXCEPTION_OWNER',
|
||||
'QL3_IMAGE_OS_VULNERABILITY_EXCEPTION_TICKET',
|
||||
'QL3_IMAGE_OS_VULNERABILITY_EXCEPTION_RATIONALE',
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects unscoped images and non-OS package purls', () => {
|
||||
const audit = auditImageOsVulnerabilityPolicy(
|
||||
policy([
|
||||
exception({
|
||||
images: ['unknown'],
|
||||
purls: ['pkg:npm/example@1.0.0'],
|
||||
}),
|
||||
]),
|
||||
{ now: () => NOW },
|
||||
);
|
||||
assert.deepEqual(
|
||||
audit.findings.map((finding) => finding.code),
|
||||
[
|
||||
'QL3_IMAGE_OS_VULNERABILITY_EXCEPTION_IMAGES',
|
||||
'QL3_IMAGE_OS_VULNERABILITY_EXCEPTION_PURLS',
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects duplicate, unsorted and extensible exception identities', () => {
|
||||
for (const exceptions of [
|
||||
[exception(), exception()],
|
||||
[
|
||||
exception({ id: 'CVE-2026-99999' }),
|
||||
exception({ id: 'CVE-2026-12345' }),
|
||||
],
|
||||
[{ ...exception(), extra: true }],
|
||||
]) {
|
||||
const audit = auditImageOsVulnerabilityPolicy(policy(exceptions), {
|
||||
now: () => NOW,
|
||||
});
|
||||
assert.equal(audit.compatible, false);
|
||||
assert.equal(
|
||||
audit.findings.some(
|
||||
(finding) =>
|
||||
finding.code === 'QL3_IMAGE_OS_VULNERABILITY_EXCEPTION_ID',
|
||||
),
|
||||
true,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('creates one private no-replace Trivy ignore file through the exact CLI', () => {
|
||||
const directory = fs.realpathSync(
|
||||
fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-os-policy-')),
|
||||
);
|
||||
temporaryDirectories.push(directory);
|
||||
const policyDirectory = path.join(directory, 'deploy/containers');
|
||||
fs.mkdirSync(policyDirectory, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(policyDirectory, 'ql3-os-vulnerability-exceptions.json'),
|
||||
`${JSON.stringify(policy([exception()]))}\n`,
|
||||
);
|
||||
const output = path.join(directory, 'admin.trivyignore.yaml');
|
||||
runCli([`--image=admin`, `--output=${output}`], directory, {
|
||||
now: () => NOW,
|
||||
});
|
||||
assert.equal(fs.statSync(output).mode & 0o777, 0o600);
|
||||
assert.match(fs.readFileSync(output, 'utf8'), /QLSEC-123/);
|
||||
assert.throws(
|
||||
() =>
|
||||
runCli([`--image=admin`, `--output=${output}`], directory, {
|
||||
now: () => NOW,
|
||||
}),
|
||||
/output path must be unused/,
|
||||
);
|
||||
});
|
||||
|
||||
test('parses only audit mode or exact image/output render arguments', () => {
|
||||
assert.deepEqual(parseArguments([]), { mode: 'audit' });
|
||||
assert.equal(
|
||||
parseArguments(['--image=control', '--output=/tmp/ignore.yaml']).image,
|
||||
'control',
|
||||
);
|
||||
assert.throws(
|
||||
() => parseArguments(['--image=control']),
|
||||
ImageOsVulnerabilityPolicyError,
|
||||
);
|
||||
assert.throws(
|
||||
() => parseArguments(['--image=control', '--output=/tmp/a', '--extra=x']),
|
||||
ImageOsVulnerabilityPolicyError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,360 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
RESOURCE_TIERS,
|
||||
createWorkloadPlans,
|
||||
parseArguments,
|
||||
parseCpuMax,
|
||||
parseKeyValueFile,
|
||||
parseLimit,
|
||||
parseMountOptions,
|
||||
parseNodeTestReport,
|
||||
validateEnvelope,
|
||||
} = require('../../scripts/ql3-linux-resource-gate.cjs');
|
||||
|
||||
function envelopeFor(tierName) {
|
||||
const tier = RESOURCE_TIERS[tierName];
|
||||
return {
|
||||
memoryMaxBytes: tier.memoryMaxBytes,
|
||||
memoryPeakBytes: 64 * 1024 * 1024,
|
||||
swapMaxBytes: tier.swapMaxBytes,
|
||||
cpuQuotaCores: tier.cpuQuotaCores,
|
||||
pidsMax: tier.pidsMax,
|
||||
noNewPrivileges: 1,
|
||||
seccompMode: 2,
|
||||
mounts: new Map([
|
||||
['/', ['ro', 'relatime']],
|
||||
['/workspace', ['ro', 'nodev']],
|
||||
['/tmp', ['rw', 'nosuid', 'nodev']],
|
||||
]),
|
||||
};
|
||||
}
|
||||
|
||||
test('defines separate router stress, edge release and cluster control envelopes', () => {
|
||||
assert.deepEqual(Object.keys(RESOURCE_TIERS), [
|
||||
'router-stress-ci',
|
||||
'edge-release-ci',
|
||||
'cluster-control-ci',
|
||||
]);
|
||||
assert.equal(RESOURCE_TIERS['router-stress-ci'].supportedMinimum, false);
|
||||
assert.equal(
|
||||
RESOURCE_TIERS['router-stress-ci'].memoryMaxBytes,
|
||||
128 * 1024 * 1024,
|
||||
);
|
||||
assert.equal(
|
||||
RESOURCE_TIERS['edge-release-ci'].memoryMaxBytes,
|
||||
256 * 1024 * 1024,
|
||||
);
|
||||
assert.equal(
|
||||
RESOURCE_TIERS['cluster-control-ci'].memoryMaxBytes,
|
||||
512 * 1024 * 1024,
|
||||
);
|
||||
assert.notEqual(
|
||||
RESOURCE_TIERS['router-stress-ci'].workload,
|
||||
RESOURCE_TIERS['cluster-control-ci'].workload,
|
||||
);
|
||||
});
|
||||
|
||||
test('parses bounded cgroup v2 and mount evidence', () => {
|
||||
assert.equal(parseLimit('134217728\n', 'memory.max'), 134217728);
|
||||
assert.equal(parseLimit('max\n', 'memory.max'), Number.POSITIVE_INFINITY);
|
||||
assert.equal(parseCpuMax('50000 100000\n'), 0.5);
|
||||
assert.deepEqual(parseKeyValueFile('oom 0\noom_kill 1\n', 'events'), {
|
||||
oom: 0,
|
||||
oom_kill: 1,
|
||||
});
|
||||
const mounts = parseMountOptions(
|
||||
'overlay / overlay ro,relatime 0 0\ntmpfs /tmp tmpfs rw,nosuid,nodev 0 0\nsource /workspace fakeowner ro,nodev 0 0\n',
|
||||
);
|
||||
assert.deepEqual(mounts.get('/'), ['ro', 'relatime']);
|
||||
assert.deepEqual(mounts.get('/tmp'), ['rw', 'nosuid', 'nodev']);
|
||||
assert.deepEqual(mounts.get('/workspace'), ['ro', 'nodev']);
|
||||
});
|
||||
|
||||
test('fails open hosts, root execution and a widened resource envelope', () => {
|
||||
const valid = envelopeFor('router-stress-ci');
|
||||
assert.deepEqual(
|
||||
validateEnvelope('router-stress-ci', valid, {
|
||||
platform: 'linux',
|
||||
architecture: 'arm64',
|
||||
uid: 65532,
|
||||
}),
|
||||
[],
|
||||
);
|
||||
const widened = {
|
||||
...valid,
|
||||
memoryMaxBytes: 256 * 1024 * 1024,
|
||||
mounts: new Map([
|
||||
['/', ['rw']],
|
||||
['/workspace', ['rw']],
|
||||
['/tmp', ['rw']],
|
||||
]),
|
||||
};
|
||||
assert.deepEqual(
|
||||
validateEnvelope('router-stress-ci', widened, {
|
||||
platform: 'darwin',
|
||||
architecture: 'arm64',
|
||||
uid: 0,
|
||||
}),
|
||||
[
|
||||
'memory.max 268435456 did not equal 134217728',
|
||||
'platform darwin is not linux',
|
||||
'resource workload must be non-root',
|
||||
'/ must be mounted read-only',
|
||||
'/workspace must be mounted read-only',
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('builds tier-specific workload plans without shell commands', () => {
|
||||
const edge = createWorkloadPlans('/workspace', 'router-stress-ci');
|
||||
assert.deepEqual(
|
||||
edge.map(({ name }) => name),
|
||||
[
|
||||
'edge-executor',
|
||||
'node-sqlite',
|
||||
'local-workflow-product',
|
||||
'local-workflow-sqlite-lock',
|
||||
'local-workflow-admission-crash-recovery',
|
||||
'local-workflow-control-crash-recovery',
|
||||
],
|
||||
);
|
||||
assert.match(edge[0].script, /ql3-edge-benchmark\.cjs$/);
|
||||
assert.ok(edge[0].args.includes('--max-rss-delta-mb=64'));
|
||||
assert.equal(edge[2].format, 'node_test');
|
||||
assert.ok(
|
||||
edge[2].nodeArgs.some((argument) =>
|
||||
/ql3-local-application\/test\/activation\.test\.cjs$/.test(argument),
|
||||
),
|
||||
);
|
||||
assert.equal(edge[2].maxProcessRssBytes, 96 * 1024 * 1024);
|
||||
assert.equal(edge[2].contract, undefined);
|
||||
assert.equal(
|
||||
edge[2].nodeArgs.includes(
|
||||
'--test-name-pattern=executes one admitted Workflow',
|
||||
),
|
||||
true,
|
||||
);
|
||||
const edgeRelease = createWorkloadPlans(
|
||||
'/workspace',
|
||||
'edge-release-ci',
|
||||
);
|
||||
assert.deepEqual(
|
||||
edgeRelease.map(({ name }) => name),
|
||||
[
|
||||
'edge-executor',
|
||||
'node-sqlite',
|
||||
'local-workflow-product',
|
||||
'local-ai-prompt-durable-output-edge',
|
||||
'local-ai-prompt-durable-output-standalone',
|
||||
'local-workflow-sqlite-lock',
|
||||
'local-workflow-admission-crash-recovery',
|
||||
'local-workflow-control-crash-recovery',
|
||||
'local-ai-prompt-model-invocation-crash-recovery',
|
||||
'local-ai-prompt-outer-transaction-crash-recovery',
|
||||
],
|
||||
);
|
||||
assert.equal(
|
||||
edgeRelease[2].nodeArgs.includes(
|
||||
'--test-name-pattern=executes one admitted Workflow|stops one running Workflow Task',
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.deepEqual(edgeRelease[2].contract, {
|
||||
kind: 'local_workflow_product_lifecycle',
|
||||
profile: 'edge',
|
||||
completedWorkflowSteps: 2,
|
||||
completedAttempts: 2,
|
||||
cancelCommandStatus: 'accepted',
|
||||
exactReplay: true,
|
||||
processIdentityObserved: true,
|
||||
processExited: true,
|
||||
parentRunStatus: 'cancelled',
|
||||
attemptStatus: 'cancelled',
|
||||
cancelledStepRuns: 2,
|
||||
cancelEvents: 1,
|
||||
cancelAudits: 1,
|
||||
physicalPowerLossProven: false,
|
||||
});
|
||||
assert.equal(edgeRelease[3].maxProcessRssBytes, 192 * 1024 * 1024);
|
||||
assert.deepEqual(edgeRelease[3].env, {
|
||||
QL3_PROMPT_RESOURCE_PROFILE: 'edge',
|
||||
QL3_PROMPT_RESOURCE_OUTPUT_BYTES: String(512 * 1024),
|
||||
});
|
||||
assert.equal(edgeRelease[3].contract.providerCalls, 2);
|
||||
assert.equal(edgeRelease[3].contract.exactReplay, true);
|
||||
assert.equal(edgeRelease[3].contract.contentFree, true);
|
||||
assert.equal(edgeRelease[3].contract.durableOutputBytes, 512 * 1024);
|
||||
assert.equal(
|
||||
edgeRelease[3].contract.maxWalWriteAmplificationPermille,
|
||||
0,
|
||||
);
|
||||
assert.equal(edgeRelease[4].contract.profile, 'standalone');
|
||||
assert.equal(edgeRelease[4].contract.journalMode, 'wal');
|
||||
assert.equal(edgeRelease[4].contract.requireWalGrowth, true);
|
||||
assert.match(
|
||||
edge[3].script,
|
||||
/ql3-local-workflow-resource-benchmark\.cjs$/,
|
||||
);
|
||||
assert.ok(edge[3].args.includes('--lock-samples=16'));
|
||||
assert.ok(edge[3].args.includes('--max-lock-p95-ms=500'));
|
||||
assert.equal(edge[4].contract.scenarios, 16);
|
||||
assert.equal(edge[4].contract.physicalPowerLossProven, false);
|
||||
assert.equal(edge[5].contract.scenarios, 16);
|
||||
assert.equal(edge[5].contract.conclusiveStopObserved, true);
|
||||
assert.equal(edge[5].contract.physicalPowerLossProven, false);
|
||||
assert.equal(edgeRelease[8].contract.scenarios, 14);
|
||||
assert.deepEqual(edgeRelease[8].contract.boundaries, [
|
||||
'model_start',
|
||||
'model_completion',
|
||||
]);
|
||||
assert.equal(edgeRelease[8].contract.physicalPowerLossProven, false);
|
||||
assert.equal(edgeRelease[9].contract.scenarios, 20);
|
||||
assert.deepEqual(edgeRelease[9].contract.operations, [
|
||||
'admission',
|
||||
'finalization',
|
||||
]);
|
||||
assert.equal(edgeRelease[9].contract.exactReplay, true);
|
||||
assert.equal(edgeRelease[9].contract.contentFree, true);
|
||||
assert.equal(
|
||||
edgeRelease[9].contract.promptAdmissionFinalizationCrashProven,
|
||||
true,
|
||||
);
|
||||
assert.equal(edgeRelease[9].contract.physicalPowerLossProven, false);
|
||||
const cluster = createWorkloadPlans('/workspace', 'cluster-control-ci');
|
||||
assert.deepEqual(
|
||||
cluster.map(({ name }) => name),
|
||||
['cluster-control'],
|
||||
);
|
||||
assert.match(cluster[0].script, /ql3-cluster-control-benchmark\.cjs$/);
|
||||
});
|
||||
|
||||
test('rejects unknown tiers and architecture labels', () => {
|
||||
assert.deepEqual(
|
||||
parseArguments(['--tier=edge-release-ci', '--expected-arch=x64', '--json']),
|
||||
{ tier: 'edge-release-ci', expectedArch: 'x64', json: true },
|
||||
);
|
||||
assert.throws(
|
||||
() => parseArguments(['--tier=router']),
|
||||
/--tier must be one of/,
|
||||
);
|
||||
assert.throws(
|
||||
() => parseArguments(['--tier=router-stress-ci', '--expected-arch=arm']),
|
||||
/--expected-arch must be x64 or arm64/,
|
||||
);
|
||||
});
|
||||
|
||||
test('fails closed when durable Prompt resource evidence drifts', () => {
|
||||
const [edgePlan, standalonePlan] = createWorkloadPlans(
|
||||
'/workspace',
|
||||
'edge-release-ci',
|
||||
).filter(({ contract }) => contract?.kind === 'durable_prompt_output_resource');
|
||||
const evidence = {
|
||||
profile: 'edge',
|
||||
journalMode: 'delete',
|
||||
durableOutputBytes: 512 * 1024,
|
||||
providerCalls: 2,
|
||||
keyLoads: 1,
|
||||
keyResolutions: 1,
|
||||
liveOnlyKeyLoads: 0,
|
||||
exactReplay: true,
|
||||
contentFree: true,
|
||||
durableFacts: { attempts: 0 },
|
||||
peakProcessRssBytes: 100 * 1024 * 1024,
|
||||
databaseLogicalWriteAmplificationPermille: 1_383,
|
||||
databaseAllocatedWriteAmplificationPermille: 1_383,
|
||||
walWriteAmplificationPermille: 0,
|
||||
walGrowthBytes: 0,
|
||||
physicalPowerLossProven: false,
|
||||
};
|
||||
const output = (value) =>
|
||||
`tests 1\npass 1\nfail 0\nskipped 0\nQL3_RESOURCE_EVIDENCE=${JSON.stringify(value)}\n`;
|
||||
assert.equal(
|
||||
parseNodeTestReport(output(evidence), edgePlan).evidence.profile,
|
||||
'edge',
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
parseNodeTestReport(
|
||||
output({
|
||||
...evidence,
|
||||
databaseLogicalWriteAmplificationPermille: 3_001,
|
||||
}),
|
||||
edgePlan,
|
||||
),
|
||||
/durable Prompt output evidence violated its contract/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
parseNodeTestReport(
|
||||
output({
|
||||
...evidence,
|
||||
profile: 'standalone',
|
||||
journalMode: 'wal',
|
||||
walWriteAmplificationPermille: 0,
|
||||
walGrowthBytes: 0,
|
||||
}),
|
||||
standalonePlan,
|
||||
),
|
||||
/durable Prompt output evidence violated its contract/,
|
||||
);
|
||||
});
|
||||
|
||||
test('fails closed when authenticated Workflow cancellation evidence drifts', () => {
|
||||
const plan = createWorkloadPlans(
|
||||
'/workspace',
|
||||
'edge-release-ci',
|
||||
).find(
|
||||
({ contract }) =>
|
||||
contract?.kind === 'local_workflow_product_lifecycle',
|
||||
);
|
||||
assert.ok(plan);
|
||||
const completionEvidence = {
|
||||
schemaVersion: 1,
|
||||
profile: 'edge',
|
||||
workflowSteps: 2,
|
||||
attempts: 2,
|
||||
peakProcessRssBytes: 79 * 1024 * 1024,
|
||||
};
|
||||
const cancellationEvidence = {
|
||||
schemaVersion: 1,
|
||||
profile: 'edge',
|
||||
cancelCommandStatus: 'accepted',
|
||||
exactReplay: true,
|
||||
processIdentityObserved: true,
|
||||
processExited: true,
|
||||
parentRunStatus: 'cancelled',
|
||||
attemptStatus: 'cancelled',
|
||||
cancelledStepRuns: 2,
|
||||
cancelEvents: 1,
|
||||
cancelAudits: 1,
|
||||
peakProcessRssBytes: 80 * 1024 * 1024,
|
||||
physicalPowerLossProven: false,
|
||||
};
|
||||
const output = (value) =>
|
||||
`tests 2\npass 2\nfail 0\nskipped 0\nQL3_RESOURCE_EVIDENCE=${JSON.stringify(completionEvidence)}\nQL3_RESOURCE_EVIDENCE=${JSON.stringify(value)}\n`;
|
||||
assert.equal(
|
||||
parseNodeTestReport(output(cancellationEvidence), plan)
|
||||
.evidenceRecords[1].processExited,
|
||||
true,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
parseNodeTestReport(
|
||||
output({ ...cancellationEvidence, cancelAudits: 0 }),
|
||||
plan,
|
||||
),
|
||||
/local Workflow cancellation evidence violated its contract/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
parseNodeTestReport(
|
||||
output({
|
||||
...cancellationEvidence,
|
||||
peakProcessRssBytes: 161 * 1024 * 1024,
|
||||
}),
|
||||
plan,
|
||||
),
|
||||
/process RSS exceeded its tier budget/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,327 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { spawnSync } = require('node:child_process');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
RESOURCE_TIERS,
|
||||
createWorkloadPlans,
|
||||
} = require('../../scripts/ql3-linux-resource-gate.cjs');
|
||||
const {
|
||||
TIER_NAMES,
|
||||
bundleArchitectureEvidence,
|
||||
evidenceDigest,
|
||||
mergeCrossArchitectureEvidence,
|
||||
normalizeSource,
|
||||
readJsonFile,
|
||||
validateArchitectureEvidence,
|
||||
} = require('../../scripts/ql3-linux-resource-release-evidence.cjs');
|
||||
|
||||
const scriptPath = path.resolve(
|
||||
__dirname,
|
||||
'../../scripts/ql3-linux-resource-release-evidence.cjs',
|
||||
);
|
||||
|
||||
function fixtureSource(overrides = {}) {
|
||||
return {
|
||||
repository: 'whyour/qinglong',
|
||||
revision: 'a'.repeat(40),
|
||||
workflow: 'QingLong 3.0 CI',
|
||||
runId: '123456',
|
||||
runAttempt: 1,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fixtureTierReport(tierName, architecture) {
|
||||
const tier = RESOURCE_TIERS[tierName];
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
tier: tierName,
|
||||
evidenceClass: tier.evidenceClass,
|
||||
supportedMinimum: tier.supportedMinimum,
|
||||
identity: {
|
||||
platform: 'linux',
|
||||
architecture,
|
||||
node: 'v24.18.0',
|
||||
uid: 65532,
|
||||
gid: 65532,
|
||||
},
|
||||
envelope: {
|
||||
memoryMaxBytes: tier.memoryMaxBytes,
|
||||
memoryPeakBytes: Math.min(64 * 1024 * 1024, tier.memoryMaxBytes),
|
||||
swapMaxBytes: tier.swapMaxBytes,
|
||||
cpuQuotaCores: tier.cpuQuotaCores,
|
||||
pidsMax: tier.pidsMax,
|
||||
noNewPrivileges: 1,
|
||||
seccompMode: 2,
|
||||
rootReadOnly: true,
|
||||
workspaceReadOnly: true,
|
||||
tmpWritable: true,
|
||||
memoryEventsBefore: {
|
||||
low: 0,
|
||||
high: 0,
|
||||
max: 0,
|
||||
oom: 0,
|
||||
oom_kill: 0,
|
||||
oom_group_kill: 0,
|
||||
},
|
||||
memoryEventsAfter: {
|
||||
low: 0,
|
||||
high: 0,
|
||||
max: 0,
|
||||
oom: 0,
|
||||
oom_kill: 0,
|
||||
oom_group_kill: 0,
|
||||
},
|
||||
},
|
||||
workloads: createWorkloadPlans('/workspace', tierName).map(({ name }) => ({
|
||||
name,
|
||||
report: { passed: true },
|
||||
})),
|
||||
gates: { passed: true, violations: [] },
|
||||
};
|
||||
}
|
||||
|
||||
function fixtureReports(architecture) {
|
||||
return Object.fromEntries(
|
||||
TIER_NAMES.map((tierName) => [
|
||||
tierName,
|
||||
fixtureTierReport(tierName, architecture),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
function fixtureBundle(architecture, source = fixtureSource()) {
|
||||
return bundleArchitectureEvidence({
|
||||
source,
|
||||
architecture,
|
||||
reports: fixtureReports(architecture),
|
||||
});
|
||||
}
|
||||
|
||||
function writeJson(filePath, value) {
|
||||
fs.writeFileSync(filePath, `${JSON.stringify(value)}\n`, 'utf8');
|
||||
}
|
||||
|
||||
function runCli(arguments_) {
|
||||
return spawnSync(process.execPath, [scriptPath, ...arguments_], {
|
||||
encoding: 'utf8',
|
||||
});
|
||||
}
|
||||
|
||||
function sourceArguments(source = fixtureSource()) {
|
||||
return [
|
||||
`--repository=${source.repository}`,
|
||||
`--revision=${source.revision}`,
|
||||
`--workflow=${source.workflow}`,
|
||||
`--run-id=${source.runId}`,
|
||||
`--run-attempt=${source.runAttempt}`,
|
||||
];
|
||||
}
|
||||
|
||||
test('bundles native x64 and arm64 reports into source-bound release evidence', () => {
|
||||
const source = fixtureSource();
|
||||
const x64 = fixtureBundle('x64', source);
|
||||
const arm64 = fixtureBundle('arm64', source);
|
||||
const release = mergeCrossArchitectureEvidence({ source, x64, arm64 });
|
||||
|
||||
assert.equal(
|
||||
release.fixture,
|
||||
'qinglong/linux-resource-cross-architecture-evidence@v1',
|
||||
);
|
||||
assert.deepEqual(
|
||||
release.architectures.map(({ architecture }) => architecture),
|
||||
['x64', 'arm64'],
|
||||
);
|
||||
assert.equal(release.architectures[0].tiers.length, 3);
|
||||
assert.equal(release.architectures[1].tiers.length, 3);
|
||||
assert.equal(release.gates.passed, true);
|
||||
assert.equal(release.releaseDigest.length, 64);
|
||||
assert.notEqual(x64.bundleDigest, arm64.bundleDigest);
|
||||
assert.deepEqual(release.limitations, [
|
||||
'CI cgroup evidence is not a supported minimum hardware claim',
|
||||
'CI evidence does not replace fixed-device power-loss, flash, thermal, or soak evidence',
|
||||
'GitHub workflow identity binding is not a cryptographic hardware attestation',
|
||||
]);
|
||||
});
|
||||
|
||||
test('rejects architecture, gate, memory event and schema drift', () => {
|
||||
const source = fixtureSource();
|
||||
const wrongArchitecture = fixtureReports('x64');
|
||||
wrongArchitecture['router-stress-ci'].identity.architecture = 'arm64';
|
||||
assert.throws(
|
||||
() =>
|
||||
bundleArchitectureEvidence({
|
||||
source,
|
||||
architecture: 'x64',
|
||||
reports: wrongArchitecture,
|
||||
}),
|
||||
/reviewed native identity/,
|
||||
);
|
||||
|
||||
const failedGate = fixtureReports('x64');
|
||||
failedGate['edge-release-ci'].gates = {
|
||||
passed: false,
|
||||
violations: ['benchmark failed'],
|
||||
};
|
||||
assert.throws(
|
||||
() =>
|
||||
bundleArchitectureEvidence({
|
||||
source,
|
||||
architecture: 'x64',
|
||||
reports: failedGate,
|
||||
}),
|
||||
/gate did not pass/,
|
||||
);
|
||||
|
||||
const memoryEvent = fixtureReports('x64');
|
||||
memoryEvent['cluster-control-ci'].envelope.memoryEventsAfter.oom_kill = 1;
|
||||
assert.throws(
|
||||
() =>
|
||||
bundleArchitectureEvidence({
|
||||
source,
|
||||
architecture: 'x64',
|
||||
reports: memoryEvent,
|
||||
}),
|
||||
/memory event oom_kill changed/,
|
||||
);
|
||||
|
||||
const widened = fixtureReports('x64');
|
||||
widened['router-stress-ci'].unexpected = true;
|
||||
assert.throws(
|
||||
() =>
|
||||
bundleArchitectureEvidence({
|
||||
source,
|
||||
architecture: 'x64',
|
||||
reports: widened,
|
||||
}),
|
||||
/report fields are invalid/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects source identifiers that only coerce to the reviewed text shape', () => {
|
||||
for (const source of [
|
||||
fixtureSource({ repository: 123 }),
|
||||
fixtureSource({ revision: 123 }),
|
||||
fixtureSource({ runId: 123456 }),
|
||||
]) {
|
||||
assert.throws(() => normalizeSource(source), /source (repository|revision|runId)/);
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects tampered bundles, cross-source mixing and duplicate architecture', () => {
|
||||
const source = fixtureSource();
|
||||
const x64 = fixtureBundle('x64', source);
|
||||
const arm64 = fixtureBundle('arm64', source);
|
||||
const tamperedX64 = { ...x64, bundleDigest: '0'.repeat(64) };
|
||||
assert.throws(
|
||||
() => validateArchitectureEvidence(tamperedX64, source, 'x64'),
|
||||
/digest or gates drifted/,
|
||||
);
|
||||
|
||||
const otherSource = fixtureSource({ revision: 'b'.repeat(40) });
|
||||
assert.throws(
|
||||
() =>
|
||||
mergeCrossArchitectureEvidence({
|
||||
source,
|
||||
x64,
|
||||
arm64: fixtureBundle('arm64', otherSource),
|
||||
}),
|
||||
/belongs to another source/,
|
||||
);
|
||||
assert.throws(
|
||||
() => mergeCrossArchitectureEvidence({ source, x64, arm64: x64 }),
|
||||
/arm64 architecture evidence digest or gates drifted/,
|
||||
);
|
||||
assert.throws(
|
||||
() => mergeCrossArchitectureEvidence({ source, x64, arm64: undefined }),
|
||||
/plain object/,
|
||||
);
|
||||
});
|
||||
|
||||
test('enforces a shared canonical node budget across sibling branches', () => {
|
||||
assert.throws(
|
||||
() => evidenceDigest(Array.from({ length: 100_000 }, () => null)),
|
||||
/node budget exceeded/,
|
||||
);
|
||||
});
|
||||
|
||||
test('CLI creates non-overwriting native bundles and merged evidence', (t) => {
|
||||
const temporaryDirectory = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), 'ql3-linux-resource-release-evidence-'),
|
||||
);
|
||||
t.after(() => fs.rmSync(temporaryDirectory, { recursive: true, force: true }));
|
||||
const source = fixtureSource();
|
||||
|
||||
const bundlePaths = {};
|
||||
for (const architecture of ['x64', 'arm64']) {
|
||||
const reports = fixtureReports(architecture);
|
||||
const reportArguments = [];
|
||||
for (const tierName of TIER_NAMES) {
|
||||
const reportPath = path.join(
|
||||
temporaryDirectory,
|
||||
`${architecture}-${tierName}.json`,
|
||||
);
|
||||
writeJson(reportPath, reports[tierName]);
|
||||
reportArguments.push(`--${tierName}=${reportPath}`);
|
||||
}
|
||||
bundlePaths[architecture] = path.join(
|
||||
temporaryDirectory,
|
||||
`${architecture}.json`,
|
||||
);
|
||||
const result = runCli([
|
||||
'--mode=bundle',
|
||||
...sourceArguments(source),
|
||||
`--architecture=${architecture}`,
|
||||
...reportArguments,
|
||||
`--output=${bundlePaths[architecture]}`,
|
||||
]);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.equal(JSON.parse(result.stdout).passed, true);
|
||||
validateArchitectureEvidence(
|
||||
JSON.parse(fs.readFileSync(bundlePaths[architecture], 'utf8')),
|
||||
source,
|
||||
architecture,
|
||||
);
|
||||
}
|
||||
|
||||
const releasePath = path.join(temporaryDirectory, 'release.json');
|
||||
const merge = runCli([
|
||||
'--mode=merge',
|
||||
...sourceArguments(source),
|
||||
`--x64=${bundlePaths.x64}`,
|
||||
`--arm64=${bundlePaths.arm64}`,
|
||||
`--output=${releasePath}`,
|
||||
]);
|
||||
assert.equal(merge.status, 0, merge.stderr);
|
||||
assert.equal(JSON.parse(merge.stdout).passed, true);
|
||||
assert.equal(
|
||||
JSON.parse(fs.readFileSync(releasePath, 'utf8')).releaseDigest.length,
|
||||
64,
|
||||
);
|
||||
|
||||
const overwrite = runCli([
|
||||
'--mode=merge',
|
||||
...sourceArguments(source),
|
||||
`--x64=${bundlePaths.x64}`,
|
||||
`--arm64=${bundlePaths.arm64}`,
|
||||
`--output=${releasePath}`,
|
||||
]);
|
||||
assert.notEqual(overwrite.status, 0);
|
||||
assert.match(overwrite.stderr, /EEXIST/);
|
||||
});
|
||||
|
||||
test('rejects symlink evidence inputs', (t) => {
|
||||
const temporaryDirectory = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), 'ql3-linux-resource-symlink-'),
|
||||
);
|
||||
t.after(() => fs.rmSync(temporaryDirectory, { recursive: true, force: true }));
|
||||
const target = path.join(temporaryDirectory, 'target.json');
|
||||
const link = path.join(temporaryDirectory, 'link.json');
|
||||
writeJson(target, { passed: true });
|
||||
fs.symlinkSync(target, link);
|
||||
assert.throws(() => readJsonFile(link, 'evidence'), /non-symlink file/);
|
||||
});
|
||||
@@ -0,0 +1,169 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const workflowPath = path.resolve(
|
||||
__dirname,
|
||||
'../../.github/workflows/ql3-ci.yml',
|
||||
);
|
||||
|
||||
test('runs each Linux resource tier on native x64 and arm64 Node 24 runners', () => {
|
||||
const workflow = fs.readFileSync(workflowPath, 'utf8');
|
||||
const resourceJob = workflow.match(
|
||||
/ linux-resource-envelopes:\n([\s\S]*?)\n linux-resource-release-evidence:/,
|
||||
)?.[1];
|
||||
assert.ok(resourceJob, 'linux-resource-envelopes job is missing');
|
||||
assert.match(resourceJob, /runner: ubuntu-24\.04\n\s+arch: x64/);
|
||||
assert.match(resourceJob, /runner: ubuntu-24\.04-arm\n\s+arch: arm64/);
|
||||
assert.match(resourceJob, /node-version: '24\.18\.0'/);
|
||||
assert.match(resourceJob, /Verify native runner architecture/);
|
||||
for (const tier of [
|
||||
'router-stress-ci',
|
||||
'edge-release-ci',
|
||||
'cluster-control-ci',
|
||||
]) {
|
||||
assert.equal(
|
||||
resourceJob.match(new RegExp(`--tier=${tier}`, 'g'))?.length,
|
||||
1,
|
||||
`${tier} must run exactly once per native matrix entry`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('keeps all resource containers fail-closed and exactly bounded', () => {
|
||||
const workflow = fs.readFileSync(workflowPath, 'utf8');
|
||||
const resourceJob = workflow.match(
|
||||
/ linux-resource-envelopes:\n([\s\S]*?)\n linux-resource-release-evidence:/,
|
||||
)?.[1];
|
||||
assert.ok(resourceJob);
|
||||
assert.equal(resourceJob.match(/docker run --rm --read-only/g)?.length, 3);
|
||||
assert.equal(
|
||||
resourceJob.match(/--security-opt no-new-privileges/g)?.length,
|
||||
3,
|
||||
);
|
||||
assert.equal(resourceJob.match(/--user 65532:65532/g)?.length, 3);
|
||||
assert.equal(
|
||||
resourceJob.match(/--expected-arch=\$\{\{ matrix\.arch \}\}/g)?.length,
|
||||
3,
|
||||
);
|
||||
for (const expected of [
|
||||
'--memory=128m\n --memory-swap=128m\n --cpus=0.5\n --pids-limit=64',
|
||||
'--memory=256m\n --memory-swap=256m\n --cpus=1\n --pids-limit=128',
|
||||
'--memory=512m\n --memory-swap=512m\n --cpus=2\n --pids-limit=256',
|
||||
]) {
|
||||
assert.ok(
|
||||
resourceJob.includes(expected),
|
||||
`missing exact envelope ${expected}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('uploads one strict source-bound evidence bundle per native architecture', () => {
|
||||
const workflow = fs.readFileSync(workflowPath, 'utf8');
|
||||
const resourceJob = workflow.match(
|
||||
/ linux-resource-envelopes:\n([\s\S]*?)\n linux-resource-release-evidence:/,
|
||||
)?.[1];
|
||||
assert.ok(resourceJob);
|
||||
for (const tier of [
|
||||
'router-stress-ci',
|
||||
'edge-release-ci',
|
||||
'cluster-control-ci',
|
||||
]) {
|
||||
assert.equal(
|
||||
resourceJob.match(
|
||||
new RegExp(
|
||||
`> "\\$\\{RUNNER_TEMP\\}/ql3-linux-resource-evidence/${tier}\\.json"`,
|
||||
'g',
|
||||
),
|
||||
)?.length,
|
||||
1,
|
||||
`${tier} raw evidence must be captured exactly once`,
|
||||
);
|
||||
assert.match(
|
||||
resourceJob,
|
||||
new RegExp(
|
||||
`--${tier}="\\$\\{RUNNER_TEMP\\}/ql3-linux-resource-evidence/${tier}\\.json"`,
|
||||
),
|
||||
);
|
||||
}
|
||||
assert.match(resourceJob, /--mode=bundle/);
|
||||
assert.match(resourceJob, /--repository="\$\{SOURCE_REPOSITORY\}"/);
|
||||
assert.match(resourceJob, /--revision="\$\{SOURCE_REVISION\}"/);
|
||||
assert.match(resourceJob, /--workflow="\$\{SOURCE_WORKFLOW\}"/);
|
||||
assert.match(resourceJob, /--run-id="\$\{SOURCE_RUN_ID\}"/);
|
||||
assert.match(resourceJob, /--run-attempt="\$\{SOURCE_RUN_ATTEMPT\}"/);
|
||||
assert.match(resourceJob, /--architecture="\$\{\{ matrix\.arch \}\}"/);
|
||||
assert.equal(
|
||||
resourceJob.match(
|
||||
/actions\/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a/g,
|
||||
)?.length,
|
||||
1,
|
||||
);
|
||||
assert.match(
|
||||
resourceJob,
|
||||
/name: ql3-linux-resource-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}-\$\{\{ matrix\.arch \}\}/,
|
||||
);
|
||||
assert.match(resourceJob, /if-no-files-found: error/);
|
||||
assert.match(resourceJob, /overwrite: false/);
|
||||
assert.doesNotMatch(resourceJob, /continue-on-error/);
|
||||
});
|
||||
|
||||
test('merges exact x64 and arm64 artifacts into one release evidence artifact', () => {
|
||||
const workflow = fs.readFileSync(workflowPath, 'utf8');
|
||||
const releaseJob = workflow.match(
|
||||
/ linux-resource-release-evidence:\n([\s\S]*?)\n supply-chain:/,
|
||||
)?.[1];
|
||||
assert.ok(releaseJob, 'linux-resource-release-evidence job is missing');
|
||||
assert.match(releaseJob, /needs: linux-resource-envelopes/);
|
||||
assert.equal(
|
||||
releaseJob.match(
|
||||
/actions\/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c/g,
|
||||
)?.length,
|
||||
2,
|
||||
);
|
||||
for (const architecture of ['x64', 'arm64']) {
|
||||
assert.match(
|
||||
releaseJob,
|
||||
new RegExp(
|
||||
`name: ql3-linux-resource-\\$\\{\\{ github\\.run_id \\}\\}-\\$\\{\\{ github\\.run_attempt \\}\\}-${architecture}`,
|
||||
),
|
||||
);
|
||||
assert.match(
|
||||
releaseJob,
|
||||
new RegExp(
|
||||
`path: \\$\\{\\{ runner\\.temp \\}\\}/ql3-linux-resource-evidence/${architecture}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
assert.match(releaseJob, /--mode=merge/);
|
||||
assert.match(
|
||||
releaseJob,
|
||||
/--x64="\$\{RUNNER_TEMP\}\/ql3-linux-resource-evidence\/x64\/x64\.json"/,
|
||||
);
|
||||
assert.match(
|
||||
releaseJob,
|
||||
/--arm64="\$\{RUNNER_TEMP\}\/ql3-linux-resource-evidence\/arm64\/arm64\.json"/,
|
||||
);
|
||||
assert.match(
|
||||
releaseJob,
|
||||
/name: ql3-linux-resource-release-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}/,
|
||||
);
|
||||
assert.equal(
|
||||
releaseJob.match(
|
||||
/actions\/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a/g,
|
||||
)?.length,
|
||||
1,
|
||||
);
|
||||
assert.doesNotMatch(releaseJob, /pattern:|merge-multiple:|continue-on-error/);
|
||||
});
|
||||
|
||||
test('does not duplicate the legacy edge-only budget in backend matrices', () => {
|
||||
const workflow = fs.readFileSync(workflowPath, 'utf8');
|
||||
const backendJob = workflow.match(
|
||||
/ backend:\n([\s\S]*?)\n linux-resource-envelopes:/,
|
||||
)?.[1];
|
||||
assert.ok(backendJob);
|
||||
assert.doesNotMatch(backendJob, /ql3-linux-resource-gate/);
|
||||
assert.doesNotMatch(backendJob, /Enforce 256 MiB edge process budget/);
|
||||
});
|
||||
@@ -0,0 +1,225 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const test = require('node:test');
|
||||
|
||||
const {
|
||||
auditLocalImageContract,
|
||||
} = require('../../scripts/ql3-local-image-audit.cjs');
|
||||
|
||||
const root = path.resolve(__dirname, '../..');
|
||||
const source = path.join(root, 'deploy/containers/ql3-local-application');
|
||||
|
||||
function fixture() {
|
||||
const temporaryRoot = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), 'ql3-local-image-audit-'),
|
||||
);
|
||||
const target = path.join(
|
||||
temporaryRoot,
|
||||
'deploy/containers/ql3-local-application',
|
||||
);
|
||||
fs.mkdirSync(path.dirname(target), { recursive: true });
|
||||
fs.cpSync(source, target, { recursive: true });
|
||||
const workflowTarget = path.join(
|
||||
temporaryRoot,
|
||||
'.github/workflows/ql3-ci.yml',
|
||||
);
|
||||
fs.mkdirSync(path.dirname(workflowTarget), { recursive: true });
|
||||
fs.copyFileSync(
|
||||
path.join(root, '.github/workflows/ql3-ci.yml'),
|
||||
workflowTarget,
|
||||
);
|
||||
return {
|
||||
root: temporaryRoot,
|
||||
target,
|
||||
close() {
|
||||
fs.rmSync(temporaryRoot, { recursive: true, force: true });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('accepts the exact AI-excluded local application image contract', () => {
|
||||
const report = auditLocalImageContract(root);
|
||||
assert.equal(report.compatible, true);
|
||||
assert.deepEqual(report.findings, []);
|
||||
assert.deepEqual(report.runtimePackages, [
|
||||
'@qinglong/local-admin',
|
||||
'@qinglong/local-application',
|
||||
'@qinglong/local-command-file',
|
||||
'@qinglong/local-execution',
|
||||
'@qinglong/local-process',
|
||||
'@qinglong/local-secret',
|
||||
'@qinglong/local-sqlite',
|
||||
'@qinglong/runtime-core',
|
||||
'croner',
|
||||
'semver',
|
||||
]);
|
||||
});
|
||||
|
||||
test('rejects a mutable or build-argument-controlled base image', () => {
|
||||
const current = fixture();
|
||||
try {
|
||||
const dockerfilePath = path.join(current.target, 'Dockerfile');
|
||||
const dockerfile = fs.readFileSync(dockerfilePath, 'utf8');
|
||||
fs.writeFileSync(
|
||||
dockerfilePath,
|
||||
`ARG NODE_IMAGE=node:24-bookworm-slim\n${dockerfile.replaceAll(
|
||||
/node:24\.18\.0-bookworm-slim@sha256:[0-9a-f]{64}/g,
|
||||
'${NODE_IMAGE}',
|
||||
)}`,
|
||||
);
|
||||
const report = auditLocalImageContract(current.root);
|
||||
assert.equal(report.compatible, false);
|
||||
assert.ok(
|
||||
report.findings.some(
|
||||
({ code }) => code === 'BASE_IMAGE_NOT_EXACTLY_PINNED',
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
report.findings.some(
|
||||
({ code }) => code === 'BASE_IMAGE_OVERRIDE_AUTHORITY',
|
||||
),
|
||||
);
|
||||
} finally {
|
||||
current.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects AI or an unreviewed dependency in the runtime closure', () => {
|
||||
const current = fixture();
|
||||
try {
|
||||
const dockerfilePath = path.join(current.target, 'Dockerfile');
|
||||
fs.appendFileSync(
|
||||
dockerfilePath,
|
||||
'\nCOPY --from=workspace /workspace/packages/ql3-ai/dist node_modules/@qinglong/ai/dist\n',
|
||||
);
|
||||
const runtimeManifestPath = path.join(
|
||||
current.target,
|
||||
'runtime-dependencies/package.json',
|
||||
);
|
||||
const runtimeManifest = JSON.parse(
|
||||
fs.readFileSync(runtimeManifestPath, 'utf8'),
|
||||
);
|
||||
runtimeManifest.dependencies['drizzle-orm'] = '1.0.0-rc.4';
|
||||
fs.writeFileSync(
|
||||
runtimeManifestPath,
|
||||
`${JSON.stringify(runtimeManifest, null, 2)}\n`,
|
||||
);
|
||||
const report = auditLocalImageContract(current.root);
|
||||
assert.equal(report.compatible, false);
|
||||
assert.ok(
|
||||
report.findings.some(
|
||||
({ code }) => code === 'AI_PRESENT_IN_RUNTIME_STAGE',
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
report.findings.some(({ code }) => code === 'RUNTIME_DEPENDENCY_DRIFT'),
|
||||
);
|
||||
} finally {
|
||||
current.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects retaining npm bin links, debug maps or declarations in the production image', () => {
|
||||
const current = fixture();
|
||||
try {
|
||||
const dockerfilePath = path.join(current.target, 'Dockerfile');
|
||||
const dockerfile = fs
|
||||
.readFileSync(dockerfilePath, 'utf8')
|
||||
.replace(
|
||||
'RUN rm -rf node_modules/.bin \\\n' +
|
||||
' && node /tmp/ql3-prune-runtime-artifact.cjs node_modules/@qinglong \\\n' +
|
||||
' @qinglong/local-application \\\n' +
|
||||
' @qinglong/local-application/process \\\n' +
|
||||
' @qinglong/local-application/plugin-package-recovery-catalog \\\n' +
|
||||
' --exclude=@qinglong/ai \\\n' +
|
||||
' && rm /tmp/ql3-prune-runtime-artifact.cjs\n\n',
|
||||
'',
|
||||
);
|
||||
fs.writeFileSync(dockerfilePath, dockerfile);
|
||||
const report = auditLocalImageContract(current.root);
|
||||
assert.equal(report.compatible, false);
|
||||
assert.ok(
|
||||
report.findings.some(
|
||||
({ code }) => code === 'RUNTIME_NONESSENTIAL_FILES_NOT_REMOVED',
|
||||
),
|
||||
);
|
||||
} finally {
|
||||
current.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects removal of the SQLite rollout compatibility labels', () => {
|
||||
const current = fixture();
|
||||
try {
|
||||
const dockerfilePath = path.join(current.target, 'Dockerfile');
|
||||
const dockerfile = fs
|
||||
.readFileSync(dockerfilePath, 'utf8')
|
||||
.replace(' io.qinglong.local.sqlite-write-contract="43" \\\n', '');
|
||||
fs.writeFileSync(dockerfilePath, dockerfile);
|
||||
const report = auditLocalImageContract(current.root);
|
||||
assert.equal(report.compatible, false);
|
||||
assert.ok(
|
||||
report.findings.some(
|
||||
({ code }) => code === 'RUNTIME_IDENTITY_OR_LABEL_DRIFT',
|
||||
),
|
||||
);
|
||||
} finally {
|
||||
current.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects runtime lifecycle scripts or closure lock drift', () => {
|
||||
const current = fixture();
|
||||
try {
|
||||
const lockPath = path.join(
|
||||
current.target,
|
||||
'runtime-dependencies/package-lock.json',
|
||||
);
|
||||
const lock = JSON.parse(fs.readFileSync(lockPath, 'utf8'));
|
||||
lock.packages['node_modules/croner'].hasInstallScript = true;
|
||||
lock.packages['node_modules/unreviewed'] = {
|
||||
version: '1.0.0',
|
||||
resolved: 'file:../unreviewed',
|
||||
integrity: 'sha512-invalid',
|
||||
};
|
||||
fs.writeFileSync(lockPath, `${JSON.stringify(lock, null, 2)}\n`);
|
||||
const report = auditLocalImageContract(current.root);
|
||||
assert.equal(report.compatible, false);
|
||||
assert.ok(
|
||||
report.findings.some(({ code }) => code === 'LOCKED_PACKAGE_UNSAFE'),
|
||||
);
|
||||
assert.ok(
|
||||
report.findings.some(({ code }) => code === 'RUNTIME_LOCK_CLOSURE_DRIFT'),
|
||||
);
|
||||
} finally {
|
||||
current.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects removal of either Profile from the native image CI gate', () => {
|
||||
const current = fixture();
|
||||
try {
|
||||
const workflowPath = path.join(
|
||||
current.root,
|
||||
'.github/workflows/ql3-ci.yml',
|
||||
);
|
||||
const workflow = fs
|
||||
.readFileSync(workflowPath, 'utf8')
|
||||
.replace(
|
||||
' node scripts/ql3-local-image-live-contract.cjs --image="${IMAGE}" --profile=standalone\n',
|
||||
'',
|
||||
);
|
||||
fs.writeFileSync(workflowPath, workflow);
|
||||
const report = auditLocalImageContract(current.root);
|
||||
assert.equal(report.compatible, false);
|
||||
assert.ok(
|
||||
report.findings.some(
|
||||
({ code }) => code === 'LOCAL_IMAGE_CI_CONTRACT_DRIFT',
|
||||
),
|
||||
);
|
||||
} finally {
|
||||
current.close();
|
||||
}
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,180 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
MAX_REVIEW_BYTES,
|
||||
ROW_COUNT,
|
||||
buildReport,
|
||||
commandFixture,
|
||||
normalizeManifest,
|
||||
parseArguments,
|
||||
validateReport,
|
||||
} = require('../../scripts/ql3-physical-edge-adoption-scale.cjs');
|
||||
|
||||
function manifest(overrides = {}) {
|
||||
return normalizeManifest({
|
||||
schemaVersion: 1,
|
||||
evidenceClass: 'physical_edge_adoption_scale_candidate',
|
||||
profile: 'edge',
|
||||
deviceId: 'router-a1',
|
||||
expectedArchitecture: 'arm64',
|
||||
expectedFilesystem: 'ext4',
|
||||
rowCount: 100_000,
|
||||
maxReviewFileBytes: 32 * 1024 * 1024,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
function command(operation, options) {
|
||||
return { schemaVersion: 1, operation, options };
|
||||
}
|
||||
|
||||
function storage() {
|
||||
return { logicalBytes: 4096, allocatedBytes: 4096, files: [] };
|
||||
}
|
||||
|
||||
test('fixes the physical adoption workload at 100000 reviewed rows', () => {
|
||||
assert.equal(manifest().rowCount, ROW_COUNT);
|
||||
assert.equal(manifest().maxReviewFileBytes, MAX_REVIEW_BYTES);
|
||||
assert.throws(() => manifest({ rowCount: 10_000 }), /fixed workload/);
|
||||
assert.deepEqual(
|
||||
parseArguments([
|
||||
'--manifest=/data/manifest.json',
|
||||
'--data-path=/data',
|
||||
'--issue-command=/data/issue.json',
|
||||
'--commit-command=/data/commit.json',
|
||||
'--output=/data/report.json',
|
||||
'--json',
|
||||
]),
|
||||
{
|
||||
manifestPath: '/data/manifest.json',
|
||||
dataPath: '/data',
|
||||
issueCommandPath: '/data/issue.json',
|
||||
commitCommandPath: '/data/commit.json',
|
||||
outputPath: '/data/report.json',
|
||||
json: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts only one contained issue and commit command pair', () => {
|
||||
const shared = {
|
||||
deploymentRoot: '/data/adoption',
|
||||
profile: 'edge',
|
||||
sourcePath: '/data/adoption/legacy.sqlite',
|
||||
authorizationPath: '/data/adoption/authorization.ndjson',
|
||||
credentialFilePath: '/data/adoption/credential.json',
|
||||
issuerKeyringPath: '/data/adoption/issuer.json',
|
||||
ownerPepperKeyringDirectory: '/data/adoption/pepper',
|
||||
expectedPlanDigest: 'a'.repeat(64),
|
||||
};
|
||||
const issue = command('legacy-crontab.decision.issue', {
|
||||
...shared,
|
||||
databasePath: '/data/adoption/target.sqlite',
|
||||
reviewFilePath: '/data/adoption/review.ndjson',
|
||||
decisionId: '019a2b3c-4d5e-7f60-8123-456789abcdef',
|
||||
});
|
||||
const commit = command('legacy-crontab.adoption.commit', {
|
||||
...shared,
|
||||
targetPath: '/data/adoption/target.sqlite',
|
||||
expectedDecisionId: issue.options.decisionId,
|
||||
});
|
||||
assert.equal(
|
||||
commandFixture(issue, commit, '/data').targetPath,
|
||||
'/data/adoption/target.sqlite',
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
commandFixture(
|
||||
issue,
|
||||
{
|
||||
...commit,
|
||||
options: { ...commit.options, sourcePath: '/tmp/other.sqlite' },
|
||||
},
|
||||
'/data',
|
||||
),
|
||||
/do not describe one adoption/,
|
||||
);
|
||||
});
|
||||
|
||||
test('builds digest-bound evidence without overstating flash or power loss', () => {
|
||||
const measurement = {
|
||||
durationMs: 100,
|
||||
peakRssBytes: 30_000_000,
|
||||
sampleCount: 10,
|
||||
readBytes: 4096,
|
||||
writeBytes: 8192,
|
||||
cancelledWriteBytes: 0,
|
||||
exitCode: 0,
|
||||
};
|
||||
const report = buildReport({
|
||||
manifest: manifest(),
|
||||
observed: {
|
||||
platform: 'linux',
|
||||
architecture: 'arm64',
|
||||
node: 'v24.18.0',
|
||||
bootId: 'boot-a',
|
||||
dataPath: '/data',
|
||||
dataFilesystem: 'ext4',
|
||||
dataMountOptions: ['rw'],
|
||||
virtualizationIndicators: [],
|
||||
},
|
||||
preflight: {
|
||||
sourceRowCount: 100_000,
|
||||
reviewFileBytes: 20_000_000,
|
||||
targetLedgerCount: 0,
|
||||
targetStorage: storage(),
|
||||
},
|
||||
issue: measurement,
|
||||
commit: measurement,
|
||||
final: {
|
||||
ledgerCount: 1,
|
||||
adoptedTaskCount: 100_000,
|
||||
adoptedTriggerCount: 100_000,
|
||||
targetStorage: storage(),
|
||||
},
|
||||
generatedAt: '2026-07-22T00:00:00.000Z',
|
||||
});
|
||||
assert.equal(report.qualification.passed, true);
|
||||
assert.equal(report.supported, false);
|
||||
assert.equal(report.sha256.length, 64);
|
||||
assert.ok(
|
||||
report.qualification.doesNotProve.includes(
|
||||
'whole_device_flash_or_nand_write_amplification',
|
||||
),
|
||||
);
|
||||
assert.deepEqual(
|
||||
validateReport(report, manifest(), {
|
||||
architecture: 'arm64',
|
||||
bootId: 'boot-a',
|
||||
dataPath: '/data',
|
||||
dataFilesystem: 'ext4',
|
||||
dataMountOptions: ['rw'],
|
||||
}),
|
||||
[],
|
||||
);
|
||||
const tamperedBody = {
|
||||
...report,
|
||||
workload: {
|
||||
...report.workload,
|
||||
widened: true,
|
||||
},
|
||||
};
|
||||
const { sha256: _sha256, ...body } = tamperedBody;
|
||||
const tampered = {
|
||||
...body,
|
||||
sha256: require('node:crypto')
|
||||
.createHash('sha256')
|
||||
.update(JSON.stringify(body))
|
||||
.digest('hex'),
|
||||
};
|
||||
assert.match(
|
||||
validateReport(tampered, manifest(), {
|
||||
architecture: 'arm64',
|
||||
bootId: 'boot-a',
|
||||
dataPath: '/data',
|
||||
dataFilesystem: 'ext4',
|
||||
dataMountOptions: ['rw'],
|
||||
}).join('; '),
|
||||
/incomplete/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,399 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const crypto = require('node:crypto');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
buildApplicationStartReport,
|
||||
collectArtifactIdentity,
|
||||
normalizeApplicationStartManifest,
|
||||
normalizeSession,
|
||||
parseArguments,
|
||||
parseEventLines,
|
||||
preflightArtifactMetadata,
|
||||
validateApplicationStartReport,
|
||||
validateArtifactAgainstManifest,
|
||||
validateBootObservation,
|
||||
} = require('../../scripts/ql3-physical-edge-application-start.cjs');
|
||||
const {
|
||||
canonicalDigest,
|
||||
} = require('../../scripts/ql3-physical-edge-evidence.cjs');
|
||||
|
||||
const packages = [
|
||||
'@qinglong/local-admin',
|
||||
'@qinglong/local-application',
|
||||
'@qinglong/local-command-file',
|
||||
'@qinglong/local-execution',
|
||||
'@qinglong/local-process',
|
||||
'@qinglong/local-secret',
|
||||
'@qinglong/local-sqlite',
|
||||
'@qinglong/runtime-core',
|
||||
'croner',
|
||||
'semver',
|
||||
];
|
||||
|
||||
function writePrivate(filePath, contents) {
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
|
||||
fs.writeFileSync(filePath, contents, { mode: 0o600 });
|
||||
}
|
||||
|
||||
function artifactFixture(t) {
|
||||
const root = fs.realpathSync(
|
||||
fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-application-artifact-')),
|
||||
);
|
||||
fs.chmodSync(root, 0o700);
|
||||
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
|
||||
for (const packageName of packages) {
|
||||
const packageRoot = path.join(
|
||||
root,
|
||||
'node_modules',
|
||||
...packageName.split('/'),
|
||||
);
|
||||
const manifest =
|
||||
packageName === '@qinglong/local-application'
|
||||
? {
|
||||
name: packageName,
|
||||
version: '3.0.0-alpha.0',
|
||||
engines: { node: '>=24.18.0 <25' },
|
||||
bin: { 'ql3-local-application': 'dist/cli.js' },
|
||||
}
|
||||
: { name: packageName, version: '1.0.0' };
|
||||
writePrivate(
|
||||
path.join(packageRoot, 'package.json'),
|
||||
`${JSON.stringify(manifest)}\n`,
|
||||
);
|
||||
writePrivate(path.join(packageRoot, 'dist', 'index.js'), `'use strict';\n`);
|
||||
}
|
||||
writePrivate(
|
||||
path.join(
|
||||
root,
|
||||
'node_modules',
|
||||
'@qinglong',
|
||||
'local-application',
|
||||
'dist',
|
||||
'cli.js',
|
||||
),
|
||||
`#!/usr/bin/env node\n'use strict';\n`,
|
||||
);
|
||||
return { root, identity: collectArtifactIdentity(root) };
|
||||
}
|
||||
|
||||
function manifest(artifact, overrides = {}) {
|
||||
return normalizeApplicationStartManifest({
|
||||
schemaVersion: 1,
|
||||
evidenceClass: 'physical_edge_application_start_candidate',
|
||||
profile: 'edge',
|
||||
deviceId: 'router-a1',
|
||||
expectedArchitecture: 'arm64',
|
||||
expectedFilesystem: 'ext4',
|
||||
expectedArtifactSha256: artifact.artifactSha256,
|
||||
expectedArtifactFiles: artifact.artifactFiles,
|
||||
expectedArtifactBytes: artifact.artifactBytes,
|
||||
expectedNodeSha256: 'b'.repeat(64),
|
||||
maximumBootAgeMs: 180_000,
|
||||
maximumFirstActiveMs: 30_000,
|
||||
maximumSampledRssBytes: 256 * 1024 * 1024,
|
||||
sampleIntervalMs: 10,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
function boot(bootId, bootAgeMs = 1000, overrides = {}) {
|
||||
return {
|
||||
platform: 'linux',
|
||||
architecture: 'arm64',
|
||||
bootId,
|
||||
dataFilesystem: 'ext4',
|
||||
nodeExecutable: '/usr/bin/node',
|
||||
nodeSha256: 'b'.repeat(64),
|
||||
nodeVersion: 'v24.18.0',
|
||||
bootAgeMs,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function sessionFixture(artifact) {
|
||||
const sessionId = '019f0000-0000-4000-8000-000000000010';
|
||||
const dataPath = '/mnt/ql3-evidence';
|
||||
const deploymentRoot = path.join(
|
||||
dataPath,
|
||||
`.ql3-application-start-${sessionId}`,
|
||||
);
|
||||
const artifactRoot = '/opt/qinglong3-release';
|
||||
const body = {
|
||||
schemaVersion: 1,
|
||||
evidenceClass: 'physical_edge_application_start_session',
|
||||
sessionId,
|
||||
manifestDigest: canonicalDigest(manifest(artifact)),
|
||||
uid: 1000,
|
||||
preparedAt: '2026-07-29T00:00:00.000Z',
|
||||
artifact,
|
||||
environment: boot('019f0000-0000-4000-8000-000000000001', 50_000),
|
||||
paths: {
|
||||
dataPath,
|
||||
deploymentRoot,
|
||||
artifactRoot,
|
||||
applicationEntrypoint: path.join(
|
||||
artifactRoot,
|
||||
'node_modules',
|
||||
'@qinglong',
|
||||
'local-application',
|
||||
'dist',
|
||||
'cli.js',
|
||||
),
|
||||
applicationConfig: path.join(deploymentRoot, 'local-application.json'),
|
||||
},
|
||||
};
|
||||
return { ...body, sha256: canonicalDigest(body) };
|
||||
}
|
||||
|
||||
function reportFixture(artifact, overrides = {}) {
|
||||
const session = normalizeSession(sessionFixture(artifact));
|
||||
return buildApplicationStartReport({
|
||||
manifest: manifest(artifact),
|
||||
session,
|
||||
observed: {
|
||||
before: session.environment,
|
||||
after: boot('019f0000-0000-4000-8000-000000000002'),
|
||||
artifact,
|
||||
},
|
||||
measurements: {
|
||||
firstActiveMs: 1200,
|
||||
maximumSampledRssBytes: 80 * 1024 * 1024,
|
||||
processReadBytes: 4096,
|
||||
processWriteBytes: 8192,
|
||||
sampleCount: 20,
|
||||
eventCount: 5,
|
||||
},
|
||||
outcomes: {
|
||||
activeEventCount: 1,
|
||||
aiStatus: 'deployment_excluded',
|
||||
gracefulStop: true,
|
||||
exitCode: 0,
|
||||
exitSignal: null,
|
||||
stderrBytes: 0,
|
||||
sqliteContractVersion: 41,
|
||||
},
|
||||
generatedAt: '2026-07-29T00:01:00.000Z',
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
test('normalizes exact Edge application start budgets', () => {
|
||||
const artifact = {
|
||||
artifactSha256: 'a'.repeat(64),
|
||||
artifactMetadataSha256: 'd'.repeat(64),
|
||||
artifactFiles: 100,
|
||||
artifactBytes: 2 * 1024 * 1024,
|
||||
};
|
||||
assert.equal(manifest(artifact).maximumBootAgeMs, 180_000);
|
||||
assert.throws(
|
||||
() => manifest(artifact, { maximumBootAgeMs: 601_000 }),
|
||||
/measurement budget/,
|
||||
);
|
||||
assert.throws(
|
||||
() => manifest(artifact, { maximumSampledRssBytes: 1 }),
|
||||
/measurement budget/,
|
||||
);
|
||||
});
|
||||
|
||||
test('requires phase-specific absolute paths', () => {
|
||||
assert.deepEqual(
|
||||
parseArguments([
|
||||
'inspect',
|
||||
'--artifact-root=/opt/qinglong3-release',
|
||||
'--json',
|
||||
]),
|
||||
{
|
||||
phase: 'inspect',
|
||||
artifactRoot: '/opt/qinglong3-release',
|
||||
json: true,
|
||||
},
|
||||
);
|
||||
assert.equal(
|
||||
parseArguments([
|
||||
'prepare',
|
||||
'--manifest=/mnt/data/manifest.json',
|
||||
'--data-path=/mnt/data',
|
||||
'--artifact-root=/opt/qinglong3-release',
|
||||
'--session=/mnt/data/session.json',
|
||||
]).phase,
|
||||
'prepare',
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
parseArguments([
|
||||
'resume',
|
||||
'--manifest=manifest.json',
|
||||
'--session=/data/session.json',
|
||||
'--output=/data/report.json',
|
||||
]),
|
||||
/manifestPath must be absolute/,
|
||||
);
|
||||
});
|
||||
|
||||
test('hashes one exact AI-excluded native release closure', (t) => {
|
||||
const fixture = artifactFixture(t);
|
||||
assert.deepEqual(fixture.identity.artifact.packages, packages);
|
||||
assert.match(fixture.identity.artifact.artifactSha256, /^[a-f0-9]{64}$/);
|
||||
assert.equal(
|
||||
fixture.identity.applicationEntrypoint.endsWith('/dist/cli.js'),
|
||||
true,
|
||||
);
|
||||
assert.deepEqual(
|
||||
validateArtifactAgainstManifest(
|
||||
manifest(fixture.identity.artifact),
|
||||
fixture.identity.artifact,
|
||||
{ nodeSha256: 'b'.repeat(64) },
|
||||
),
|
||||
[],
|
||||
);
|
||||
writePrivate(
|
||||
path.join(fixture.root, 'node_modules', 'unexpected', 'package.json'),
|
||||
'{}\n',
|
||||
);
|
||||
assert.throws(
|
||||
() => collectArtifactIdentity(fixture.root),
|
||||
/package closure is invalid/,
|
||||
);
|
||||
});
|
||||
|
||||
test('detects cross-boot artifact metadata drift without reading file content', (t) => {
|
||||
const fixture = artifactFixture(t);
|
||||
const entrypoint = fixture.identity.applicationEntrypoint;
|
||||
const before = preflightArtifactMetadata(fixture.root, entrypoint);
|
||||
assert.equal(
|
||||
before.artifactMetadataSha256,
|
||||
fixture.identity.artifact.artifactMetadataSha256,
|
||||
);
|
||||
fs.chmodSync(entrypoint, 0o400);
|
||||
const after = preflightArtifactMetadata(fixture.root, entrypoint);
|
||||
assert.notEqual(
|
||||
after.artifactMetadataSha256,
|
||||
fixture.identity.artifact.artifactMetadataSha256,
|
||||
);
|
||||
});
|
||||
|
||||
test('binds the pre-reboot session to deterministic deployment paths', () => {
|
||||
const artifact = {
|
||||
artifactSha256: 'a'.repeat(64),
|
||||
artifactMetadataSha256: 'd'.repeat(64),
|
||||
artifactFiles: 100,
|
||||
artifactBytes: 2 * 1024 * 1024,
|
||||
entrypointSha256: 'c'.repeat(64),
|
||||
packages,
|
||||
};
|
||||
const session = normalizeSession(sessionFixture(artifact));
|
||||
assert.equal(session.environment.bootAgeMs, 50_000);
|
||||
const escaped = sessionFixture(artifact);
|
||||
escaped.paths.deploymentRoot = '/opt/qinglong3';
|
||||
const { sha256: ignored, ...body } = escaped;
|
||||
assert.throws(
|
||||
() => normalizeSession({ ...body, sha256: canonicalDigest(body) }),
|
||||
/session is invalid or drifted/,
|
||||
);
|
||||
});
|
||||
|
||||
test('parses bounded production active events', () => {
|
||||
const events = [];
|
||||
const remaining = parseEventLines(
|
||||
`${JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-local-application',
|
||||
level: 'info',
|
||||
event: 'active',
|
||||
instanceId: 'edge-a',
|
||||
profile: 'edge',
|
||||
aiStatus: 'deployment_excluded',
|
||||
})}\npartial`,
|
||||
events,
|
||||
);
|
||||
assert.equal(remaining, 'partial');
|
||||
assert.equal(events[0].event, 'active');
|
||||
assert.throws(
|
||||
() => parseEventLines(`${'x'.repeat(4097)}\n`, []),
|
||||
/event line exceeded/,
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts only different-boot bounded native application activation', () => {
|
||||
const artifact = {
|
||||
artifactSha256: 'a'.repeat(64),
|
||||
artifactMetadataSha256: 'd'.repeat(64),
|
||||
artifactFiles: 100,
|
||||
artifactBytes: 2 * 1024 * 1024,
|
||||
entrypointSha256: 'c'.repeat(64),
|
||||
packages,
|
||||
};
|
||||
const report = reportFixture(artifact);
|
||||
assert.equal(report.supported, false);
|
||||
assert.equal(report.qualification.passed, true);
|
||||
assert.ok(
|
||||
report.qualification.doesNotProve.includes(
|
||||
'cold_node_runtime_or_dynamic_linker_cache',
|
||||
),
|
||||
);
|
||||
assert.deepEqual(
|
||||
validateApplicationStartReport(report, manifest(artifact), {
|
||||
bootId: report.observed.after.bootId,
|
||||
architecture: 'arm64',
|
||||
dataFilesystem: 'ext4',
|
||||
dataPath: '/mnt/ql3-evidence',
|
||||
}),
|
||||
[],
|
||||
);
|
||||
assert.deepEqual(
|
||||
validateBootObservation(
|
||||
manifest(artifact),
|
||||
report.observed.after,
|
||||
'/mnt/ql3-evidence',
|
||||
),
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
test('fails same boot, latency, sampled RSS and lifecycle drift', () => {
|
||||
const artifact = {
|
||||
artifactSha256: 'a'.repeat(64),
|
||||
artifactMetadataSha256: 'd'.repeat(64),
|
||||
artifactFiles: 100,
|
||||
artifactBytes: 2 * 1024 * 1024,
|
||||
entrypointSha256: 'c'.repeat(64),
|
||||
packages,
|
||||
};
|
||||
const session = normalizeSession(sessionFixture(artifact));
|
||||
const report = buildApplicationStartReport({
|
||||
manifest: manifest(artifact),
|
||||
session,
|
||||
observed: {
|
||||
before: session.environment,
|
||||
after: session.environment,
|
||||
artifact,
|
||||
},
|
||||
measurements: {
|
||||
firstActiveMs: 31_000,
|
||||
maximumSampledRssBytes: 300 * 1024 * 1024,
|
||||
processReadBytes: 0,
|
||||
processWriteBytes: 0,
|
||||
sampleCount: 1,
|
||||
eventCount: 1,
|
||||
},
|
||||
outcomes: {
|
||||
activeEventCount: 0,
|
||||
aiStatus: 'active',
|
||||
gracefulStop: false,
|
||||
exitCode: 1,
|
||||
exitSignal: null,
|
||||
stderrBytes: 1,
|
||||
sqliteContractVersion: 34,
|
||||
},
|
||||
generatedAt: '2026-07-29T00:01:00.000Z',
|
||||
});
|
||||
assert.equal(report.qualification.passed, false);
|
||||
assert.match(
|
||||
report.qualification.violations.join('; '),
|
||||
/reboot boundary.*measurement budget.*lifecycle outcome/,
|
||||
);
|
||||
});
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user