feat(ql3): establish 3.0 incubation baseline

This commit is contained in:
whyour
2026-08-12 00:25:26 +08:00
parent 4bf92dcfeb
commit c699c32461
2817 changed files with 779642 additions and 653 deletions
@@ -0,0 +1,82 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
ApiCredentialUnavailableError,
InvalidApiCredentialValueError,
} = require('@qinglong/runtime-core/api-credential');
const {
PostgresApiCredentialRepository,
} = require('@qinglong/cluster-postgres/runtime');
function row(overrides = {}) {
return {
credentialId: 'app_primary',
version: '2',
state: 'active',
subjectType: 'api_app',
subjectId: 'app_primary',
subjectStatus: 'active',
pepperKeyId: 'legacy-v1',
secretDigest: 'a'.repeat(64),
createdAtMs: '100',
notBeforeAtMs: '100',
expiresAtMs: '1000',
...overrides,
};
}
test('resolves only the latest normalized credential and stable subject', async () => {
const calls = [];
const repository = new PostgresApiCredentialRepository({
async query(sql, values) {
calls.push({ sql, values });
return { rows: [row()] };
},
});
assert.deepEqual(await repository.resolve('app_primary'), {
credentialId: 'app_primary',
version: 2,
pepperKeyId: 'legacy-v1',
state: 'active',
subject: { type: 'api_app', id: 'app_primary' },
subjectStatus: 'active',
secretDigest: 'a'.repeat(64),
createdAtMs: 100,
notBeforeAtMs: 100,
expiresAtMs: 1000,
});
assert.deepEqual(calls[0].values, ['app_primary']);
assert.match(calls[0].sql, /ORDER BY credential\.version DESC/);
assert.match(calls[0].sql, /identity_subjects/);
});
test('returns null for unknown credentials and rejects invalid ids before SQL', async () => {
let calls = 0;
const repository = new PostgresApiCredentialRepository({
async query() {
calls += 1;
return { rows: [] };
},
});
assert.equal(await repository.resolve('unknown'), null);
await assert.rejects(
repository.resolve('../escape'),
InvalidApiCredentialValueError,
);
assert.equal(calls, 1);
});
test('fails closed on corrupt rows and database errors', async () => {
for (const query of [
async () => ({ rows: [row({ secretDigest: 'corrupt' })] }),
async () => {
throw new Error('driver detail');
},
]) {
const repository = new PostgresApiCredentialRepository({ query });
await assert.rejects(
repository.resolve('app_primary'),
ApiCredentialUnavailableError,
);
}
});
@@ -0,0 +1,168 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
approvalRequestDigest,
createApprovalRequest,
} = require('@qinglong/runtime-core/approved-action');
const {
createToolInvocationPreviewArtifact,
} = require('@qinglong/runtime-core/tool-invocation-artifact');
const {
PostgresApprovalRequestSource,
} = require('@qinglong/cluster-postgres/approval-discovery');
function request(id, atMs) {
return createApprovalRequest({
id,
projectId: 'default',
action: {
permission: 'run.start',
actionType: 'tool.invoke',
actionRef: `tool:${id}`,
actionDigest: 'a'.repeat(64),
previewDigest: 'b'.repeat(64),
},
risk: 'medium',
decisionMode: 'separation_of_duty',
requestedBy: { type: 'agent', id: 'agent-planner' },
requestedAtMs: atMs,
expiresAtMs: atMs + 60_000,
requestFence: { projectVersion: 1, bindingVersion: 1 },
});
}
function row(value) {
return {
requestJson: value,
requestDigest: approvalRequestDigest(value),
updatedAtMs: value.requestedAtMs,
};
}
test('uses the Project descending keyset and returns limit plus one', async () => {
const calls = [];
const values = [request('approval-3', 3_000), request('approval-2', 2_000)];
const source = new PostgresApprovalRequestSource({
async query(sql, parameters) {
calls.push({ sql, parameters });
return { rows: values.map(row) };
},
});
const page = await source.listApprovalRequests({
projectId: 'default',
limit: 1,
after: { updatedAtMs: 4_000, requestId: 'approval-4' },
});
assert.deepEqual(calls[0].parameters, [
'default',
4_000,
'approval-4',
2,
]);
assert.match(calls[0].sql, /ORDER BY updated_at_ms DESC, request_id DESC/);
assert.deepEqual(page.requests.map(({ id }) => id), ['approval-3']);
assert.equal(page.truncated, true);
assert.deepEqual(page.next, {
updatedAtMs: 3_000,
requestId: 'approval-3',
});
});
test('fails closed on malformed rows and database errors', async () => {
const value = request('approval-1', 1_000);
for (const pool of [
{ async query() { return { rows: [{ ...row(value), updatedAtMs: 2_000 }] }; } },
{ async query() { throw new Error('private database detail'); } },
]) {
await assert.rejects(
new PostgresApprovalRequestSource(pool).listApprovalRequests({
projectId: 'default',
limit: 1,
}),
{ code: 'APPROVAL_UNAVAILABLE' },
);
}
});
test('reads one Approval detail through the exact Project and Artifact binding', async () => {
const previewArtifact = createToolInvocationPreviewArtifact({
artifactId: 'preview-1',
projectId: 'default',
actionRef: 'tool:approval-1',
actionDigest: 'c'.repeat(64),
redactionContractDigest: 'd'.repeat(64),
sealedAtMs: 1_000,
preview: {
title: 'Run task',
summary: 'Runs one selected task.',
fields: [{ kind: 'redacted', label: 'Token', value: null }],
warnings: ['external_effect'],
},
});
const approval = createApprovalRequest({
id: 'approval-1',
projectId: 'default',
action: {
permission: 'run.start',
actionType: 'tool.invoke',
actionRef: previewArtifact.actionRef,
actionDigest: previewArtifact.actionDigest,
previewDigest: previewArtifact.previewDigest,
},
risk: 'medium',
decisionMode: 'human_confirmation',
requestedBy: { type: 'agent', id: 'agent-planner' },
requestedAtMs: 1_000,
expiresAtMs: 61_000,
requestFence: { projectVersion: 1, bindingVersion: 1 },
});
const detailRow = {
...row(approval),
previewArtifactId: previewArtifact.artifactId,
previewProjectId: previewArtifact.projectId,
previewActionRef: previewArtifact.actionRef,
previewActionDigest: previewArtifact.actionDigest,
storedPreviewDigest: previewArtifact.previewDigest,
redactionContractDigest: previewArtifact.redactionContractDigest,
previewArtifactDigest: previewArtifact.artifactDigest,
previewByteLength: previewArtifact.byteLength,
previewSealedAtMs: previewArtifact.sealedAtMs,
previewArtifactJson: previewArtifact,
};
const calls = [];
const source = new PostgresApprovalRequestSource({
async query(sql, parameters) {
calls.push({ sql, parameters });
return { rows: [detailRow] };
},
});
const detail = await source.getApprovalRequestDetail({
projectId: 'default',
requestId: 'approval-1',
});
assert.deepEqual(calls[0].parameters, ['default', 'approval-1']);
assert.match(calls[0].sql, /LEFT JOIN "ql3"\."tool_invocation_preview_artifacts"/);
assert.equal(detail.request.id, 'approval-1');
assert.equal(detail.preview.title, 'Run task');
await assert.rejects(
new PostgresApprovalRequestSource({
async query() {
return { rows: [{ ...detailRow, previewByteLength: 1 }] };
},
}).getApprovalRequestDetail({ projectId: 'default', requestId: 'approval-1' }),
{ code: 'APPROVAL_UNAVAILABLE' },
);
});
test('exports only through the read authority subpath', () => {
const root = require('@qinglong/cluster-postgres');
const mutation = require('@qinglong/cluster-postgres/approved-action');
const discovery = require('@qinglong/cluster-postgres/approval-discovery');
assert.equal(root.PostgresApprovalRequestSource, undefined);
assert.equal(mutation.PostgresApprovalRequestSource, undefined);
assert.equal(
discovery.PostgresApprovalRequestSource,
PostgresApprovalRequestSource,
);
});
@@ -0,0 +1,131 @@
const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { test } = require('node:test');
const {
POSTGRES_CA_MAX_CERTIFICATES,
POSTGRES_CA_MAX_FILE_BYTES,
PostgresCertificateAuthorityFileError,
inspectPostgresCertificateAuthorityFile,
loadPostgresCertificateAuthorityFile,
} = require('@qinglong/cluster-postgres/runtime');
const FIXTURES = path.resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls',
);
const CA = fs.readFileSync(path.join(FIXTURES, 'ca-cert.pem'), 'utf8');
function temporaryDirectory(t) {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-pg-ca-'));
t.after(() => fs.rmSync(directory, { recursive: true, force: true }));
return directory;
}
function expectCode(callback, code) {
assert.throws(
callback,
(error) =>
error instanceof PostgresCertificateAuthorityFileError &&
error.code === code,
);
}
test('loads a bounded CA bundle through a projected-Secret symlink', (t) => {
const directory = temporaryDirectory(t);
const dataDirectory = path.join(directory, '..data');
fs.mkdirSync(dataDirectory);
fs.writeFileSync(path.join(dataDirectory, 'ca.crt'), CA, { mode: 0o444 });
fs.symlinkSync(path.join('..data', 'ca.crt'), path.join(directory, 'ca.crt'));
const bundle = loadPostgresCertificateAuthorityFile(
path.join(directory, 'ca.crt'),
);
assert.match(bundle, /^-----BEGIN CERTIFICATE-----/);
assert.match(bundle, /-----END CERTIFICATE-----\n$/);
const inspection = inspectPostgresCertificateAuthorityFile(
path.join(directory, 'ca.crt'),
);
assert.equal(inspection.bundle, bundle);
assert.equal(inspection.fingerprints256.length, 1);
assert.match(
inspection.fingerprints256[0],
/^(?:[0-9A-F]{2}:){31}[0-9A-F]{2}$/,
);
assert.equal(Object.isFrozen(inspection), true);
assert.equal(Object.isFrozen(inspection.fingerprints256), true);
});
test('rejects ambiguous paths, file types, permissions and sizes', (t) => {
const directory = temporaryDirectory(t);
expectCode(
() => loadPostgresCertificateAuthorityFile('relative-ca.pem'),
'QL3_POSTGRES_CA_INVALID_PATH',
);
expectCode(
() =>
loadPostgresCertificateAuthorityFile(path.join(directory, 'missing.pem')),
'QL3_POSTGRES_CA_UNAVAILABLE',
);
expectCode(
() => loadPostgresCertificateAuthorityFile(directory),
'QL3_POSTGRES_CA_NOT_REGULAR',
);
const writable = path.join(directory, 'writable.pem');
fs.writeFileSync(writable, CA, { mode: 0o666 });
fs.chmodSync(writable, 0o666);
expectCode(
() => loadPostgresCertificateAuthorityFile(writable),
'QL3_POSTGRES_CA_INSECURE_PERMISSIONS',
);
const oversized = path.join(directory, 'oversized.pem');
fs.writeFileSync(oversized, Buffer.alloc(POSTGRES_CA_MAX_FILE_BYTES + 1), {
mode: 0o444,
});
expectCode(
() => loadPostgresCertificateAuthorityFile(oversized),
'QL3_POSTGRES_CA_INVALID_SIZE',
);
});
test('rejects malformed, non-CA, duplicate and oversized bundles', (t) => {
const directory = temporaryDirectory(t);
const malformed = path.join(directory, 'malformed.pem');
fs.writeFileSync(malformed, `${CA}\nunreviewed`, { mode: 0o444 });
expectCode(
() => loadPostgresCertificateAuthorityFile(malformed),
'QL3_POSTGRES_CA_INVALID_PEM',
);
const nonCa = path.join(directory, 'non-ca.pem');
fs.copyFileSync(path.join(FIXTURES, 'server-cert.pem'), nonCa);
fs.chmodSync(nonCa, 0o444);
expectCode(
() => loadPostgresCertificateAuthorityFile(nonCa),
'QL3_POSTGRES_CA_NOT_CA',
);
const duplicate = path.join(directory, 'duplicate.pem');
fs.writeFileSync(duplicate, `${CA}\n${CA}`, { mode: 0o444 });
expectCode(
() => loadPostgresCertificateAuthorityFile(duplicate),
'QL3_POSTGRES_CA_DUPLICATE_CERTIFICATE',
);
const tooMany = path.join(directory, 'too-many.pem');
fs.writeFileSync(
tooMany,
Array.from({ length: POSTGRES_CA_MAX_CERTIFICATES + 1 }, () => CA).join(
'\n',
),
{ mode: 0o444 },
);
expectCode(
() => loadPostgresCertificateAuthorityFile(tooMany),
'QL3_POSTGRES_CA_TOO_MANY_CERTIFICATES',
);
});
@@ -0,0 +1,87 @@
const assert = require('node:assert/strict');
const { createHash } = require('node:crypto');
const { test } = require('node:test');
const {
PostgresClusterDispatchSource,
} = require('../dist/remote-execution/clusterDispatchRepository');
const SESSION = '018f0000-0000-7000-8000-000000000001';
function candidateRow(overrides = {}) {
return {
observedAtMs: '1000',
runId: 'run-1',
attemptId: 'attempt-1',
projectId: 'project-1',
taskId: 'task-1',
taskRevision: `qltd:v1:1:${'a'.repeat(64)}`,
priority: 4,
queuedAtMs: '100',
attemptCreatedAtMs: '101',
attemptNumber: 1,
executorType: 'remote_worker',
...overrides,
};
}
function leaseRow(overrides = {}) {
return candidateRow({
leaseStatus: 'leased',
leaseVersion: 0,
leaseGeneration: 1,
workerId: 'edge-1',
workerSessionId: SESSION,
workerGeneration: 2,
leaseTokenDigest: createHash('sha256').update('x'.repeat(32)).digest('hex'),
acquiredAtMs: '900',
renewedAtMs: '900',
expiresAtMs: '1900',
releasedAtMs: null,
releaseReason: null,
completedAtMs: null,
leaseUpdatedAtMs: '900',
workerCurrent: true,
...overrides,
});
}
test('lists a bounded PostgreSQL-clock candidate page with stable cursor parameters', async () => {
let observed;
const source = new PostgresClusterDispatchSource({
async query(sql, params) {
observed = { sql, params };
return { rows: [candidateRow(), candidateRow({ attemptId: 'attempt-2' })] };
},
});
const page = await source.listClusterDispatchCandidates({ limit: 1 });
assert.equal(page.observedAtMs, 1000);
assert.equal(page.candidates.length, 1);
assert.equal(page.truncated, true);
assert.equal(page.next.attemptId, 'attempt-1');
assert.deepEqual(observed.params, [null, null, null, null, 2]);
assert.match(observed.sql, /clock_timestamp\(\)/);
assert.match(observed.sql, /lease\.expires_at_ms <= observation\.observed_at_ms/);
assert.match(
observed.sql,
/plugin_package_workflow_task_attempt_admissions/,
);
assert.match(observed.sql, /workflow_step\.status = 'ready'/);
assert.match(
observed.sql,
/newer\.step_run_id = attempt\.step_run_id/,
);
});
test('returns one exact offer recovery with durable lease and Worker fence evidence', async () => {
const source = new PostgresClusterDispatchSource({
async query(sql, params) {
assert.match(sql, /worker\.lease_expires_at_ms > observation\.observed_at_ms/);
assert.deepEqual(params, ['offer-1']);
return { rows: [leaseRow()] };
},
});
const recovery = await source.findClusterDispatchRecovery('offer-1');
assert.equal(recovery.workerCurrent, true);
assert.equal(recovery.lease.workerSessionId, SESSION);
assert.equal(recovery.candidate.executorType, 'remote_worker');
});
@@ -0,0 +1,272 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const { ClusterControlRecoveryStoreError } = require('@qinglong/runtime-core');
const { PostgresClusterControlRecoveryClaimRepository } = require('../dist');
function sourceRows(observedAtMs = '1000') {
return [
{
observedAtMs,
kind: 'attempt',
id: 'attempt-1',
runId: 'run-1',
status: 'running',
createdAtMs: '900',
},
];
}
function claimRows() {
return [
{
targetKind: 'attempt',
targetId: 'attempt-1',
runId: 'run-1',
targetStatus: 'running',
targetCreatedAtMs: '900',
observedAtMs: '1000',
claimOwner: 'replica-a',
claimToken: '00000000-0000-4000-8000-000000000001',
claimVersion: 1,
claimExpiresAtMs: '31000',
},
];
}
test('discovers and claims one bounded page in a short transaction', async () => {
const calls = [];
let released = false;
const client = {
async query(text, values = []) {
calls.push({ text, values });
if (text.includes('run_candidates AS')) return { rows: sourceRows() };
if (text.startsWith('INSERT INTO "ql3"."run_recovery_controls"')) {
return { rows: [], rowCount: 1 };
}
if (text.includes('FOR UPDATE OF control SKIP LOCKED')) {
return { rows: claimRows(), rowCount: 1 };
}
return { rows: [] };
},
release() {
released = true;
},
};
const repository = new PostgresClusterControlRecoveryClaimRepository(
{
async connect() {
return client;
},
async query() {
throw new Error('pool query not expected');
},
},
() => '00000000-0000-4000-8000-000000000001',
);
const page = await repository.claim({
ownerId: 'replica-a',
limit: 4,
leaseMs: 30_000,
});
assert.equal(page.discovered, 1);
assert.equal(page.hasMore, false);
assert.deepEqual(page.claims[0], {
candidate: {
kind: 'attempt',
id: 'attempt-1',
runId: 'run-1',
status: 'running',
createdAtMs: 900,
},
observedAtMs: 1000,
ownerId: 'replica-a',
token: '00000000-0000-4000-8000-000000000001',
version: 1,
expiresAtMs: 31000,
});
assert.deepEqual(
calls.map(({ text }) => text.split('\n', 1)[0]),
[
'BEGIN ISOLATION LEVEL READ COMMITTED',
"SET LOCAL statement_timeout = '5000ms'",
"SET LOCAL lock_timeout = '1000ms'",
'WITH observation AS (',
'INSERT INTO "ql3"."run_recovery_controls" (',
'WITH discovered AS (',
'COMMIT',
],
);
assert.deepEqual(calls[5].values.slice(1), [
1000,
4,
'replica-a',
'00000000-0000-4000-8000-000000000001',
30000,
]);
assert.equal(released, true);
});
test('uses an injected runtime-only discovery source without widening claim authority', async () => {
let sourceQueryable;
let sourceLimit;
const client = {
async query(text) {
if (text.startsWith('INSERT INTO "ql3"."run_recovery_controls"')) {
return { rows: [], rowCount: 1 };
}
if (text.includes('FOR UPDATE OF control SKIP LOCKED')) {
return { rows: claimRows(), rowCount: 1 };
}
return { rows: [] };
},
release() {},
};
const repository = new PostgresClusterControlRecoveryClaimRepository(
{
async connect() {
return client;
},
async query() {
throw new Error('not expected');
},
},
() => '00000000-0000-4000-8000-000000000001',
(queryable) => {
sourceQueryable = queryable;
return {
async listOutstanding(limit) {
sourceLimit = limit;
return {
observedAtMs: 1000,
candidates: [{
kind: 'attempt',
id: 'attempt-1',
runId: 'run-1',
status: 'running',
createdAtMs: 900,
}],
hasMore: false,
};
},
};
},
);
const page = await repository.claim({
ownerId: 'replica-a',
limit: 4,
leaseMs: 30_000,
});
assert.equal(sourceQueryable, client);
assert.equal(sourceLimit, 4);
assert.equal(page.claims.length, 1);
});
test('rolls back and wraps claim-store failures without leaking the client', async () => {
const calls = [];
let released = false;
const repository = new PostgresClusterControlRecoveryClaimRepository(
{
async connect() {
return {
async query(text) {
calls.push(text);
if (text.includes('run_candidates AS')) throw new Error('offline');
return { rows: [] };
},
release() {
released = true;
},
};
},
async query() {
throw new Error('not expected');
},
},
() => '00000000-0000-4000-8000-000000000002',
);
await assert.rejects(
repository.claim({ ownerId: 'replica-b', limit: 1, leaseMs: 1000 }),
ClusterControlRecoveryStoreError,
);
assert.equal(calls.at(-1), 'ROLLBACK');
assert.equal(released, true);
});
test('settles only under the full owner-token-version-expiry fence', async () => {
const calls = [];
const results = [
{ rows: [{ targetId: 'attempt-1' }], rowCount: 1 },
{ rows: [], rowCount: 0 },
];
const repository = new PostgresClusterControlRecoveryClaimRepository({
async connect() {
throw new Error('not expected');
},
async query(text, values) {
calls.push({ text, values });
return results.shift();
},
});
const claim = Object.freeze({
candidate: Object.freeze({
kind: 'attempt',
id: 'attempt-1',
runId: 'run-1',
status: 'running',
createdAtMs: 900,
}),
observedAtMs: 1000,
ownerId: 'replica-a',
token: '00000000-0000-4000-8000-000000000001',
version: 7,
expiresAtMs: 31000,
});
assert.equal(
await repository.settle(claim, { status: 'retry', delayMs: 2500 }),
'settled',
);
assert.equal(
await repository.settle(claim, { status: 'resolved' }),
'fenced',
);
assert.match(
calls[0].text,
/claim_expires_at_ms > observation\.observed_at_ms/,
);
assert.deepEqual(calls[0].values, [
'attempt',
'attempt-1',
'replica-a',
'00000000-0000-4000-8000-000000000001',
7,
'retry',
2500,
]);
});
test('rejects unsafe options before acquiring a database connection', async () => {
let connects = 0;
const repository = new PostgresClusterControlRecoveryClaimRepository({
async connect() {
connects += 1;
throw new Error('not expected');
},
async query() {
throw new Error('not expected');
},
});
await assert.rejects(
repository.claim({ ownerId: 'bad owner', limit: 1, leaseMs: 1000 }),
/ownerId is invalid/,
);
await assert.rejects(
repository.claim({ ownerId: 'ok', limit: 129, leaseMs: 1000 }),
/claim limit/,
);
assert.equal(connects, 0);
});
@@ -0,0 +1,438 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
PostgresClusterControlRecoveryResolutionRepository,
} = require('../dist');
const {
pluginPackageWorkflowTaskAttemptAdmissionReceiptDigest,
} = require('@qinglong/runtime-core/plugin-package-workflow-task-attempt-admission');
const {
createStepRunRecord,
} = require('@qinglong/runtime-core/step-run');
const RUN_ID = '019f70b0-0000-7000-8000-000000000101';
const ATTEMPT_ID = '019f70b0-0000-7000-8000-000000000102';
const STEP_RUN_ID = '019f70b0-0000-7000-8000-000000000103';
function workflowStepRun() {
return createStepRunRecord({
id: STEP_RUN_ID,
runId: RUN_ID,
stepKey: 'collect',
kind: 'task',
definitionRef: 'pkg:demo:alpha',
definitionDigest: 'a'.repeat(64),
required: true,
initialStatus: 'ready',
mutationId: '019f70b0-0000-7000-8000-000000000104',
createdAtMs: 300,
});
}
function workflowAdmission(stepRun) {
const unsigned = {
schema:
'qinglong/plugin-package-workflow-task-attempt-admission@v1',
attemptId: ATTEMPT_ID,
planDigest: 'c'.repeat(64),
runId: RUN_ID,
stepRunId: STEP_RUN_ID,
stepRunVersion: stepRun.version,
stepRunDigest: stepRun.stepRunDigest,
resourceTaskId: 'alpha',
taskReconciliationReceiptDigest: 'd'.repeat(64),
taskId: 'pkg:demo:alpha',
taskRevision: `qltd:v1:1:${'a'.repeat(64)}`,
taskDefinitionDigest: 'a'.repeat(64),
executorType: 'remote_worker',
executionDigest: 'e'.repeat(64),
attemptNumber: 1,
eventId: '019f70b0-0000-7000-8000-000000000105',
runVersion: 3,
runEventSequence: 3,
admittedAtMs: 400,
};
return {
...unsigned,
receiptDigest:
pluginPackageWorkflowTaskAttemptAdmissionReceiptDigest(unsigned),
};
}
function claim() {
return {
candidate: {
kind: 'attempt',
id: ATTEMPT_ID,
runId: RUN_ID,
status: 'claimed',
createdAtMs: 200,
},
observedAtMs: 1000,
ownerId: 'replica-a',
token: '00000000-0000-4000-8000-000000000001',
version: 1,
expiresAtMs: 31000,
};
}
function runRow(overrides = {}) {
return {
id: RUN_ID,
projectId: 'default',
taskId: 'task-1',
taskRevision: 'v1',
taskName: null,
taskSnapshotRef: null,
legacyCronId: null,
parentRunId: null,
retryOfRunId: null,
triggerId: null,
triggerType: 'manual',
executionOrigin: 'manual',
executionOwner: 'runtime',
triggeredBy: null,
requestId: null,
scheduledForMs: null,
status: 'dispatching',
version: 1,
eventSequence: 0,
priority: 0,
idempotencyKey: null,
inputRef: null,
outputRef: null,
createdAtMs: '100',
queuedAtMs: null,
startedAtMs: null,
finishedAtMs: null,
cancelRequestedAtMs: null,
cancelReason: null,
errorCode: null,
errorSummary: null,
...overrides,
};
}
function attemptRow(overrides = {}) {
return {
id: ATTEMPT_ID,
runId: RUN_ID,
stepRunId: null,
attempt: 1,
status: 'claimed',
executorType: 'worker',
workerId: 'worker-1',
executorHandle: null,
pid: null,
logArtifactId: null,
leaseToken: 'lease-token',
leaseExpiresAtMs: '900',
deadlineAtMs: null,
callbackTokenHash: null,
callbackSequence: 0,
createdAtMs: '200',
startedAtMs: null,
finishedAtMs: null,
exitCode: null,
errorCode: null,
errorSummary: null,
...overrides,
};
}
function harness(options = {}) {
const queries = [];
let released = 0;
let runUpdates = 0;
const client = {
async query(text, values = []) {
queries.push({ text, values });
if (text.includes('run_recovery_controls')) {
return options.fenced
? { rows: [], rowCount: 0 }
: { rows: [{ observedAtMs: '1000' }], rowCount: 1 };
}
if (text.includes('FROM "ql3"."runs"')) {
return {
rows: [
runRow(
options.workflowTask
? {
taskId: 'workflow-alpha',
taskRevision: 'b'.repeat(64),
triggerType: 'plugin_package_workflow',
executionOrigin: 'system',
requestId: 'workflow-plan-1',
idempotencyKey:
'plugin-package-workflow:workflow-plan-1',
status: 'running',
version: 5,
eventSequence: 5,
startedAtMs: '200',
}
: {},
),
],
rowCount: 1,
};
}
if (text.includes('FROM "ql3"."run_attempts"')) {
return {
rows: [
attemptRow(
{
...(options.workflowTask
? {
stepRunId: STEP_RUN_ID,
executorType: 'remote_worker',
createdAtMs: '400',
}
: {}),
...(options.leased
? {
workerSessionId:
'019f70b0-0000-7000-8000-000000000201',
workerGeneration: 2,
leaseTokenDigest: 'f'.repeat(64),
leaseGeneration: 3,
leaseVersion: 7,
offerId: 'offer-1',
}
: {}),
},
),
],
rowCount: 1,
};
}
if (
text.includes(
'plugin_package_workflow_task_attempt_admissions',
)
) {
return options.workflowTask
? {
rows: [{
admissionJson: options.workflowTask.admission,
stepRunJson: options.workflowTask.stepRun,
}],
rowCount: 1,
}
: { rows: [], rowCount: 0 };
}
if (text.startsWith('UPDATE "ql3"."runs"')) {
runUpdates += 1;
return { rows: [{ id: RUN_ID }], rowCount: 1 };
}
if (text.startsWith('UPDATE "ql3"."run_attempts"')) {
return options.attemptConflict
? { rows: [], rowCount: 0 }
: { rows: [{ id: ATTEMPT_ID }], rowCount: 1 };
}
if (text.startsWith('UPDATE "ql3"."run_dispatch_leases"')) {
return options.leaseConflict
? { rows: [], rowCount: 0 }
: { rows: [], rowCount: 1 };
}
if (text.startsWith('UPDATE "ql3"."step_runs"')) {
return { rows: [], rowCount: 1 };
}
if (text.startsWith('INSERT INTO "ql3"."run_events"')) {
return { rows: [], rowCount: 1 };
}
if (
text.startsWith(
'INSERT INTO "ql3"."step_run_mutations"',
)
) {
return { rows: [], rowCount: 1 };
}
return { rows: [], rowCount: 0 };
},
release() {
released += 1;
},
};
return {
queries,
released: () => released,
runUpdates: () => runUpdates,
pool: {
async connect() {
return client;
},
query: (...args) => client.query(...args),
},
};
}
test('loads only under a live claim fence and returns a normalized snapshot', async () => {
const database = harness();
const repository = new PostgresClusterControlRecoveryResolutionRepository(
database.pool,
);
const loaded = await repository.load(claim());
assert.equal(loaded.observedAtMs, 1000);
assert.equal(loaded.run.status, 'dispatching');
assert.equal(loaded.attempt.status, 'claimed');
assert.equal(loaded.attempt.leaseExpiresAtMs, 900);
assert.equal(Object.isFrozen(loaded), true);
assert.equal(database.queries[4].text.includes('FOR UPDATE'), false);
assert.equal(database.queries.at(-1).text, 'COMMIT');
assert.equal(database.released(), 1);
});
test('locks the claim and commits both lost transitions and events atomically', async () => {
const database = harness();
let event = 0;
const repository = new PostgresClusterControlRecoveryResolutionRepository(
database.pool,
() => `00000000-0000-4000-8000-${String(++event).padStart(12, '0')}`,
);
const loaded = await repository.load(claim());
assert.equal(
await repository.applyLost(claim(), loaded, {
kind: 'mark_attempt_and_run_lost',
reason: 'unstarted_claim_expired',
}),
'applied',
);
const applyQueries = database.queries.slice(8);
assert.equal(
applyQueries.some(({ text }) => text.includes('FOR UPDATE OF control')),
true,
);
assert.equal(
applyQueries.filter(({ text }) => text.startsWith('UPDATE "ql3"."runs"'))
.length,
2,
);
assert.equal(
applyQueries.filter(({ text }) =>
text.startsWith('UPDATE "ql3"."run_attempts"'),
).length,
1,
);
const events = applyQueries.filter(({ text }) =>
text.startsWith('INSERT INTO "ql3"."run_events"'),
);
assert.equal(events.length, 2);
assert.equal(events[0].values.includes('reconciler'), true);
assert.equal(events[0].values.includes('replica-a'), true);
assert.equal(applyQueries.at(-1).text, 'COMMIT');
});
test('releases an expired dispatch lease under the same Attempt authority transaction', async () => {
const database = harness({ leased: true });
const repository = new PostgresClusterControlRecoveryResolutionRepository(
database.pool,
);
const loaded = await repository.load(claim());
assert.equal(
await repository.applyLost(claim(), loaded, {
kind: 'mark_attempt_and_run_lost',
reason: 'unstarted_claim_expired',
}),
'applied',
);
const leaseUpdate = database.queries.find(({ text }) =>
text.startsWith('UPDATE "ql3"."run_dispatch_leases"'),
);
assert.ok(leaseUpdate);
assert.deepEqual(leaseUpdate.values, [
ATTEMPT_ID,
8,
1000,
7,
3,
]);
const attemptUpdate = database.queries.find(({ text }) =>
text.startsWith('UPDATE "ql3"."run_attempts"'),
);
assert.equal(attemptUpdate.values.includes(8), true);
assert.equal(database.queries.at(-1).text, 'COMMIT');
});
test('atomically loses and requeues one admission-bound Workflow Task epoch', async () => {
const stepRun = workflowStepRun();
const database = harness({
workflowTask: {
admission: workflowAdmission(stepRun),
stepRun,
},
});
const repository =
new PostgresClusterControlRecoveryResolutionRepository(database.pool);
const loaded = await repository.load(claim());
assert.equal(loaded.workflowTask.admission.attemptId, ATTEMPT_ID);
assert.equal(loaded.workflowTask.stepRun.status, 'ready');
assert.equal(
await repository.applyLost(claim(), loaded, {
kind: 'recover_workflow_task',
reason: 'unstarted_claim_expired',
}),
'applied',
);
const runUpdate = database.queries.find(
({ text, values }) =>
text.startsWith('UPDATE "ql3"."runs"') &&
values[0] === RUN_ID &&
values.length === 5,
);
assert.ok(runUpdate);
assert.deepEqual(runUpdate.values.slice(1), [7, 7, 5, 5]);
const attemptUpdate = database.queries.find(({ text }) =>
text.startsWith('UPDATE "ql3"."run_attempts"'));
assert.ok(attemptUpdate);
assert.equal(attemptUpdate.values.includes('lost'), true);
const stepUpdate = database.queries.find(({ text }) =>
text.startsWith('UPDATE "ql3"."step_runs"'));
assert.ok(stepUpdate);
assert.equal(stepUpdate.values[0], 'ready');
assert.equal(stepUpdate.values[1], stepRun.version + 1);
assert.notEqual(stepUpdate.values[12], stepRun.stepRunDigest);
const events = database.queries.filter(({ text }) =>
text.startsWith('INSERT INTO "ql3"."run_events"'));
assert.equal(events.length, 2);
assert.equal(
database.queries.filter(({ text }) =>
text.startsWith('INSERT INTO "ql3"."step_run_mutations"'),
).length,
1,
);
assert.equal(database.queries.at(-1).text, 'COMMIT');
});
test('rolls back partial CAS work and reports a stale aggregate', async () => {
const database = harness({ attemptConflict: true });
const repository = new PostgresClusterControlRecoveryResolutionRepository(
database.pool,
);
const loaded = await repository.load(claim());
assert.equal(
await repository.applyLost(claim(), loaded, {
kind: 'mark_attempt_and_run_lost',
reason: 'unstarted_claim_expired',
}),
'stale',
);
assert.equal(database.runUpdates(), 1);
assert.equal(database.queries.at(-1).text, 'ROLLBACK');
});
test('does not read or mutate Run state after the recovery fence is lost', async () => {
const database = harness({ fenced: true });
const repository = new PostgresClusterControlRecoveryResolutionRepository(
database.pool,
);
assert.equal(await repository.load(claim()), 'fenced');
assert.equal(
database.queries.some(({ text }) => text.includes('FROM "ql3"."runs"')),
false,
);
});
@@ -0,0 +1,166 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const { PostgresClusterControlRecoverySource } = require('../dist/entrypoints/runtime');
function sourceWith(rows, observations = []) {
return new PostgresClusterControlRecoverySource({
async query(text, values) {
observations.push({ text, values });
return { rows };
},
});
}
test('reads Run and Attempt recovery candidates through one bounded query', async () => {
const observations = [];
const source = sourceWith(
[
{
observedAtMs: '100',
kind: 'run',
id: 'run-1',
runId: 'run-1',
status: 'running',
createdAtMs: '10',
},
{
observedAtMs: '100',
kind: 'attempt',
id: 'attempt-1',
runId: 'run-1',
status: 'starting',
createdAtMs: 11,
},
{
observedAtMs: '100',
kind: 'run',
id: 'run-2',
runId: 'run-2',
status: 'created',
createdAtMs: 12,
},
],
observations,
);
assert.deepEqual(await source.listOutstanding(2), {
observedAtMs: 100,
candidates: [
{
kind: 'run',
id: 'run-1',
runId: 'run-1',
status: 'running',
createdAtMs: 10,
},
{
kind: 'attempt',
id: 'attempt-1',
runId: 'run-1',
status: 'starting',
createdAtMs: 11,
},
],
hasMore: true,
});
assert.deepEqual(observations[0].values, [3]);
assert.match(observations[0].text, /WITH observation AS/);
assert.match(observations[0].text, /statement_timestamp\(\)/);
assert.match(observations[0].text, /LIMIT \$1/);
assert.match(observations[0].text, /execution_owner = 'runtime'/);
assert.match(
observations[0].text,
/trigger_type <> 'plugin_package_workflow'/,
);
assert.match(observations[0].text, /attempt_candidates/);
assert.match(
observations[0].text,
/plugin_package_workflow_task_attempt_admissions/,
);
assert.match(
observations[0].text,
/attempt_run\.trigger_type = 'plugin_package_workflow'/,
);
assert.match(
observations[0].text,
/workflow_task\.attempt_id = attempt\.id/,
);
assert.match(
observations[0].text,
/lease_expires_at_ms > observation\.observed_at_ms/,
);
assert.match(
observations[0].text,
/lease_expires_at_ms <= observation\.observed_at_ms/,
);
assert.match(observations[0].text, /INNER JOIN "ql3"\."runs" AS attempt_run/);
assert.match(observations[0].text, /attempt_run\.status = 'queued'/);
assert.match(
observations[0].text,
/attempt\.executor_type = 'remote_worker'/,
);
assert.match(observations[0].text, /attempt\.callback_sequence = 0/);
assert.match(
observations[0].text,
/newer_attempt\.attempt > attempt\.attempt/,
);
});
test('rejects unbounded page sizes before touching PostgreSQL', async () => {
let queries = 0;
const source = sourceWith([], {
push() {
queries += 1;
},
});
await assert.rejects(source.listOutstanding(0), /between 1 and 128/);
await assert.rejects(source.listOutstanding(129), /between 1 and 128/);
assert.equal(queries, 0);
});
test('fails closed on malformed or terminal PostgreSQL rows', async () => {
await assert.rejects(
sourceWith([
{
observedAtMs: '100',
kind: 'run',
id: 'run-1',
runId: 'run-1',
status: 'succeeded',
createdAtMs: 1,
},
]).listOutstanding(1),
/kind or status is invalid/,
);
await assert.rejects(
sourceWith([
{
observedAtMs: '100',
kind: 'attempt',
id: 'attempt-1',
runId: 'run-1',
status: 'running',
createdAtMs: '9007199254740992',
},
]).listOutstanding(1),
/createdAtMs is invalid/,
);
});
test('represents an empty page without losing the database observation', async () => {
const source = sourceWith([
{
observedAtMs: '100',
kind: null,
id: null,
runId: null,
status: null,
createdAtMs: null,
},
]);
assert.deepEqual(await source.listOutstanding(1), {
observedAtMs: 100,
candidates: [],
hasMore: false,
});
});
@@ -0,0 +1,183 @@
'use strict';
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
ClusterRunCancellationConvergenceUnavailableError,
} = require('@qinglong/runtime-core/cluster-run-cancellation-convergence');
const {
PostgresClusterRunCancellationConvergenceRepository,
} = require('../dist/entrypoints/runtime');
function row(overrides = {}) {
return {
observedAtMs: '1750000000100',
runId: 'run-1',
runStatus: 'queued',
runVersion: 3,
eventSequence: 4,
runCreatedAtMs: '1750000000000',
runQueuedAtMs: '1750000000010',
runStartedAtMs: null,
cancelRequestedAtMs: '1750000000050',
cancelReason: 'user',
attemptId: 'attempt-1',
attemptStatus: 'claimed',
stepRunId: null,
attemptNumber: 1,
attemptCreatedAtMs: '1750000000010',
attemptStartedAtMs: null,
attemptFinishedAtMs: null,
leaseStatus: null,
...overrides,
};
}
function fixture(options = {}) {
const calls = [];
let runUpdates = 0;
const client = {
async query(sql, params = []) {
const normalized = sql.replace(/\s+/g, ' ').trim();
calls.push({ sql: normalized, params });
if (
normalized === 'BEGIN' || normalized === 'COMMIT' ||
normalized === 'ROLLBACK' || normalized.startsWith('SELECT set_config')
) return { rows: [], rowCount: 0 };
if (normalized.startsWith('WITH observation AS MATERIALIZED')) {
return { rows: options.rows ?? [row()], rowCount: options.rows?.length ?? 1 };
}
if (normalized.startsWith('UPDATE "ql3"."run_attempts"')) {
return { rows: [], rowCount: options.attemptRowCount ?? 1 };
}
if (normalized.startsWith('UPDATE "ql3"."runs"')) {
runUpdates += 1;
return {
rows: [],
rowCount: options.failRunUpdate === runUpdates ? 0 : 1,
};
}
if (normalized.startsWith('INSERT INTO "ql3"."run_events"')) {
return { rows: [], rowCount: 1 };
}
if (normalized.startsWith('SELECT id AS "runId" FROM "ql3"."runs"')) {
return {
rows: options.workflowRows ?? [],
rowCount: options.workflowRows?.length ?? 0,
};
}
if (normalized.startsWith('SELECT EXISTS')) {
return { rows: [{ hasMore: options.hasMore ?? false }], rowCount: 1 };
}
throw new Error(`Unexpected SQL: ${normalized}`);
},
release() { calls.push({ sql: 'RELEASE', params: [] }); },
};
return {
repository: new PostgresClusterRunCancellationConvergenceRepository({
async connect() { return client; },
}),
calls,
};
}
test('atomically settles queued claimed and lost terminal-attempt Runs', async () => {
const { repository, calls } = fixture({
rows: [
row(),
row({
runId: 'run-2',
runStatus: 'lost',
runVersion: 7,
eventSequence: 8,
cancelReason: 'policy',
attemptId: 'attempt-2',
attemptStatus: 'lost',
attemptFinishedAtMs: '1750000000020',
}),
],
});
assert.deepEqual(await repository.convergePage({
limit: 2,
}), {
scanned: 2,
settledRuns: 2,
settledAttempts: 1,
blocked: 0,
hasMore: false,
});
const attemptUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."run_attempts"'));
assert.equal(attemptUpdate.params[1], 'cancelled');
const runUpdates = calls.filter(({ sql }) =>
sql.startsWith('UPDATE "ql3"."runs"'));
assert.equal(runUpdates.length, 2);
assert.equal(runUpdates[0].params[5], 5);
assert.equal(runUpdates[0].params[6], 6);
assert.equal(runUpdates[1].params[5], 8);
assert.equal(runUpdates[1].params[6], 9);
const events = calls.filter(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."run_events"'));
assert.equal(events.length, 3);
assert.match(events[0].params[0], /^qca-[0-9a-f]{32}$/);
assert.match(events[1].params[0], /^qcr-[0-9a-f]{32}$/);
assert.match(events[2].params[0], /^qcr-[0-9a-f]{32}$/);
assert.equal(new Set(events.map(({ params }) => params[0])).size, 3);
const candidate = calls.find(({ sql }) =>
sql.startsWith('WITH observation AS MATERIALIZED'));
assert.match(
candidate.sql,
/run\.trigger_type <> 'plugin_package_workflow'/,
);
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), true);
});
test('maps timeout intent to timed_out and preserves database-scale time', async () => {
const { repository, calls } = fixture({
rows: [row({
attemptId: null,
attemptStatus: null,
attemptNumber: null,
attemptCreatedAtMs: null,
cancelReason: 'timeout',
})],
});
assert.equal((await repository.convergePage({
limit: 1,
})).settledRuns, 1);
const update = calls.find(({ sql }) => sql.startsWith('UPDATE "ql3"."runs"'));
assert.equal(update.params[1], 'timed_out');
assert.equal(update.params[2], 1_750_000_000_100);
assert.equal(update.params[3], 'EXECUTION_TIMED_OUT');
});
test('does not forge a terminal state for an execution that crossed start', async () => {
const { repository, calls } = fixture({
rows: [row({
runStatus: 'waiting_approval',
attemptStatus: 'running',
leaseStatus: 'leased',
})],
hasMore: true,
});
assert.deepEqual(await repository.convergePage({
limit: 1,
}), {
scanned: 1,
settledRuns: 0,
settledAttempts: 0,
blocked: 1,
hasMore: true,
});
assert.equal(calls.some(({ sql }) => sql.startsWith('UPDATE')), false);
});
test('rolls back the whole page when a Run fence changes', async () => {
const { repository, calls } = fixture({ failRunUpdate: 1 });
await assert.rejects(
repository.convergePage({ limit: 1 }),
ClusterRunCancellationConvergenceUnavailableError,
);
assert.equal(calls.some(({ sql }) => sql === 'ROLLBACK'), true);
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), false);
});
@@ -0,0 +1,264 @@
'use strict';
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
ClusterRunCancellationFenceRejectedError,
ClusterRunCancellationNotFoundError,
} = require('@qinglong/runtime-core/cluster-run-cancellation');
const {
PostgresClusterRunCancellationRepository,
} = require('../dist/entrypoints/runtime');
function command(overrides = {}) {
return {
projectId: 'project-1',
runId: 'run-1',
mutationId: 'mutation-1',
eventId: '018f0000-0000-7000-8000-000000000001',
subject: { type: 'user', id: 'user-1' },
policyFence: { projectVersion: 2, bindingVersion: 3 },
...overrides,
};
}
function run(overrides = {}) {
return {
projectId: 'project-1',
runStatus: 'running',
runVersion: 4,
eventSequence: 6,
cancelRequestedAtMs: null,
cancelReason: null,
...overrides,
};
}
function fixture(options = {}) {
const calls = [];
const client = {
async query(sql, params = []) {
const normalized = sql.replace(/\s+/g, ' ').trim();
calls.push({ sql: normalized, params });
if (
normalized.startsWith('BEGIN') || normalized === 'COMMIT' ||
normalized === 'ROLLBACK' || normalized.startsWith('SELECT set_config')
) return { rows: [], rowCount: 0 };
if (normalized.includes('FROM "ql3"."projects"')) {
return {
rows: options.projectRows ?? [{
projectStatus: 'active',
projectVersion: 2,
}],
rowCount: 1,
};
}
if (normalized.includes('FROM "ql3"."project_role_bindings"')) {
return {
rows: options.bindingRows ?? [{
bindingVersion: 3,
bindingState: 'active',
bindingRole: 'operator',
}],
rowCount: 1,
};
}
if (
normalized.includes('FROM "ql3"."runs"') &&
normalized.startsWith('SELECT')
) {
return {
rows: options.runRows ?? [run()],
rowCount: options.runRows?.length ?? 1,
};
}
if (
normalized.includes(
'FROM "ql3"."plugin_package_workflow_admissions"',
)
) {
const rows = options.workflowAdmissionRows ?? [
{
projectId: 'project-1',
packageName: 'example',
workflowId: 'daily',
},
];
return { rows, rowCount: rows.length };
}
if (normalized.includes('statement_timestamp()')) {
return { rows: [{ nowMs: options.nowMs ?? 1_000 }], rowCount: 1 };
}
if (normalized.startsWith('UPDATE "ql3"."runs"')) {
return {
rows: options.updatedRows ?? [run({
runVersion: 5,
eventSequence: 7,
cancelRequestedAtMs: options.nowMs ?? 1_000,
cancelReason: 'user',
})],
rowCount: options.updatedRows?.length ?? 1,
};
}
if (normalized.startsWith('INSERT INTO "ql3"."run_events"')) {
return { rows: [], rowCount: 1 };
}
throw new Error(`Unexpected SQL: ${normalized}`);
},
release() { calls.push({ sql: 'RELEASE', params: [] }); },
};
return {
repository: new PostgresClusterRunCancellationRepository({
async connect() { return client; },
}),
calls,
};
}
test('revalidates policy authority and commits one database-timed intent', async () => {
const { repository, calls } = fixture();
assert.deepEqual(await repository.requestUserCancellation(command()), {
status: 'accepted',
projectId: 'project-1',
runId: 'run-1',
runStatus: 'running',
runVersion: 5,
eventSequence: 7,
cancelRequestedAtMs: 1_000,
cancelReason: 'user',
});
const projectIndex = calls.findIndex(({ sql }) =>
sql.includes('FROM "ql3"."projects"'));
const bindingIndex = calls.findIndex(({ sql }) =>
sql.includes('FROM "ql3"."project_role_bindings"'));
const runIndex = calls.findIndex(({ sql }) =>
sql.includes('FROM "ql3"."runs"'));
assert.ok(projectIndex < bindingIndex && bindingIndex < runIndex);
const update = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."runs"'));
assert.deepEqual(update.params, ['run-1', 1_000, 5, 7, 4]);
const event = calls.find(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."run_events"'));
assert.equal(event.params[0], command().eventId);
assert.equal(event.params[3], 'user-cancel:mutation-1');
assert.equal(event.params[4], 'user');
assert.equal(JSON.parse(event.params[6]).reason, 'user');
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), true);
});
test('returns existing intent and terminal state without adding an event', async () => {
const existing = fixture({
runRows: [run({ cancelRequestedAtMs: 900, cancelReason: 'timeout' })],
});
assert.equal(
(await existing.repository.requestUserCancellation(command())).status,
'already_requested',
);
assert.equal(existing.calls.some(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."run_events"')), false);
const terminal = fixture({
runRows: [run({ runStatus: 'succeeded', runVersion: 5 })],
});
assert.equal(
(await terminal.repository.requestUserCancellation(command())).status,
'already_terminal',
);
assert.equal(terminal.calls.some(({ sql }) =>
sql.startsWith('UPDATE "ql3"."runs"')), false);
});
test('accepts cancellation for a lost Run that still owns retry authority', async () => {
const { repository } = fixture({
runRows: [run({ runStatus: 'lost' })],
updatedRows: [run({
runStatus: 'lost',
runVersion: 5,
eventSequence: 7,
cancelRequestedAtMs: 1_000,
cancelReason: 'user',
})],
});
assert.equal(
(await repository.requestUserCancellation(command())).status,
'accepted',
);
});
test('rejects a revoked policy fence before locking the Run', async () => {
const { repository, calls } = fixture({
bindingRows: [{
bindingVersion: 4,
bindingState: 'revoked',
bindingRole: null,
}],
});
await assert.rejects(
repository.requestUserCancellation(command()),
(error) =>
error instanceof ClusterRunCancellationFenceRejectedError &&
error.reason === 'authorization_changed',
);
assert.equal(calls.some(({ sql }) => sql.includes('FROM "ql3"."runs"')), false);
assert.equal(calls.some(({ sql }) => sql === 'ROLLBACK'), true);
});
test('masks cross-Project and missing Runs', async () => {
for (const runRows of [[], [run({ projectId: 'project-other' })]]) {
const { repository } = fixture({ runRows });
await assert.rejects(
repository.requestUserCancellation(command()),
ClusterRunCancellationNotFoundError,
);
}
});
test('binds Workflow cancellation to the immutable admission target', async () => {
const targeted = command({
workflowTarget: { packageName: 'example', workflowId: 'daily' },
});
const accepted = fixture();
assert.equal(
(await accepted.repository.requestUserCancellation(targeted)).status,
'accepted',
);
const admission = accepted.calls.find(({ sql }) =>
sql.includes('FROM "ql3"."plugin_package_workflow_admissions"'),
);
assert.deepEqual(admission.params, ['run-1']);
assert.equal(
admission.sql.includes('FOR SHARE'),
false,
'immutable admission lookup must not require UPDATE authority',
);
for (const workflowAdmissionRows of [
[],
[
{
projectId: 'project-1',
packageName: 'other',
workflowId: 'daily',
},
],
[
{
projectId: 'project-1',
packageName: 'example',
workflowId: 'other',
},
],
]) {
const rejected = fixture({ workflowAdmissionRows });
await assert.rejects(
rejected.repository.requestUserCancellation(targeted),
ClusterRunCancellationNotFoundError,
);
assert.equal(
rejected.calls.some(({ sql }) =>
sql.startsWith('UPDATE "ql3"."runs"'),
),
false,
);
}
});
@@ -0,0 +1,244 @@
'use strict';
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
PostgresClusterRunLostRetryRepository,
} = require('../dist/entrypoints/runtime');
function runRow(status) {
return {
id: 'run-1',
projectId: 'project-1',
taskId: 'task-1',
taskRevision: 'revision-1',
taskName: null,
taskSnapshotRef: null,
legacyCronId: null,
parentRunId: null,
retryOfRunId: null,
triggerId: null,
triggerType: 'manual',
executionOrigin: 'manual',
executionOwner: 'runtime',
triggeredBy: null,
requestId: null,
scheduledForMs: null,
status,
version: 4,
eventSequence: 4,
priority: 0,
idempotencyKey: null,
inputRef: null,
outputRef: null,
createdAtMs: '100',
queuedAtMs: '110',
startedAtMs: '150',
finishedAtMs: null,
cancelRequestedAtMs: null,
cancelReason: null,
errorCode: 'CLUSTER_RECOVERY_EXECUTION_NOT_RUNNING',
errorSummary: 'lost',
};
}
function attemptRow() {
return {
id: 'attempt-1',
runId: 'run-1',
stepRunId: null,
attempt: 1,
status: 'lost',
executorType: 'remote_worker',
workerId: 'worker-1',
workerSessionId: null,
workerGeneration: null,
executorHandle: null,
pid: null,
logArtifactId: null,
leaseToken: null,
leaseTokenDigest: null,
leaseGeneration: null,
leaseVersion: null,
leaseExpiresAtMs: null,
offerId: null,
deadlineAtMs: null,
callbackTokenHash: null,
callbackSequence: 0,
createdAtMs: '120',
startedAtMs: '150',
finishedAtMs: '200',
exitCode: null,
errorCode: 'CLUSTER_RECOVERY_EXECUTION_NOT_RUNNING',
errorSummary: 'lost',
};
}
function policyRow(nextAttemptAtMs = null) {
return {
runId: 'run-1',
maxAttempts: 3,
retryOnLost: true,
safety: 'idempotent',
backoffBaseMs: '1000',
backoffMaxMs: '8000',
nextAttemptAtMs,
version: 0,
createdAtMs: '100',
updatedAtMs: '100',
};
}
function harness(status = 'lost') {
const calls = [];
let id = 0;
const client = {
async query(text, values = []) {
calls.push({ text, values });
if (text.includes('pg_advisory_xact_lock')) {
return { rows: [{ locked: true }], rowCount: 1 };
}
if (
text.includes('FROM "ql3"."runs" AS run') &&
text.includes('FOR UPDATE')
) {
return { rows: [{ runId: 'run-1' }], rowCount: 1 };
}
if (
text.startsWith('SELECT') &&
text.includes('FROM "ql3"."runs" WHERE')
) {
return { rows: [runRow(status)], rowCount: 1 };
}
if (
text.startsWith('SELECT') &&
text.includes('FROM "ql3"."run_attempts"')
) {
return { rows: [attemptRow()], rowCount: 1 };
}
if (
text.startsWith('SELECT') &&
text.includes('FROM "ql3"."run_retry_policies" WHERE') &&
!text.includes('FOR UPDATE')
) {
return {
rows: [policyRow(status === 'retry_wait' ? '250' : null)],
rowCount: 1,
};
}
if (
text.includes('FROM "ql3"."run_retry_policies"') &&
text.includes('FOR UPDATE')
) {
return { rows: [{ runId: 'run-1' }], rowCount: 1 };
}
if (text.includes('AS "observedAtMs"')) {
return { rows: [{ observedAtMs: '300' }], rowCount: 1 };
}
if (text.startsWith('UPDATE "ql3"."runs"')) {
return { rows: [{ id: 'run-1' }], rowCount: 1 };
}
if (text.startsWith('UPDATE "ql3"."run_retry_policies"')) {
return { rows: [{ run_id: 'run-1' }], rowCount: 1 };
}
if (text.startsWith('INSERT INTO "ql3"."run_attempts"')) {
return { rows: [], rowCount: 1 };
}
if (text.startsWith('INSERT INTO "ql3"."run_events"')) {
return { rows: [], rowCount: 1 };
}
return { rows: [], rowCount: 0 };
},
release() {},
};
return {
calls,
repository: new PostgresClusterRunLostRetryRepository(
{
async query(text, values) {
calls.push({ text, values });
assert.match(text, /JOIN LATERAL/);
return {
rows: [{ runId: 'run-1', attemptId: 'attempt-1' }],
rowCount: 1,
};
},
async connect() {
return client;
},
},
() => `00000000-0000-4000-8000-${String(++id).padStart(12, '0')}`,
),
};
}
test('schedules one safe lost Run in a bounded atomic page', async () => {
const database = harness('lost');
assert.deepEqual(await database.repository.reconcilePage({ limit: 2 }), {
scanned: 1,
scheduled: 1,
requeued: 0,
failed: 0,
raced: 0,
hasMore: false,
});
assert.equal(
database.calls.filter(({ text }) =>
text.startsWith('UPDATE "ql3"."runs"'),
).length,
1,
);
assert.equal(
database.calls.filter(({ text }) =>
text.startsWith('UPDATE "ql3"."run_retry_policies"'),
).length,
1,
);
assert.equal(
database.calls.filter(({ text }) =>
text.startsWith('INSERT INTO "ql3"."run_events"'),
).length,
1,
);
assert.equal(database.calls.at(-1).text, 'COMMIT');
});
test('creates exactly one fresh Attempt when retry_wait is due', async () => {
const database = harness('retry_wait');
assert.deepEqual(await database.repository.reconcilePage({ limit: 1 }), {
scanned: 1,
scheduled: 0,
requeued: 1,
failed: 0,
raced: 0,
hasMore: false,
});
assert.equal(
database.calls.filter(({ text }) =>
text.startsWith('UPDATE "ql3"."runs"'),
).length,
2,
);
assert.equal(
database.calls.filter(({ text }) =>
text.startsWith('INSERT INTO "ql3"."run_attempts"'),
).length,
1,
);
assert.equal(
database.calls.filter(({ text }) =>
text.startsWith('INSERT INTO "ql3"."run_events"'),
).length,
2,
);
});
test('rejects widened pages before querying PostgreSQL', async () => {
const database = harness('lost');
await assert.rejects(
database.repository.reconcilePage({ limit: 65 }),
/page size/,
);
assert.equal(database.calls.length, 0);
});
@@ -0,0 +1,115 @@
'use strict';
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
PostgresClusterRuntimeRecoverySource,
} = require('../dist/entrypoints/runtime');
function source(rows, calls = []) {
return new PostgresClusterRuntimeRecoverySource({
async query(text, values) {
calls.push({ text, values });
return { rows };
},
});
}
test('returns only one bounded page of active expired Attempts', async () => {
const calls = [];
const recovery = source(
[
{
observedAtMs: '1000',
attemptId: 'attempt-1',
runId: 'run-1',
status: 'starting',
createdAtMs: '100',
},
{
observedAtMs: '1000',
attemptId: 'attempt-2',
runId: 'run-2',
status: 'running',
createdAtMs: '200',
},
{
observedAtMs: '1000',
attemptId: 'attempt-3',
runId: 'run-3',
status: 'claimed',
createdAtMs: '300',
},
],
calls,
);
assert.deepEqual(await recovery.listOutstanding(2), {
observedAtMs: 1000,
candidates: [
{
kind: 'attempt',
id: 'attempt-1',
runId: 'run-1',
status: 'starting',
createdAtMs: 100,
},
{
kind: 'attempt',
id: 'attempt-2',
runId: 'run-2',
status: 'running',
createdAtMs: 200,
},
],
hasMore: true,
});
assert.deepEqual(calls[0].values, [3]);
assert.match(calls[0].text, /lease_expires_at_ms <= observation/);
assert.match(calls[0].text, /run\.status IN \('dispatching', 'running', 'lost'\)/);
assert.match(calls[0].text, /attempt\.worker_id IS NULL/);
assert.doesNotMatch(calls[0].text, /status = 'created'/);
});
test('retains a database observation for an empty runtime page', async () => {
const recovery = source([
{
observedAtMs: '1000',
attemptId: null,
runId: null,
status: null,
createdAtMs: null,
},
]);
assert.deepEqual(await recovery.listOutstanding(1), {
observedAtMs: 1000,
candidates: [],
hasMore: false,
});
});
test('rejects malformed rows and unbounded requests', async () => {
let queries = 0;
const recovery = new PostgresClusterRuntimeRecoverySource({
async query() {
queries += 1;
return { rows: [] };
},
});
await assert.rejects(recovery.listOutstanding(0), /between 1 and 128/);
assert.equal(queries, 0);
await assert.rejects(
source([
{
observedAtMs: '1000',
attemptId: 'attempt-1',
runId: 'run-1',
status: 'succeeded',
createdAtMs: '100',
},
]).listOutstanding(1),
/Attempt status is invalid/,
);
});
@@ -0,0 +1,443 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
resolveClusterScheduleDecision,
} = require('@qinglong/runtime-core/cluster-scheduler');
const {
createTaskDefinitionRecord,
} = require('@qinglong/runtime-core/task-definition');
const {
createBuiltInTaskSpecSemanticRegistry,
} = require('@qinglong/runtime-core/task-spec-semantic');
const {
compileClusterCommandTaskDefinition,
} = require('@qinglong/runtime-core/cluster-execution-revision');
const {
createBuiltInTriggerSpecSemanticRegistry,
createTriggerRecord,
} = require('@qinglong/runtime-core/trigger');
const {
PostgresClusterScheduleRepository,
PostgresClusterScheduleUnavailableError,
} = require('../dist/scheduling/clusterScheduleRepository');
function nextMinute(schedule, afterMs) {
if (schedule.expression !== '* * * * *' || schedule.timezone !== 'UTC') {
throw new Error('unsupported test schedule');
}
return Math.floor(afterMs / 60_000 + 1) * 60_000;
}
const CLAIM_TOKEN = '019f7700-0000-7000-8000-000000000001';
const TASK_COMMAND = Object.freeze({
projectId: 'default',
taskId: 'task-00001',
expectedRevision: null,
mutationId: '019f7700-0000-7000-8000-000000000010',
name: 'Scheduled Task',
kind: 'command',
spec: Object.freeze({
schema: 'qinglong/command@v1',
config: Object.freeze({
command: Object.freeze({
kind: 'argv',
file: '/bin/echo',
args: Object.freeze(['scheduled']),
}),
}),
}),
labels: Object.freeze({}),
enabled: true,
occurredAtMs: 101,
});
const taskSemantics = createBuiltInTaskSpecSemanticRegistry();
const TASK = createTaskDefinitionRecord(
Object.freeze({
...TASK_COMMAND,
spec: taskSemantics.normalize({
projectId: TASK_COMMAND.projectId,
taskId: TASK_COMMAND.taskId,
kind: TASK_COMMAND.kind,
spec: TASK_COMMAND.spec,
}),
}),
TASK_COMMAND.occurredAtMs,
);
const EXECUTION = compileClusterCommandTaskDefinition(TASK, taskSemantics);
const TRIGGER_COMMAND = Object.freeze({
projectId: TASK.projectId,
triggerId: 'trigger-00001',
expectedRevision: null,
mutationId: '019f7700-0000-7000-8000-000000000011',
taskId: TASK.taskId,
taskRevision: TASK.revision,
taskContentDigest: TASK.contentDigest,
spec: Object.freeze({
schema: 'qinglong/cron@v1',
config: Object.freeze({
expression: '* * * * *',
timezone: 'UTC',
misfirePolicy: 'skip',
}),
}),
enabled: true,
occurredAtMs: 201,
});
const triggerSemantics = createBuiltInTriggerSpecSemanticRegistry();
const TRIGGER = createTriggerRecord(
Object.freeze({
...TRIGGER_COMMAND,
spec: triggerSemantics.normalize({
projectId: TRIGGER_COMMAND.projectId,
triggerId: TRIGGER_COMMAND.triggerId,
taskId: TRIGGER_COMMAND.taskId,
taskRevision: TRIGGER_COMMAND.taskRevision,
spec: TRIGGER_COMMAND.spec,
}),
}),
TRIGGER_COMMAND.occurredAtMs,
);
function claim(overrides = {}) {
return {
projectId: TRIGGER.projectId,
triggerId: TRIGGER.triggerId,
triggerRevision: TRIGGER.revision,
triggerContentDigest: TRIGGER.contentDigest,
triggerUpdatedAtMs: TRIGGER.updatedAtMs,
taskId: TRIGGER.taskId,
taskRevision: TRIGGER.taskRevision,
taskContentDigest: TRIGGER.taskContentDigest,
expression: TRIGGER.spec.config.expression,
timezone: TRIGGER.spec.config.timezone,
misfirePolicy: TRIGGER.spec.config.misfirePolicy,
stateVersion: 1,
nextFireAtMs: 60_000,
claimOwner: 'scheduler-a',
claimToken: CLAIM_TOKEN,
claimVersion: 1,
claimAcquiredAtMs: 61_000,
claimExpiresAtMs: 91_000,
...overrides,
};
}
function claimRow(overrides = {}) {
const value = claim(overrides);
return {
projectId: value.projectId,
triggerId: value.triggerId,
triggerRevision: value.triggerRevision,
triggerContentDigest: value.triggerContentDigest,
triggerUpdatedAtMs: String(value.triggerUpdatedAtMs),
taskId: value.taskId,
taskRevision: value.taskRevision,
taskContentDigest: value.taskContentDigest,
specJson: TRIGGER.spec,
taskName: TASK.name,
stateVersion: value.stateVersion,
nextFireAtMs:
value.nextFireAtMs === null ? null : String(value.nextFireAtMs),
claimOwner: value.claimOwner,
claimToken: value.claimToken,
claimVersion: value.claimVersion,
claimAcquiredAtMs: String(value.claimAcquiredAtMs),
claimExpiresAtMs: String(value.claimExpiresAtMs),
commitObservedAtMs: String(overrides.commitObservedAtMs ?? 62_000),
};
}
function executionRow(overrides = {}) {
return {
projectId: EXECUTION.projectId,
taskId: EXECUTION.taskId,
sourceRevision: EXECUTION.sourceRevision,
taskRevision: EXECUTION.taskRevision,
sourceContentDigest: EXECUTION.sourceContentDigest,
executorType: EXECUTION.executorType,
planSchema: EXECUTION.planSchema,
planJson: {
command: EXECUTION.command,
environment: EXECUTION.environment,
...(EXECUTION.workingDirectory === undefined
? {}
: { workingDirectory: EXECUTION.workingDirectory }),
...(EXECUTION.timeoutMs === undefined
? {}
: { timeoutMs: EXECUTION.timeoutMs }),
...(EXECUTION.placement === undefined
? {}
: { placement: EXECUTION.placement }),
},
contentDigest: EXECUTION.contentDigest,
createdAtMs: String(EXECUTION.createdAtMs),
...overrides,
};
}
function fixture(options = {}) {
const queries = [];
let commitFailuresRemaining = options.failCommitOnce ? 1 : 0;
const client = {
async query(text, values) {
queries.push({ text, values });
if (text === 'COMMIT' && commitFailuresRemaining > 0) {
commitFailuresRemaining -= 1;
const error = new Error('injected committed response loss');
error.code = '40001';
throw error;
}
if (text.includes('FOR UPDATE OF schedule')) {
return {
rows: options.missingClaim
? []
: [
claimRow({
...options.claimOverrides,
commitObservedAtMs: options.commitObservedAtMs,
}),
],
};
}
if (text.includes('FROM "ql3"."task_execution_revisions"')) {
return {
rows: options.missingExecution
? []
: [executionRow(options.executionOverrides)],
};
}
if (
options.failAt &&
text.includes(`INSERT INTO "ql3"."${options.failAt}"`)
) {
throw new Error('injected write failure');
}
if (text.includes('UPDATE "ql3"."trigger_schedules"')) {
return { rows: [], rowCount: options.advanceRaced ? 0 : 1 };
}
return { rows: [], rowCount: 1 };
},
release() {
queries.push({ text: 'RELEASE' });
},
};
return {
queries,
pool: {
async query(text, values) {
queries.push({ text, values });
return {
rows: options.noDue ? [] : [claimRow(options.claimOverrides)],
};
},
async connect() {
return client;
},
},
};
}
function admissionCommand(claimed = claim()) {
return {
claim: claimed,
decision: resolveClusterScheduleDecision(claimed, 5_000, nextMinute),
runId: '019f7700-0000-7000-8000-000000000020',
attemptId: '019f7700-0000-7000-8000-000000000021',
createdEventId: '019f7700-0000-7000-8000-000000000022',
queuedEventId: '019f7700-0000-7000-8000-000000000023',
};
}
test('claims one due schedule with ordered SKIP LOCKED lease takeover', async () => {
const db = fixture();
const repository = new PostgresClusterScheduleRepository(db.pool);
assert.deepEqual(
await repository.claimNextClusterSchedule({
ownerId: 'scheduler-a',
claimToken: CLAIM_TOKEN,
leaseMs: 30_000,
}),
claim(),
);
assert.equal(db.queries.length, 1);
assert.match(db.queries[0].text, /FOR UPDATE OF schedule SKIP LOCKED/);
assert.match(db.queries[0].text, /clock_timestamp\(\)/);
assert.match(
db.queries[0].text,
/schedule\.claim_expires_at_ms <= observation\.observed_at_ms/,
);
assert.match(db.queries[0].text, /NULLS FIRST/);
assert.match(
db.queries[0].text,
/task_head\.current_revision = task\.revision/,
);
assert.deepEqual(db.queries[0].values, ['scheduler-a', CLAIM_TOKEN, 30_000]);
const empty = fixture({ noDue: true });
assert.equal(
await new PostgresClusterScheduleRepository(
empty.pool,
).claimNextClusterSchedule({
ownerId: 'scheduler-a',
claimToken: CLAIM_TOKEN,
leaseMs: 30_000,
}),
null,
);
});
test('admits Run, Attempt and two events before advancing the exact claim', async () => {
const db = fixture();
const result = await new PostgresClusterScheduleRepository(
db.pool,
).commitClusterScheduleDecision(admissionCommand());
assert.deepEqual(result, {
status: 'admitted',
disposition: 'admit',
runId: '019f7700-0000-7000-8000-000000000020',
attemptId: '019f7700-0000-7000-8000-000000000021',
});
const sql = db.queries.map(({ text }) => text);
assert.match(
sql.find((text) => text.includes('FOR UPDATE OF schedule')),
/task_head\.current_revision = task\.revision/,
);
const run = sql.findIndex((text) =>
text.includes('INSERT INTO "ql3"."runs"'),
);
const attempt = sql.findIndex((text) =>
text.includes('INSERT INTO "ql3"."run_attempts"'),
);
const events = sql.filter((text) =>
text.includes('INSERT INTO "ql3"."run_events"'),
);
const advance = sql.findIndex((text) =>
text.includes('UPDATE "ql3"."trigger_schedules"'),
);
assert.ok(
run > 0 && attempt > run && events.length === 2 && advance > attempt,
);
assert.match(sql[attempt], /'remote_worker'/);
assert.match(sql[advance], /claim_token = \$10::uuid/);
assert.equal(
db.queries
.find(({ text }) => text.includes('INSERT INTO "ql3"."runs"'))
.values.at(-1),
62_000,
);
assert.equal(sql.includes('COMMIT'), true);
assert.equal(sql.at(-1), 'RELEASE');
});
test('does not retry after COMMIT was sent and its response is lost', async () => {
const db = fixture({ failCommitOnce: true });
await assert.rejects(
new PostgresClusterScheduleRepository(
db.pool,
).commitClusterScheduleDecision(admissionCommand()),
PostgresClusterScheduleUnavailableError,
);
const sql = db.queries.map(({ text }) => text);
assert.equal(sql.filter((text) => text === 'COMMIT').length, 1);
assert.equal(sql.includes('ROLLBACK'), false);
});
test('returns raced without writes when the durable claim fence changed', async () => {
const db = fixture({ claimOverrides: { stateVersion: 2 } });
assert.deepEqual(
await new PostgresClusterScheduleRepository(
db.pool,
).commitClusterScheduleDecision(admissionCommand()),
{ status: 'raced' },
);
const sql = db.queries.map(({ text }) => text);
assert.equal(
sql.some((text) => text.includes('INSERT INTO')),
false,
);
assert.equal(sql.includes('ROLLBACK'), true);
});
test('uses the database commit clock for expiry and rejects clock regression', async () => {
const expired = fixture({ commitObservedAtMs: 91_000 });
assert.deepEqual(
await new PostgresClusterScheduleRepository(
expired.pool,
).commitClusterScheduleDecision(admissionCommand()),
{ status: 'raced' },
);
assert.equal(
expired.queries.some(({ text }) => text.includes('INSERT INTO')),
false,
);
const backwards = fixture({ commitObservedAtMs: 60_999 });
await assert.rejects(
new PostgresClusterScheduleRepository(
backwards.pool,
).commitClusterScheduleDecision(admissionCommand()),
/clock moved backwards/,
);
assert.equal(
backwards.queries.some(({ text }) => text === 'ROLLBACK'),
true,
);
});
test('advances a skipped occurrence without creating a Run', async () => {
const claimed = claim({
claimAcquiredAtMs: 900_000,
claimExpiresAtMs: 930_000,
});
const db = fixture({
claimOverrides: {
claimAcquiredAtMs: 900_000,
claimExpiresAtMs: 930_000,
},
commitObservedAtMs: 900_001,
});
const decision = resolveClusterScheduleDecision(claimed, 0, nextMinute);
assert.equal(decision.disposition, 'skip');
assert.deepEqual(
await new PostgresClusterScheduleRepository(
db.pool,
).commitClusterScheduleDecision({ claim: claimed, decision }),
{ status: 'advanced', disposition: 'skip' },
);
assert.equal(
db.queries.some(({ text }) => text.includes('INSERT INTO "ql3"."runs"')),
false,
);
});
test('rolls back partial admission and rejects corrupt execution revisions', async () => {
const failed = fixture({ failAt: 'run_attempts' });
await assert.rejects(
new PostgresClusterScheduleRepository(
failed.pool,
).commitClusterScheduleDecision(admissionCommand()),
PostgresClusterScheduleUnavailableError,
);
assert.equal(
failed.queries.some(({ text }) => text === 'ROLLBACK'),
true,
);
assert.equal(
failed.queries.some(({ text }) => text === 'COMMIT'),
false,
);
const corrupt = fixture({
executionOverrides: { contentDigest: 'f'.repeat(64) },
});
await assert.rejects(
new PostgresClusterScheduleRepository(
corrupt.pool,
).commitClusterScheduleDecision(admissionCommand()),
PostgresClusterScheduleUnavailableError,
);
assert.equal(
corrupt.queries.some(({ text }) => text === 'ROLLBACK'),
true,
);
});
@@ -0,0 +1,101 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
PostgresConnectionEnvironmentError,
loadPostgresConnectionEnvironment,
} = require('../dist/entrypoints/runtime.js');
const KEYS = Object.freeze({
connectionString: 'POSTGRES_URL',
host: 'POSTGRES_HOST',
port: 'POSTGRES_PORT',
database: 'POSTGRES_DATABASE',
user: 'POSTGRES_USER',
password: 'POSTGRES_PASSWORD',
});
test('loads a legacy URL without exposing TLS query overrides', () => {
assert.deepEqual(
loadPostgresConnectionEnvironment(
{
POSTGRES_URL:
'postgresql://ql3_runtime:secret@postgres-rw.internal:5432/qinglong',
},
KEYS,
),
{
connectionString:
'postgresql://ql3_runtime:secret@postgres-rw.internal:5432/qinglong',
},
);
assert.throws(
() =>
loadPostgresConnectionEnvironment(
{
POSTGRES_URL:
'postgresql://ql3_runtime:secret@postgres-rw.internal/qinglong?sslmode=disable',
},
KEYS,
),
/TLS query parameters are forbidden/,
);
});
test('loads an exact discrete operator credential with the default port', () => {
assert.deepEqual(
loadPostgresConnectionEnvironment(
{
POSTGRES_HOST: 'ql3-postgres-rw.qinglong3-system.svc',
POSTGRES_DATABASE: 'qinglong',
POSTGRES_USER: 'ql3_runtime',
POSTGRES_PASSWORD: 'secret',
},
KEYS,
),
{
host: 'ql3-postgres-rw.qinglong3-system.svc',
port: 5432,
database: 'qinglong',
user: 'ql3_runtime',
password: 'secret',
},
);
});
test('rejects mixed, partial, unbounded and unsafe discrete credentials', () => {
for (const environment of [
{
POSTGRES_URL: 'postgresql://ql3_runtime:secret@database/qinglong',
POSTGRES_HOST: 'database',
},
{
POSTGRES_HOST: 'database',
POSTGRES_DATABASE: 'qinglong',
POSTGRES_USER: 'ql3_runtime',
},
{
POSTGRES_HOST: 'database',
POSTGRES_PORT: '0',
POSTGRES_DATABASE: 'qinglong',
POSTGRES_USER: 'ql3_runtime',
POSTGRES_PASSWORD: 'secret',
},
{
POSTGRES_HOST: 'database',
POSTGRES_DATABASE: 'qinglong',
POSTGRES_USER: 'role-with-hyphen',
POSTGRES_PASSWORD: 'secret',
},
{
POSTGRES_HOST: 'database',
POSTGRES_DATABASE: 'qinglong',
POSTGRES_USER: 'ql3_runtime',
POSTGRES_PASSWORD: 'line\nbreak',
},
]) {
assert.throws(
() => loadPostgresConnectionEnvironment(environment, KEYS),
PostgresConnectionEnvironmentError,
);
}
});
@@ -0,0 +1,880 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
TaskDefinitionConflictError,
TaskDefinitionUnavailableError,
createTaskDefinitionRecord,
} = require('@qinglong/runtime-core/task-definition');
const {
createBuiltInTaskSpecSemanticRegistry,
} = require('@qinglong/runtime-core/task-spec-semantic');
const {
TriggerConflictError,
TriggerUnavailableError,
createBuiltInTriggerSpecSemanticRegistry,
createTriggerRecord,
} = require('@qinglong/runtime-core/trigger');
const {
compileClusterCommandTaskDefinition,
} = require('@qinglong/runtime-core/cluster-execution-revision');
const {
TaskDefinitionAdministrationAuthorizationFenceConflictError,
TaskDefinitionAdministrationMutationConflictError,
TaskDefinitionAdministrationReadConflictError,
} = require('@qinglong/runtime-core/task-definition-administration');
const {
TriggerAdministrationAuthorizationFenceConflictError,
TriggerAdministrationReadConflictError,
} = require('@qinglong/runtime-core/trigger-administration');
const {
PostgresTaskDefinitionRepository,
PostgresTaskDefinitionSource,
PostgresTaskExecutionRevisionSource,
} = require('../dist/automation/taskDefinitionRepository');
const {
PostgresTriggerRepository,
PostgresTriggerSource,
} = require('../dist/scheduling/triggerRepository');
const {
PostgresTaskDefinitionAdministrationRepository,
PostgresTriggerAdministrationRepository,
} = require('../dist/automation/automationAdministrationRepository');
const TASK_COMMAND = Object.freeze({
projectId: 'default',
taskId: 'task-00001',
expectedRevision: null,
mutationId: '123e4567-e89b-42d3-a456-426614174001',
name: 'Task 1',
kind: 'command',
spec: Object.freeze({
schema: 'qinglong/command@v1',
config: Object.freeze({
command: Object.freeze({
kind: 'argv',
file: '/bin/echo',
args: Object.freeze(['1']),
}),
}),
}),
labels: Object.freeze({ source: 'postgres-test' }),
enabled: true,
occurredAtMs: 101,
});
const NORMALIZED_TASK_COMMAND = Object.freeze({
...TASK_COMMAND,
spec: createBuiltInTaskSpecSemanticRegistry().normalize({
projectId: TASK_COMMAND.projectId,
taskId: TASK_COMMAND.taskId,
kind: TASK_COMMAND.kind,
spec: TASK_COMMAND.spec,
}),
});
const TASK = createTaskDefinitionRecord(
NORMALIZED_TASK_COMMAND,
TASK_COMMAND.occurredAtMs,
);
const DRIFT_TASK = createTaskDefinitionRecord(
Object.freeze({ ...NORMALIZED_TASK_COMMAND, name: 'drift' }),
TASK_COMMAND.occurredAtMs,
);
const TASK_EXECUTION = compileClusterCommandTaskDefinition(
TASK,
createBuiltInTaskSpecSemanticRegistry(),
);
const TRIGGER_COMMAND = Object.freeze({
projectId: 'default',
triggerId: 'trigger-00001',
expectedRevision: null,
mutationId: '123e4567-e89b-42d3-a456-426614174002',
taskId: TASK.taskId,
taskRevision: TASK.revision,
taskContentDigest: TASK.contentDigest,
spec: Object.freeze({
schema: 'qinglong/cron@v1',
config: Object.freeze({
expression: '*/5 * * * *',
timezone: 'Etc/UTC',
misfirePolicy: 'skip',
}),
}),
enabled: true,
occurredAtMs: 201,
});
const NORMALIZED_TRIGGER_COMMAND = Object.freeze({
...TRIGGER_COMMAND,
spec: createBuiltInTriggerSpecSemanticRegistry().normalize({
projectId: TRIGGER_COMMAND.projectId,
triggerId: TRIGGER_COMMAND.triggerId,
taskId: TRIGGER_COMMAND.taskId,
taskRevision: TRIGGER_COMMAND.taskRevision,
spec: TRIGGER_COMMAND.spec,
}),
});
const TRIGGER = createTriggerRecord(
NORMALIZED_TRIGGER_COMMAND,
TRIGGER_COMMAND.occurredAtMs,
);
const DRIFT_TRIGGER = createTriggerRecord(
Object.freeze({ ...NORMALIZED_TRIGGER_COMMAND, enabled: false }),
TRIGGER_COMMAND.occurredAtMs,
);
const ACTOR = Object.freeze({ type: 'user', id: 'usr_cluster_admin' });
const FENCE = Object.freeze({ projectVersion: 1, bindingVersion: 1 });
const TASK_AUDIT = Object.freeze({
eventId: TASK_COMMAND.mutationId,
requestId: 'request-task-create-00001',
operationId: 'task.create',
projectId: TASK_COMMAND.projectId,
subject: ACTOR,
authenticationId: 'oidc:session-task-00001',
outcome: 'allowed',
reasons: Object.freeze(['role_grant']),
fence: FENCE,
occurredAtMs: 102,
});
const TRIGGER_AUDIT = Object.freeze({
...TASK_AUDIT,
eventId: TRIGGER_COMMAND.mutationId,
requestId: 'request-trigger-create-00001',
operationId: 'trigger.create',
occurredAtMs: 202,
});
const TASK_READ_AUDIT = Object.freeze({
...TASK_AUDIT,
eventId: '123e4567-e89b-42d3-a456-426614174010',
requestId: 'request-task-read-00001',
operationId: 'task.read',
occurredAtMs: 302,
});
const TRIGGER_READ_AUDIT = Object.freeze({
...TASK_AUDIT,
eventId: '123e4567-e89b-42d3-a456-426614174011',
requestId: 'request-trigger-read-00001',
operationId: 'trigger.read',
occurredAtMs: 303,
});
function administrationAuditRow(audit) {
return {
auditEventId: audit.eventId,
auditRequestId: audit.requestId,
auditOperationId: audit.operationId,
auditProjectId: audit.projectId,
auditSubjectType: audit.subject.type,
auditSubjectId: audit.subject.id,
auditAuthenticationId: audit.authenticationId,
auditOutcome: audit.outcome,
auditReasons: [...audit.reasons],
auditProjectVersion: audit.fence.projectVersion,
auditBindingVersion: audit.fence.bindingVersion,
auditOccurredAtMs: String(audit.occurredAtMs),
};
}
function taskRow(overrides = {}) {
return {
projectId: TASK.projectId,
taskId: TASK.taskId,
revision: TASK.revision,
mutationId: TASK.mutationId,
name: TASK.name,
description: null,
kind: TASK.kind,
specJson: TASK.spec,
labelsJson: TASK.labels,
enabled: TASK.enabled,
contentDigest: TASK.contentDigest,
createdAtMs: String(TASK.createdAtMs),
updatedAtMs: String(TASK.updatedAtMs),
...overrides,
};
}
function triggerRow(overrides = {}) {
return {
projectId: TRIGGER.projectId,
triggerId: TRIGGER.triggerId,
revision: TRIGGER.revision,
mutationId: TRIGGER.mutationId,
taskId: TRIGGER.taskId,
taskRevision: TRIGGER.taskRevision,
taskContentDigest: TRIGGER.taskContentDigest,
specJson: TRIGGER.spec,
enabled: TRIGGER.enabled,
contentDigest: TRIGGER.contentDigest,
createdAtMs: String(TRIGGER.createdAtMs),
updatedAtMs: String(TRIGGER.updatedAtMs),
...overrides,
};
}
function taskExecutionRow(overrides = {}) {
return {
projectId: TASK_EXECUTION.projectId,
taskId: TASK_EXECUTION.taskId,
sourceRevision: TASK_EXECUTION.sourceRevision,
taskRevision: TASK_EXECUTION.taskRevision,
sourceContentDigest: TASK_EXECUTION.sourceContentDigest,
executorType: TASK_EXECUTION.executorType,
planSchema: TASK_EXECUTION.planSchema,
planJson: {
command: TASK_EXECUTION.command,
environment: TASK_EXECUTION.environment,
...(TASK_EXECUTION.workingDirectory === undefined
? {}
: { workingDirectory: TASK_EXECUTION.workingDirectory }),
...(TASK_EXECUTION.timeoutMs === undefined
? {}
: { timeoutMs: TASK_EXECUTION.timeoutMs }),
...(TASK_EXECUTION.placement === undefined
? {}
: { placement: TASK_EXECUTION.placement }),
},
contentDigest: TASK_EXECUTION.contentDigest,
createdAtMs: String(TASK_EXECUTION.createdAtMs),
...overrides,
};
}
function sourcePool(rows) {
const queries = [];
return {
queries,
pool: {
async query(text, values) {
queries.push({ text, values });
return { rows };
},
async connect() {
throw new Error('not used');
},
},
};
}
function appendPool(kind, options = {}) {
const events = [];
const queries = [];
let connection = 0;
return {
events,
queries,
pool: {
async query() {
throw new Error('not used');
},
async connect() {
connection += 1;
return {
async query(text, values) {
events.push(text.trim().split('\n', 1)[0]);
queries.push({ text, values });
if (text.includes('WHERE revision.mutation_id = $1')) {
return {
rows: options.replay
? [kind === 'task' ? taskRow(options.replay) : triggerRow(options.replay)]
: [],
};
}
if (text.includes('FROM "ql3"."task_execution_revisions"')) {
return {
rows: options.missingExecution ? [] : [taskExecutionRow()],
};
}
if (text.includes('FROM "ql3"."trigger_schedules"')) {
return {
rows: options.missingSchedule
? []
: [{ triggerRevision: TRIGGER.revision }],
};
}
if (text.includes('FROM "ql3"."projects"')) {
return {
rows: [{
status: options.projectStatus ?? 'active',
version: options.projectVersion ?? 1,
}],
};
}
if (text.includes('FROM "ql3"."project_role_bindings"')) {
return {
rows: options.missingBinding
? []
: [{
version: options.bindingVersion ?? 1,
state: options.bindingState ?? 'active',
}],
};
}
if (text.includes('FROM "ql3"."security_audit_events"')) {
return {
rows: options.audit ? [administrationAuditRow(options.audit)] : [],
};
}
if (text.includes('INSERT INTO "ql3"."task_definitions"')) {
return { rows: [{ taskId: TASK.taskId }], rowCount: 1 };
}
if (text.includes('FROM "ql3"."task_definitions"') &&
text.includes('FOR UPDATE')) {
return {
rows: [{
currentRevision: 1,
createdAtMs: '101',
updatedAtMs: '101',
}],
};
}
if (text.includes('INSERT INTO "ql3"."task_definition_revisions"')) {
if (options.retryOnce && connection === 1) {
throw Object.assign(new Error('serialization'), { code: '40001' });
}
return { rows: [], rowCount: 1 };
}
if (text.includes('INSERT INTO "ql3"."triggers"')) {
return { rows: [{ triggerId: TRIGGER.triggerId }], rowCount: 1 };
}
if (text.includes('FROM "ql3"."triggers"') &&
text.includes('FOR UPDATE')) {
return {
rows: [{
taskId: TASK.taskId,
currentRevision: 1,
createdAtMs: '201',
updatedAtMs: '201',
}],
};
}
if (text.includes('FROM "ql3"."task_definitions"') &&
text.includes('revision.revision = $3')) {
return { rows: options.missingTask ? [] : [taskRow(options.task)] };
}
if (text.includes('FROM "ql3"."task_definitions"') &&
text.includes('JOIN "ql3"."task_definition_revisions"')) {
return { rows: options.readAbsent ? [] : [taskRow(options.task)] };
}
if (text.includes('FROM "ql3"."triggers"') &&
text.includes('JOIN "ql3"."trigger_revisions"')) {
return {
rows: options.readAbsent ? [] : [triggerRow(options.trigger)],
};
}
if (text.includes('INSERT INTO "ql3"."trigger_revisions"')) {
return { rows: [], rowCount: 1 };
}
return { rows: [], rowCount: 1 };
},
release() {
events.push(`release:${connection}`);
},
};
},
},
};
}
test('runtime sources read normalized immutable facts without write authority', async () => {
const taskFixture = sourcePool([taskRow()]);
const taskSource = new PostgresTaskDefinitionSource(taskFixture.pool);
assert.deepEqual(
await taskSource.findCurrentTaskDefinition(TASK.projectId, TASK.taskId),
TASK,
);
assert.equal('appendTaskDefinitionRevision' in taskSource, false);
assert.deepEqual(taskFixture.queries[0].values, [TASK.projectId, TASK.taskId]);
const listed = sourcePool([taskRow()]);
assert.equal(
(await new PostgresTaskDefinitionSource(listed.pool).listTaskDefinitions({
projectId: TASK.projectId,
limit: 1,
})).definitions.length,
1,
);
assert.equal(
(listed.queries[0].text.match(/ORDER BY/g) ?? []).length,
1,
);
const executionSource = new PostgresTaskExecutionRevisionSource(
sourcePool([taskExecutionRow()]).pool,
);
assert.deepEqual(
await executionSource.resolveClusterTaskExecutionRevision({
projectId: TASK.projectId,
taskId: TASK.taskId,
sourceRevision: TASK.revision,
}),
TASK_EXECUTION,
);
const triggerFixture = sourcePool([triggerRow()]);
const triggerSource = new PostgresTriggerSource(triggerFixture.pool);
assert.deepEqual(
await triggerSource.findCurrentTrigger(TRIGGER.projectId, TRIGGER.triggerId),
TRIGGER,
);
assert.equal('appendTriggerRevision' in triggerSource, false);
});
test('publishes TaskDefinition atomically and replays the exact mutation', async () => {
const fixture = appendPool('task');
const created = await new PostgresTaskDefinitionRepository(
fixture.pool,
).appendTaskDefinitionRevision(TASK_COMMAND);
assert.deepEqual(created, { status: 'created', definition: TASK });
assert.ok(fixture.events.includes('BEGIN ISOLATION LEVEL SERIALIZABLE'));
assert.ok(fixture.events.includes('COMMIT'));
assert.equal(
fixture.events.some((event) =>
event.includes('INSERT INTO "ql3"."task_execution_revisions"'),
),
true,
);
assert.equal(fixture.events.at(-1), 'release:1');
const replayFixture = appendPool('task', { replay: {} });
assert.deepEqual(
await new PostgresTaskDefinitionRepository(
replayFixture.pool,
).appendTaskDefinitionRevision(TASK_COMMAND),
{ status: 'existing', definition: TASK },
);
assert.equal(
replayFixture.events.some((event) => event.includes('INSERT INTO')),
false,
);
});
test('runs TaskDefinition transaction hooks for create and replay before COMMIT', async () => {
const createdFixture = appendPool('task');
const createdHook = [];
await new PostgresTaskDefinitionRepository(
createdFixture.pool,
).appendTaskDefinitionRevision(TASK_COMMAND, async (client, context) => {
createdHook.push(context);
await client.query('SELECT 1 AS task_hook');
});
assert.equal(createdHook.length, 1);
assert.equal(createdHook[0].replay, null);
assert.deepEqual(createdHook[0].record, TASK);
assert.ok(
createdFixture.events.indexOf('SELECT 1 AS task_hook') <
createdFixture.events.indexOf('COMMIT'),
);
const replayFixture = appendPool('task', { replay: {} });
const replayHook = [];
await new PostgresTaskDefinitionRepository(
replayFixture.pool,
).appendTaskDefinitionRevision(TASK_COMMAND, async (_client, context) => {
replayHook.push(context);
});
assert.deepEqual(replayHook[0].replay, TASK);
assert.deepEqual(replayHook[0].record, TASK);
const deniedFixture = appendPool('task');
const denial = new Error('task authorization fence changed');
await assert.rejects(
new PostgresTaskDefinitionRepository(
deniedFixture.pool,
).appendTaskDefinitionRevision(TASK_COMMAND, async () => {
throw denial;
}),
(error) => error === denial,
);
assert.equal(deniedFixture.events.includes('ROLLBACK'), true);
assert.equal(deniedFixture.events.includes('COMMIT'), false);
});
test('retries serialization and fails closed on TaskDefinition drift or corruption', async () => {
const retry = appendPool('task', { retryOnce: true });
assert.equal(
(
await new PostgresTaskDefinitionRepository(
retry.pool,
).appendTaskDefinitionRevision(TASK_COMMAND)
).status,
'created',
);
assert.equal(retry.events.filter((event) => event.startsWith('release:')).length, 2);
await assert.rejects(
new PostgresTaskDefinitionRepository(
appendPool('task', {
replay: {
name: DRIFT_TASK.name,
contentDigest: DRIFT_TASK.contentDigest,
},
}).pool,
).appendTaskDefinitionRevision(TASK_COMMAND),
TaskDefinitionConflictError,
);
await assert.rejects(
new PostgresTaskDefinitionRepository(
appendPool('task', { replay: {}, missingExecution: true }).pool,
).appendTaskDefinitionRevision(TASK_COMMAND),
TaskDefinitionUnavailableError,
);
await assert.rejects(
new PostgresTaskDefinitionSource(sourcePool([
taskRow({ contentDigest: 'b'.repeat(64) }),
]).pool).findCurrentTaskDefinition(TASK.projectId, TASK.taskId),
TaskDefinitionUnavailableError,
);
});
test('publishes Trigger only when its immutable task pin is valid', async () => {
const fixture = appendPool('trigger');
const created = await new PostgresTriggerRepository(
fixture.pool,
).appendTriggerRevision(TRIGGER_COMMAND);
assert.deepEqual(created, { status: 'created', trigger: TRIGGER });
assert.ok(
fixture.events.findIndex((event) => event.includes('task_definitions')) <
fixture.events.findIndex((event) => event.includes('trigger_revisions')),
);
assert.equal(
fixture.events.some((event) =>
event.includes('INSERT INTO "ql3"."trigger_schedules"'),
),
true,
);
const createdPin = fixture.queries.find(({ text }) =>
text.includes('$4::boolean = false'),
);
assert.equal(createdPin.values[3], true);
const replayFixture = appendPool('trigger', { replay: {} });
assert.deepEqual(
await new PostgresTriggerRepository(
replayFixture.pool,
).appendTriggerRevision(TRIGGER_COMMAND),
{ status: 'existing', trigger: TRIGGER },
);
const replayPin = replayFixture.queries.find(({ text }) =>
text.includes('$4::boolean = false'),
);
assert.equal(replayPin.values[3], false);
await assert.rejects(
new PostgresTriggerRepository(
appendPool('trigger', { missingTask: true }).pool,
).appendTriggerRevision(TRIGGER_COMMAND),
TriggerConflictError,
);
await assert.rejects(
new PostgresTriggerRepository(
appendPool('trigger', { task: { contentDigest: 'b'.repeat(64) } }).pool,
).appendTriggerRevision(TRIGGER_COMMAND),
TriggerUnavailableError,
);
});
test('runs Trigger transaction hooks atomically and preserves hook failures', async () => {
const createdFixture = appendPool('trigger');
const contexts = [];
await new PostgresTriggerRepository(
createdFixture.pool,
).appendTriggerRevision(TRIGGER_COMMAND, async (client, context) => {
contexts.push(context);
await client.query('SELECT 1 AS trigger_hook');
});
assert.equal(contexts.length, 1);
assert.equal(contexts[0].replay, null);
assert.deepEqual(contexts[0].record, TRIGGER);
assert.ok(
createdFixture.events.indexOf('SELECT 1 AS trigger_hook') <
createdFixture.events.indexOf('COMMIT'),
);
const replayFixture = appendPool('trigger', { replay: {} });
await new PostgresTriggerRepository(
replayFixture.pool,
).appendTriggerRevision(TRIGGER_COMMAND, async (_client, context) => {
contexts.push(context);
});
assert.deepEqual(contexts[1].replay, TRIGGER);
const deniedFixture = appendPool('trigger');
const denial = new Error('trigger authorization fence changed');
await assert.rejects(
new PostgresTriggerRepository(
deniedFixture.pool,
).appendTriggerRevision(TRIGGER_COMMAND, async () => {
throw denial;
}),
(error) => error === denial,
);
assert.equal(deniedFixture.events.includes('ROLLBACK'), true);
assert.equal(deniedFixture.events.includes('COMMIT'), false);
});
test('Trigger replay rejects mutation drift and corrupt durable records', async () => {
await assert.rejects(
new PostgresTriggerRepository(
appendPool('trigger', {
replay: {
enabled: DRIFT_TRIGGER.enabled,
contentDigest: DRIFT_TRIGGER.contentDigest,
},
}).pool,
).appendTriggerRevision(TRIGGER_COMMAND),
TriggerConflictError,
);
await assert.rejects(
new PostgresTriggerRepository(
appendPool('trigger', {
replay: {},
missingSchedule: true,
}).pool,
).appendTriggerRevision(TRIGGER_COMMAND),
TriggerUnavailableError,
);
await assert.rejects(
new PostgresTriggerSource(sourcePool([
triggerRow({ contentDigest: 'b'.repeat(64) }),
]).pool).findCurrentTrigger(TRIGGER.projectId, TRIGGER.triggerId),
TriggerUnavailableError,
);
});
test('atomically fences and audits authorized PostgreSQL Task mutations', async () => {
const createdFixture = appendPool('task');
assert.deepEqual(
await new PostgresTaskDefinitionAdministrationRepository(
createdFixture.pool,
).appendAuthorizedTaskDefinitionRevision({
command: TASK_COMMAND,
actor: ACTOR,
fence: FENCE,
audit: TASK_AUDIT,
}),
{ status: 'created', definition: TASK },
);
const createdSql = createdFixture.queries.map(({ text }) => text);
assert.ok(
createdSql.findIndex((text) => text.includes('project_role_bindings')) <
createdSql.findIndex((text) => text.includes('security_audit_events')),
);
assert.ok(
createdSql.findIndex((text) => text.includes('security_audit_events')) <
createdSql.indexOf('COMMIT'),
);
const replayFixture = appendPool('task', {
replay: {},
audit: TASK_AUDIT,
});
assert.equal(
(
await new PostgresTaskDefinitionAdministrationRepository(
replayFixture.pool,
).appendAuthorizedTaskDefinitionRevision({
command: TASK_COMMAND,
actor: ACTOR,
fence: FENCE,
audit: { ...TASK_AUDIT, occurredAtMs: 999 },
})
).status,
'existing',
);
assert.equal(
replayFixture.queries.some(({ text }) =>
text.includes('INSERT INTO "ql3"."security_audit_events"'),
),
false,
);
const fencedFixture = appendPool('task', { bindingVersion: 2 });
await assert.rejects(
new PostgresTaskDefinitionAdministrationRepository(
fencedFixture.pool,
).appendAuthorizedTaskDefinitionRevision({
command: TASK_COMMAND,
actor: ACTOR,
fence: FENCE,
audit: TASK_AUDIT,
}),
TaskDefinitionAdministrationAuthorizationFenceConflictError,
);
assert.equal(fencedFixture.events.includes('ROLLBACK'), true);
await assert.rejects(
new PostgresTaskDefinitionAdministrationRepository(
appendPool('task', { audit: TASK_AUDIT }).pool,
).appendAuthorizedTaskDefinitionRevision({
command: TASK_COMMAND,
actor: ACTOR,
fence: FENCE,
audit: TASK_AUDIT,
}),
TaskDefinitionAdministrationMutationConflictError,
);
});
test('atomically fences and audits authorized PostgreSQL Trigger mutations', async () => {
const createdFixture = appendPool('trigger');
assert.equal(
(
await new PostgresTriggerAdministrationRepository(
createdFixture.pool,
).appendAuthorizedTriggerRevision({
command: TRIGGER_COMMAND,
actor: ACTOR,
fence: FENCE,
audit: TRIGGER_AUDIT,
})
).status,
'created',
);
assert.equal(
createdFixture.queries.some(({ text }) =>
text.includes('INSERT INTO "ql3"."security_audit_events"'),
),
true,
);
const fencedFixture = appendPool('trigger', { projectVersion: 2 });
await assert.rejects(
new PostgresTriggerAdministrationRepository(
fencedFixture.pool,
).appendAuthorizedTriggerRevision({
command: TRIGGER_COMMAND,
actor: ACTOR,
fence: FENCE,
audit: TRIGGER_AUDIT,
}),
TriggerAdministrationAuthorizationFenceConflictError,
);
assert.equal(fencedFixture.events.includes('ROLLBACK'), true);
});
test('atomically fences, reads and audits current PostgreSQL automation facts', async () => {
const taskFixture = appendPool('task');
const taskRepository = new PostgresTaskDefinitionAdministrationRepository(
taskFixture.pool,
);
assert.deepEqual(
await taskRepository.findAuthorizedCurrentTaskDefinition({
projectId: TASK.projectId,
taskId: TASK.taskId,
actor: ACTOR,
fence: FENCE,
audit: TASK_READ_AUDIT,
}),
TASK,
);
const taskSql = taskFixture.queries.map(({ text }) => text);
assert.ok(
taskSql.findIndex((text) => text.includes('FROM "ql3"."projects"')) <
taskSql.findIndex((text) =>
text.includes('JOIN "ql3"."task_definition_revisions"'),
),
);
assert.ok(
taskSql.findIndex((text) =>
text.includes('JOIN "ql3"."task_definition_revisions"'),
) <
taskSql.findIndex((text) =>
text.includes('INSERT INTO "ql3"."security_audit_events"'),
),
);
assert.ok(
taskSql.findIndex((text) =>
text.includes('INSERT INTO "ql3"."security_audit_events"'),
) < taskSql.indexOf('COMMIT'),
);
const triggerFixture = appendPool('trigger');
assert.deepEqual(
await new PostgresTriggerAdministrationRepository(
triggerFixture.pool,
).findAuthorizedCurrentTrigger({
projectId: TRIGGER.projectId,
triggerId: TRIGGER.triggerId,
actor: ACTOR,
fence: FENCE,
audit: TRIGGER_READ_AUDIT,
}),
TRIGGER,
);
assert.equal(
triggerFixture.queries.some(({ text }) =>
text.includes('INSERT INTO "ql3"."security_audit_events"'),
),
true,
);
});
test('reads bounded automation pages and fails closed on fence or audit replay', async () => {
const taskFixture = appendPool('task');
const tasks = await new PostgresTaskDefinitionAdministrationRepository(
taskFixture.pool,
).listAuthorizedTaskDefinitions({
projectId: TASK.projectId,
limit: 1,
actor: ACTOR,
fence: FENCE,
audit: TASK_READ_AUDIT,
});
assert.deepEqual(tasks, { definitions: [TASK], truncated: false });
const triggerFixture = appendPool('trigger');
const triggers = await new PostgresTriggerAdministrationRepository(
triggerFixture.pool,
).listAuthorizedTriggers({
projectId: TRIGGER.projectId,
limit: 1,
actor: ACTOR,
fence: FENCE,
audit: TRIGGER_READ_AUDIT,
});
assert.deepEqual(triggers, { triggers: [TRIGGER], truncated: false });
const fenced = appendPool('task', { bindingVersion: 2 });
await assert.rejects(
new PostgresTaskDefinitionAdministrationRepository(
fenced.pool,
).findAuthorizedCurrentTaskDefinition({
projectId: TASK.projectId,
taskId: TASK.taskId,
actor: ACTOR,
fence: FENCE,
audit: TASK_READ_AUDIT,
}),
TaskDefinitionAdministrationAuthorizationFenceConflictError,
);
assert.equal(
fenced.queries.some(({ text }) =>
text.includes('JOIN "ql3"."task_definition_revisions"'),
),
false,
);
await assert.rejects(
new PostgresTaskDefinitionAdministrationRepository(
appendPool('task', { audit: TASK_READ_AUDIT }).pool,
).findAuthorizedCurrentTaskDefinition({
projectId: TASK.projectId,
taskId: TASK.taskId,
actor: ACTOR,
fence: FENCE,
audit: TASK_READ_AUDIT,
}),
TaskDefinitionAdministrationReadConflictError,
);
await assert.rejects(
new PostgresTriggerAdministrationRepository(
appendPool('trigger', { audit: TRIGGER_READ_AUDIT }).pool,
).findAuthorizedCurrentTrigger({
projectId: TRIGGER.projectId,
triggerId: TRIGGER.triggerId,
actor: ACTOR,
fence: FENCE,
audit: TRIGGER_READ_AUDIT,
}),
TriggerAdministrationReadConflictError,
);
});
@@ -0,0 +1,424 @@
const assert = require('node:assert/strict');
const path = require('node:path');
const { spawnSync } = require('node:child_process');
const { test } = require('node:test');
test('runtime export excludes executable migration DDL modules', () => {
const packageDirectory = path.resolve(__dirname, '..');
const script = `
const runtime = require('@qinglong/cluster-postgres/runtime');
const loaded = Object.keys(require.cache)
.filter((file) => file.includes('/ql3-cluster-postgres/dist/'))
.map((file) => file.replaceAll('\\\\', '/'));
process.stdout.write(JSON.stringify({
hasRepository: typeof runtime.PostgresRunRepository === 'function',
hasSecretAuthority: typeof runtime.PostgresRemoteWorkerSecretDeliveryAuthorityRepository === 'function',
hasReadiness: typeof runtime.assertPostgresSchemaReady === 'function',
loaded,
}));
`;
const result = spawnSync(process.execPath, ['-e', script], {
cwd: packageDirectory,
encoding: 'utf8',
});
assert.equal(result.status, 0, result.stderr);
const report = JSON.parse(result.stdout);
assert.equal(report.hasRepository, true);
assert.equal(report.hasSecretAuthority, true);
assert.equal(report.hasReadiness, true);
assert.equal(
report.loaded.some(
(file) =>
/\/dist\/migrations\/pg-\d/.test(file) ||
file.endsWith('/dist/migration/migrate.js') ||
file.endsWith('/dist/migration/migration.js') ||
file.endsWith('/dist/schema/schema.js'),
),
false,
report.loaded.join('\n'),
);
});
test('Worker ingress export cannot acquire runtime Secret authority', () => {
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
assert.equal(
ingress.PostgresRemoteWorkerSecretDeliveryAuthorityRepository,
undefined,
);
assert.equal(
ingress.PostgresRemoteWorkerCompletionRepository,
undefined,
);
assert.equal(
ingress.PostgresRemoteWorkerLeaseControlRepository,
undefined,
);
});
test('migration export exposes the reviewed runner through a public subpath', () => {
const migration = require('@qinglong/cluster-postgres/migration');
assert.equal(typeof migration.runPostgresMigrations, 'function');
assert.deepEqual(
migration.postgresqlMainMigrationManifest.migrations,
migration.postgresqlMainMigrationStream.migrations.map(
({ id, checksum }) => ({ id, checksum }),
),
);
});
test('admin export exposes administration authority without migration DDL', () => {
const packageDirectory = path.resolve(__dirname, '..');
const script = `
const admin = require('@qinglong/cluster-postgres/admin');
const loaded = Object.keys(require.cache)
.filter((file) => file.includes('/ql3-cluster-postgres/dist/'))
.map((file) => file.replaceAll('\\\\', '/'));
process.stdout.write(JSON.stringify({
hasIdentityAdministration: typeof admin.PostgresIdentityAdministrationRepository === 'function',
hasCredentialAdministration: typeof admin.PostgresApiCredentialAdministrationRepository === 'function',
hasAuditQuery: typeof admin.PostgresSecurityAuditQueryRepository === 'function',
hasReadiness: typeof admin.assertPostgresAdminSchemaReady === 'function',
loaded,
}));
`;
const result = spawnSync(process.execPath, ['-e', script], {
cwd: packageDirectory,
encoding: 'utf8',
});
assert.equal(result.status, 0, result.stderr);
const report = JSON.parse(result.stdout);
assert.equal(report.hasIdentityAdministration, true);
assert.equal(report.hasCredentialAdministration, true);
assert.equal(report.hasAuditQuery, true);
assert.equal(report.hasReadiness, true);
assert.equal(
report.loaded.some(
(file) =>
/\/dist\/migrations\/pg-\d/.test(file) ||
file.endsWith('/dist/migration/migrate.js') ||
file.endsWith('/dist/migration/migration.js') ||
file.endsWith('/dist/schema/schema.js'),
),
false,
report.loaded.join('\n'),
);
});
test('Plugin Package install authority is isolated behind its explicit subpath', () => {
const root = require('@qinglong/cluster-postgres');
const runtime = require('@qinglong/cluster-postgres/runtime');
const admin = require('@qinglong/cluster-postgres/admin');
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
const authority = require('@qinglong/cluster-postgres/plugin-package-install');
assert.equal(root.PostgresPluginPackageInstallRepository, undefined);
assert.equal(runtime.PostgresPluginPackageInstallRepository, undefined);
assert.equal(admin.PostgresPluginPackageInstallRepository, undefined);
assert.equal(ingress.PostgresPluginPackageInstallRepository, undefined);
assert.equal(
typeof authority.PostgresPluginPackageInstallRepository,
'function',
);
});
test('Package manager exposes inventory read authority without install mutation authority', () => {
const manager = require('@qinglong/cluster-postgres/package-manager');
assert.equal(
typeof manager.PostgresPluginPackageInstallInventoryReader,
'function',
);
assert.equal(manager.PostgresPluginPackageInstallRepository, undefined);
const reader = new manager.PostgresPluginPackageInstallInventoryReader({
async query() {
return { rows: [], rowCount: 0 };
},
});
assert.deepEqual(
Object.getOwnPropertyNames(Object.getPrototypeOf(reader)).sort(),
['constructor', 'findCurrent', 'listCurrentPage'],
);
});
test('Worker credential authorities expose disjoint management and execution capabilities', () => {
const manager = require('@qinglong/cluster-postgres/worker-credential-manager');
const executor = require('@qinglong/cluster-postgres/worker-credential-executor');
assert.equal(
typeof manager.assertPostgresWorkerCredentialManagerSchemaReady,
'function',
);
assert.equal(
typeof manager.PostgresWorkerCredentialManagementPlanRepository,
'function',
);
assert.equal(manager.PostgresWorkerCredentialAdministrationRepository, undefined);
assert.equal(
manager.PostgresRemoteWorkerSecretDeliveryAuthorityRepository,
undefined,
);
assert.equal(
typeof executor.assertPostgresWorkerCredentialExecutorSchemaReady,
'function',
);
assert.equal(
typeof executor.PostgresWorkerCredentialManagementPlanReader,
'function',
);
assert.equal(
typeof executor.PostgresWorkerCredentialAdministrationRepository,
'function',
);
assert.equal(
typeof executor.PostgresRemoteWorkerSecretDeliveryAuthorityRepository,
'function',
);
assert.equal(executor.PostgresWorkerCredentialManagementPlanRepository, undefined);
});
test('Package manager inventory reader performs only bounded read queries', async () => {
const { PostgresPluginPackageInstallInventoryReader } = require(
'@qinglong/cluster-postgres/package-manager'
);
const queries = [];
const reader = new PostgresPluginPackageInstallInventoryReader({
async query(text, values) {
queries.push({ text, values });
return { rows: [], rowCount: 0 };
},
});
assert.equal(await reader.findCurrent('project-reader', 'package-reader'), null);
assert.deepEqual(
await reader.listCurrentPage({
projectId: 'project-reader',
limit: 8,
after: { packageName: 'package-before' },
}),
{ items: [], truncated: false },
);
assert.equal(queries.length, 2);
assert.match(queries[0].text, /SELECT/);
assert.deepEqual(queries[0].values, ['project-reader', 'package-reader']);
assert.match(queries[1].text, /ORDER BY head\.package_name/);
assert.deepEqual(queries[1].values, [
'project-reader',
'package-before',
9,
]);
});
test('StepRun authority is limited to runtime composition and explicit subpath', () => {
const root = require('@qinglong/cluster-postgres');
const runtime = require('@qinglong/cluster-postgres/runtime');
const admin = require('@qinglong/cluster-postgres/admin');
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
const authority = require('@qinglong/cluster-postgres/step-run');
assert.equal(root.PostgresStepRunRepository, undefined);
assert.equal(
runtime.PostgresStepRunRepository,
authority.PostgresStepRunRepository,
);
assert.equal(admin.PostgresStepRunRepository, undefined);
assert.equal(ingress.PostgresStepRunRepository, undefined);
assert.equal(typeof authority.PostgresStepRunRepository, 'function');
});
test('Tool execution evidence authority is isolated behind its explicit subpath', () => {
const root = require('@qinglong/cluster-postgres');
const runtime = require('@qinglong/cluster-postgres/runtime');
const admin = require('@qinglong/cluster-postgres/admin');
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
const authority = require('@qinglong/cluster-postgres/tool-execution-evidence');
assert.equal(root.PostgresToolExecutionEvidenceRepository, undefined);
assert.equal(runtime.PostgresToolExecutionEvidenceRepository, undefined);
assert.equal(admin.PostgresToolExecutionEvidenceRepository, undefined);
assert.equal(ingress.PostgresToolExecutionEvidenceRepository, undefined);
assert.equal(
typeof authority.PostgresToolExecutionEvidenceRepository,
'function',
);
});
test('Tool start barrier authority is limited to runtime composition and explicit subpath', () => {
const root = require('@qinglong/cluster-postgres');
const runtime = require('@qinglong/cluster-postgres/runtime');
const admin = require('@qinglong/cluster-postgres/admin');
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
const authority = require('@qinglong/cluster-postgres/tool-execution-start-barrier');
assert.equal(root.PostgresToolExecutionStartBarrierRepository, undefined);
assert.equal(
runtime.PostgresToolExecutionStartBarrierRepository,
authority.PostgresToolExecutionStartBarrierRepository,
);
assert.equal(admin.PostgresToolExecutionStartBarrierRepository, undefined);
assert.equal(ingress.PostgresToolExecutionStartBarrierRepository, undefined);
assert.equal(
typeof authority.PostgresToolExecutionStartBarrierRepository,
'function',
);
});
test('Tool completion authority is limited to runtime composition and explicit subpath', () => {
const root = require('@qinglong/cluster-postgres');
const runtime = require('@qinglong/cluster-postgres/runtime');
const admin = require('@qinglong/cluster-postgres/admin');
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
const authority = require('@qinglong/cluster-postgres/tool-execution-completion');
assert.equal(root.PostgresToolExecutionCompletionRepository, undefined);
assert.equal(
runtime.PostgresToolExecutionCompletionRepository,
authority.PostgresToolExecutionCompletionRepository,
);
assert.equal(admin.PostgresToolExecutionCompletionRepository, undefined);
assert.equal(ingress.PostgresToolExecutionCompletionRepository, undefined);
assert.equal(
typeof authority.PostgresToolExecutionCompletionRepository,
'function',
);
});
test('Tool failure completion authority is limited to runtime composition and explicit subpath', () => {
const root = require('@qinglong/cluster-postgres');
const runtime = require('@qinglong/cluster-postgres/runtime');
const admin = require('@qinglong/cluster-postgres/admin');
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
const authority = require('@qinglong/cluster-postgres/tool-execution-failure-completion');
assert.equal(
root.PostgresToolExecutionFailureCompletionRepository,
undefined,
);
assert.equal(
runtime.PostgresToolExecutionFailureCompletionRepository,
authority.PostgresToolExecutionFailureCompletionRepository,
);
assert.equal(
admin.PostgresToolExecutionFailureCompletionRepository,
undefined,
);
assert.equal(
ingress.PostgresToolExecutionFailureCompletionRepository,
undefined,
);
assert.equal(
typeof authority.PostgresToolExecutionFailureCompletionRepository,
'function',
);
});
test('Tool result key catalog splits runtime read from admin mutation authority', () => {
const root = require('@qinglong/cluster-postgres');
const runtime = require('@qinglong/cluster-postgres/runtime');
const admin = require('@qinglong/cluster-postgres/admin');
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
const authority = require('@qinglong/cluster-postgres/tool-result-key-catalog');
assert.equal(root.PostgresToolResultKeyCatalogReader, undefined);
assert.equal(root.PostgresToolResultKeyCatalogRepository, undefined);
assert.equal(
runtime.PostgresToolResultKeyCatalogReader,
authority.PostgresToolResultKeyCatalogReader,
);
assert.equal(runtime.PostgresToolResultKeyCatalogRepository, undefined);
assert.equal(
admin.PostgresToolResultKeyCatalogRepository,
authority.PostgresToolResultKeyCatalogRepository,
);
assert.equal(admin.PostgresToolResultKeyCatalogReader, undefined);
assert.equal(ingress.PostgresToolResultKeyCatalogReader, undefined);
assert.equal(ingress.PostgresToolResultKeyCatalogRepository, undefined);
});
test('Tool result rekey splits runtime read from admin mutation authority', () => {
const root = require('@qinglong/cluster-postgres');
const runtime = require('@qinglong/cluster-postgres/runtime');
const admin = require('@qinglong/cluster-postgres/admin');
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
const authority = require('@qinglong/cluster-postgres/tool-result-rekey');
assert.equal(root.PostgresToolResultRekeyReader, undefined);
assert.equal(root.PostgresToolResultRekeyRepository, undefined);
assert.equal(
runtime.PostgresToolResultRekeyReader,
authority.PostgresToolResultRekeyReader,
);
assert.equal(runtime.PostgresToolResultRekeyRepository, undefined);
assert.equal(
admin.PostgresToolResultRekeyRepository,
authority.PostgresToolResultRekeyRepository,
);
assert.equal(admin.PostgresToolResultRekeyReader, undefined);
assert.equal(ingress.PostgresToolResultRekeyReader, undefined);
assert.equal(ingress.PostgresToolResultRekeyRepository, undefined);
});
test('Approved Action authority is isolated behind its explicit subpath', () => {
const root = require('@qinglong/cluster-postgres');
const runtime = require('@qinglong/cluster-postgres/runtime');
const admin = require('@qinglong/cluster-postgres/admin');
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
const authority = require('@qinglong/cluster-postgres/approved-action');
assert.equal(root.PostgresApprovalRequestRepository, undefined);
assert.equal(runtime.PostgresApprovalRequestRepository, undefined);
assert.equal(admin.PostgresApprovalRequestRepository, undefined);
assert.equal(ingress.PostgresApprovalRequestRepository, undefined);
assert.equal(typeof authority.PostgresApprovalRequestRepository, 'function');
});
test('Approved Action execution authority is isolated behind its explicit subpath', () => {
const root = require('@qinglong/cluster-postgres');
const runtime = require('@qinglong/cluster-postgres/runtime');
const admin = require('@qinglong/cluster-postgres/admin');
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
const authority = require('@qinglong/cluster-postgres/approved-action-execution');
assert.equal(root.PostgresApprovedActionExecutionRepository, undefined);
assert.equal(runtime.PostgresApprovedActionExecutionRepository, undefined);
assert.equal(admin.PostgresApprovedActionExecutionRepository, undefined);
assert.equal(ingress.PostgresApprovedActionExecutionRepository, undefined);
assert.equal(
typeof authority.PostgresApprovedActionExecutionRepository,
'function',
);
});
test('Plugin Package proposal authority is isolated behind its explicit subpath', () => {
const root = require('@qinglong/cluster-postgres');
const runtime = require('@qinglong/cluster-postgres/runtime');
const admin = require('@qinglong/cluster-postgres/admin');
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
const authority = require('@qinglong/cluster-postgres/plugin-package-proposal');
assert.equal(root.PostgresPluginPackageInstallProposalRepository, undefined);
assert.equal(
runtime.PostgresPluginPackageInstallProposalRepository,
undefined,
);
assert.equal(admin.PostgresPluginPackageInstallProposalRepository, undefined);
assert.equal(
ingress.PostgresPluginPackageInstallProposalRepository,
undefined,
);
assert.equal(
typeof authority.PostgresPluginPackageInstallProposalRepository,
'function',
);
});
test('Plugin Package lifecycle authority is limited to package executor and its explicit subpath', () => {
const root = require('@qinglong/cluster-postgres');
const runtime = require('@qinglong/cluster-postgres/runtime');
const admin = require('@qinglong/cluster-postgres/admin');
const manager = require('@qinglong/cluster-postgres/package-manager');
const executor = require('@qinglong/cluster-postgres/package-executor');
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
const authority = require('@qinglong/cluster-postgres/plugin-package-lifecycle');
assert.equal(root.PostgresPluginPackageLifecycleRepository, undefined);
assert.equal(runtime.PostgresPluginPackageLifecycleRepository, undefined);
assert.equal(admin.PostgresPluginPackageLifecycleRepository, undefined);
assert.equal(manager.PostgresPluginPackageLifecycleRepository, undefined);
assert.equal(ingress.PostgresPluginPackageLifecycleRepository, undefined);
assert.equal(
executor.PostgresPluginPackageLifecycleRepository,
authority.PostgresPluginPackageLifecycleRepository,
);
assert.equal(
typeof authority.PostgresPluginPackageLifecycleRepository,
'function',
);
});
@@ -0,0 +1,176 @@
const assert = require('node:assert/strict');
const path = require('node:path');
const { test } = require('node:test');
const {
loadPostgresCertificateAuthorityFile,
} = require('@qinglong/cluster-postgres/runtime');
const {
PostgresMigrationProcessConfigError,
loadPostgresMigrationProcessConfig,
runPostgresMigrationProcess,
} = require('@qinglong/cluster-postgres/migration-process');
const CA_FILE = path.resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/ca-cert.pem',
);
const CA_BUNDLE = loadPostgresCertificateAuthorityFile(CA_FILE);
const BASE_ENV = Object.freeze({
QL3_POSTGRES_MIGRATION_URL:
'postgresql://ql3_migration:do-not-log@postgres-rw.internal:5432/qinglong',
QL3_POSTGRES_TLS_SERVERNAME: 'postgres-rw.internal',
});
test('builds a TLS-verified single-connection migration configuration', () => {
assert.deepEqual(
loadPostgresMigrationProcessConfig({
...BASE_ENV,
QL3_POSTGRES_TLS_CA_FILE: CA_FILE,
}),
{
connection: {
connectionString: BASE_ENV.QL3_POSTGRES_MIGRATION_URL,
tls: {
mode: 'verify-full',
ca: CA_BUNDLE,
servername: 'postgres-rw.internal',
},
},
pool: {
applicationName: 'qinglong3-cluster-migration',
maxConnections: 1,
connectionTimeoutMs: 15_000,
},
},
);
});
test('loads a discrete operator-managed migration credential', () => {
const config = loadPostgresMigrationProcessConfig({
QL3_POSTGRES_MIGRATION_HOST: 'ql3-postgres-rw.qinglong3-system.svc',
QL3_POSTGRES_MIGRATION_PORT: '5432',
QL3_POSTGRES_MIGRATION_DATABASE: 'qinglong',
QL3_POSTGRES_MIGRATION_USER: 'ql3_migration',
QL3_POSTGRES_MIGRATION_PASSWORD: 'operator-secret',
QL3_POSTGRES_TLS_SERVERNAME: 'ql3-postgres-rw.qinglong3-system.svc',
});
assert.deepEqual(config.connection, {
host: 'ql3-postgres-rw.qinglong3-system.svc',
port: 5432,
database: 'qinglong',
user: 'ql3_migration',
password: 'operator-secret',
tls: {
mode: 'verify-full',
servername: 'ql3-postgres-rw.qinglong3-system.svc',
},
});
});
test('requires an explicit second gate before disabling TLS', () => {
assert.throws(
() =>
loadPostgresMigrationProcessConfig({
...BASE_ENV,
QL3_POSTGRES_TLS_MODE: 'disable',
}),
PostgresMigrationProcessConfigError,
);
assert.deepEqual(
loadPostgresMigrationProcessConfig({
...BASE_ENV,
QL3_POSTGRES_TLS_MODE: 'disable',
QL3_POSTGRES_ALLOW_INSECURE: 'true',
}).connection.tls,
{ mode: 'disable' },
);
});
test('rejects URL TLS overrides and unbounded identity fields', () => {
for (const environment of [
{},
{
...BASE_ENV,
QL3_POSTGRES_MIGRATION_URL: `${BASE_ENV.QL3_POSTGRES_MIGRATION_URL}?sslmode=disable`,
},
{
...BASE_ENV,
QL3_POSTGRES_MIGRATION_HOST: 'postgres-rw.internal',
},
{ ...BASE_ENV, QL3_POSTGRES_TLS_SERVERNAME: undefined },
{ ...BASE_ENV, QL3_POSTGRES_TLS_SERVERNAME: '127.0.0.1' },
{ ...BASE_ENV, QL3_POSTGRES_TLS_SERVERNAME: 'unsafe/name' },
{ ...BASE_ENV, QL3_POSTGRES_TLS_CA_FILE: 'relative-ca.pem' },
{
...BASE_ENV,
QL3_POSTGRES_TLS_MODE: 'disable',
QL3_POSTGRES_ALLOW_INSECURE: 'true',
QL3_POSTGRES_TLS_CA_FILE: CA_FILE,
},
{ ...BASE_ENV, QL3_POSTGRES_APPLICATION_NAME: 'x'.repeat(64) },
]) {
assert.throws(
() => loadPostgresMigrationProcessConfig(environment),
PostgresMigrationProcessConfigError,
);
}
});
test('runs the reviewed stream on one Pool and always closes it', async () => {
const events = [];
const pool = { query() {} };
const result = await runPostgresMigrationProcess({
environment: BASE_ENV,
async openDatabase() {
events.push('open');
return {
pool,
async close() {
events.push('close');
},
};
},
async migrate(options) {
events.push('migrate');
assert.equal(options.pool, pool);
},
emit(record) {
events.push(record.event);
},
});
assert.equal(result, 'migrated');
assert.deepEqual(events, [
'open',
'migration_started',
'migrate',
'migration_completed',
'close',
]);
});
test('preserves migration failure while still closing the Pool', async () => {
const failure = Object.assign(new Error('migration failed'), {
code: 'MIGRATION_FAILED',
});
const events = [];
await assert.rejects(
runPostgresMigrationProcess({
environment: BASE_ENV,
async openDatabase() {
return {
pool: {},
async close() {
events.push('close');
throw new Error('close failed');
},
};
},
async migrate() {
throw failure;
},
}),
(error) => error === failure,
);
assert.deepEqual(events, ['close']);
});
@@ -0,0 +1,89 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
PluginPackageAutomationPublicationUnavailableError,
} = require('@qinglong/runtime-core/plugin-package-automation-publication');
const {
PostgresPluginPackageAutomationPublicationRepository,
} = require('../dist/plugin-package/publication/pluginPackageAutomationPublicationRepository');
const PROJECT_ID = 'automation-project';
const PACKAGE_NAME = 'automation-package';
const PUBLICATION_DIGEST = 'a'.repeat(64);
function pool(query) {
return {
query,
async connect() {
throw new Error('transaction client is not expected');
},
};
}
test('delegates the exact automation start decision to the database guard', async () => {
const queries = [];
const repository =
new PostgresPluginPackageAutomationPublicationRepository(pool(
async (text, values) => {
queries.push({ text, values });
return { rows: [{ allowed: true }] };
},
));
assert.equal(
await repository.isStartAllowed(
PROJECT_ID,
PACKAGE_NAME,
PUBLICATION_DIGEST,
),
true,
);
assert.equal(queries.length, 1);
assert.match(
queries[0].text,
/"ql3"\."plugin_package_automation_start_allowed"/,
);
assert.deepEqual(queries[0].values, [
PROJECT_ID,
PACKAGE_NAME,
PUBLICATION_DIGEST,
]);
});
test('fails closed when the database guard returns a malformed decision', async () => {
const repository =
new PostgresPluginPackageAutomationPublicationRepository(pool(
async () => {
return { rows: [{ allowed: 1 }] };
},
));
await assert.rejects(
repository.isStartAllowed(
PROJECT_ID,
PACKAGE_NAME,
PUBLICATION_DIGEST,
),
PluginPackageAutomationPublicationUnavailableError,
);
});
test('excludes quarantined and publisher-revoked generations from recovery', async () => {
let recoverySql = '';
const repository =
new PostgresPluginPackageAutomationPublicationRepository(pool(
async (text) => {
recoverySql = text;
return { rows: [] };
},
));
assert.deepEqual(await repository.listPendingPage({ limit: 1 }), {
candidates: [],
truncated: false,
});
assert.match(recoverySql, /plugin_package_quarantine_events/);
assert.match(recoverySql, /plugin_package_publisher_provenance/);
assert.match(recoverySql, /plugin_package_publisher_revocation_receipts/);
});
@@ -0,0 +1,184 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
PostgresPluginPackageIdentityKeysetLedgerConflictError,
PostgresPluginPackageIdentityKeysetLedgerRepository,
PostgresPluginPackageIdentityKeysetLedgerUnavailableError,
} = require('../dist/entrypoints/packageManager');
function snapshot(generation, overrides = {}) {
return {
schemaVersion: 1,
generation,
digest: String.fromCharCode(64 + generation).repeat(43),
issuer: 'https://identity.example.test/',
audience: 'qinglong3-package-management',
activeKeyIds: [`issuer-key-${generation}`],
revokedKeyIds:
generation === 1
? []
: Array.from(
{ length: generation - 1 },
(_, index) => `issuer-key-${index + 1}`,
),
...overrides,
};
}
function fixture(authority = 'plugin-package-management') {
let state;
let loseCommitResponse = false;
const queries = [];
let releases = 0;
const client = {
async query(text, values = []) {
queries.push({ text, values });
if (text.startsWith('INSERT')) {
state ??= {
generation: values[1],
digest: values[2],
issuer: values[3],
audience: values[4],
activeKeyIds: JSON.parse(values[5]),
revokedKeyIds: JSON.parse(values[6]),
};
} else if (text.startsWith('SELECT')) {
return { rows: state ? [{ ...state }] : [] };
} else if (text.startsWith('UPDATE')) {
state = {
...state,
generation: values[1],
digest: values[2],
activeKeyIds: JSON.parse(values[3]),
revokedKeyIds: JSON.parse(values[4]),
};
} else if (text === 'COMMIT' && loseCommitResponse) {
loseCommitResponse = false;
throw new Error('response lost after commit');
}
return { rows: [] };
},
release() {
releases += 1;
},
};
return {
repository: new PostgresPluginPackageIdentityKeysetLedgerRepository({
async connect() {
return client;
},
async query() {
throw new Error('pool query must not bypass the transaction client');
},
}, authority),
queries,
state: () => state,
releases: () => releases,
loseNextCommitResponse() {
loseCommitResponse = true;
},
};
}
test('serializes first observation, exact replay and append-only rotation', async () => {
const value = fixture();
await value.repository.observe(snapshot(1));
const writesAfterFirst = value.queries.filter(({ text }) =>
text.startsWith('UPDATE'),
).length;
await value.repository.observe(snapshot(1));
assert.equal(
value.queries.filter(({ text }) => text.startsWith('UPDATE')).length,
writesAfterFirst,
);
await value.repository.observe(snapshot(2));
assert.deepEqual(value.state(), {
generation: 2,
digest: 'B'.repeat(43),
issuer: 'https://identity.example.test/',
audience: 'qinglong3-package-management',
activeKeyIds: ['issuer-key-2'],
revokedKeyIds: ['issuer-key-1'],
});
assert.match(
value.queries.find(({ text }) => text.startsWith('INSERT')).text,
/clock_timestamp\(\)/,
);
assert.equal(value.releases(), 3);
});
test('isolates Plugin, Worker, automation and Approval generations by authority key', async () => {
const value = fixture('worker-credential-management');
await value.repository.observe(
snapshot(1, { audience: 'qinglong3-worker-credential-management' }),
);
const insert = value.queries.find(({ text }) => text.startsWith('INSERT'));
assert.equal(insert.values[0], 'worker-credential-management');
const automation = fixture('automation-management');
await automation.repository.observe(
snapshot(1, { audience: 'qinglong3-automation-management' }),
);
assert.equal(
automation.queries.find(({ text }) => text.startsWith('INSERT')).values[0],
'automation-management',
);
const approval = fixture('approval-management');
await approval.repository.observe(
snapshot(1, { audience: 'qinglong3-approval-management' }),
);
assert.equal(
approval.queries.find(({ text }) => text.startsWith('INSERT')).values[0],
'approval-management',
);
assert.throws(
() => fixture('worker-credential-executor'),
TypeError,
);
});
test('rejects rollback, trust-domain rewrite and implicit key removal', async () => {
const value = fixture();
await value.repository.observe(snapshot(2));
await assert.rejects(
value.repository.observe(snapshot(1)),
PostgresPluginPackageIdentityKeysetLedgerConflictError,
);
await assert.rejects(
value.repository.observe(
snapshot(3, { issuer: 'https://other.example.test/' }),
),
PostgresPluginPackageIdentityKeysetLedgerConflictError,
);
await assert.rejects(
value.repository.observe(
snapshot(3, {
activeKeyIds: ['issuer-key-3'],
revokedKeyIds: [],
}),
),
PostgresPluginPackageIdentityKeysetLedgerConflictError,
);
});
test('converges an ambiguous commit and validates bounded snapshots', async () => {
const value = fixture();
value.loseNextCommitResponse();
await assert.rejects(
value.repository.observe(snapshot(1)),
PostgresPluginPackageIdentityKeysetLedgerUnavailableError,
);
await value.repository.observe(snapshot(1));
assert.equal(value.state().generation, 1);
await assert.rejects(
value.repository.observe(
snapshot(2, { activeKeyIds: ['issuer-key-2', 'issuer-key-2'] }),
),
TypeError,
);
assert.throws(
() =>
new PostgresPluginPackageIdentityKeysetLedgerRepository({ query() {} }),
TypeError,
);
});
@@ -0,0 +1,131 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
PostgresPluginPackageManagementQuotaRepository,
} = require('../dist/entrypoints/packageManager');
const {
PluginPackageManagementQuotaExceededError,
PluginPackageManagementUnavailableError,
} = require('@qinglong/runtime-core/plugin-package-management');
function command(overrides = {}) {
return {
projectId: 'default',
subject: { type: 'user', id: 'cluster-reviewer' },
operation: 'plugin-package.inspect',
idempotencyKey: 'inspection-1',
...overrides,
};
}
test('uses one database-clock UPSERT with a bounded in-row replay ledger', async () => {
const queries = [];
const pool = {
async query(text, values) {
queries.push({ text, values });
return {
rows: [
{
admitted: true,
consumedCount: '1',
resetAtMs: '60000',
observedAtMs: '1',
},
],
};
},
};
const repository = new PostgresPluginPackageManagementQuotaRepository(pool, {
windowMs: 60_000,
limits: { 'plugin-package.inspect': 2 },
});
assert.deepEqual(await repository.consume(command()), {
remaining: 1,
resetAtMs: 60_000,
observedAtMs: 1,
});
assert.equal(queries.length, 1);
assert.match(queries[0].text, /clock_timestamp\(\)/);
assert.match(queries[0].text, /ON CONFLICT/);
assert.match(queries[0].text, /receipt_ids \? \$5::text/);
assert.match(queries[0].text, /jsonb_build_array\(\$5::text\)/);
assert.doesNotMatch(queries[0].text, /\bBEGIN\b|\bCOMMIT\b/);
assert.deepEqual(queries[0].values, [
'default',
'user',
'cluster-reviewer',
'plugin-package.inspect',
'inspection-1',
60_000,
2,
]);
});
test('preserves one quota unit across exact retries and exposes reset delay', async () => {
let calls = 0;
const repository = new PostgresPluginPackageManagementQuotaRepository(
{
async query() {
calls += 1;
return {
rows: [
calls <= 2
? {
admitted: true,
consumedCount: '1',
resetAtMs: '60000',
observedAtMs: String(calls),
}
: {
admitted: false,
consumedCount: '1',
resetAtMs: '60000',
observedAtMs: '1000',
},
],
};
},
},
{
limits: { 'plugin-package.inspect': 1 },
},
);
assert.equal((await repository.consume(command())).remaining, 0);
assert.equal((await repository.consume(command())).remaining, 0);
await assert.rejects(
repository.consume(command({ idempotencyKey: 'inspection-2' })),
(error) =>
error instanceof PluginPackageManagementQuotaExceededError &&
error.retryAfterMs === 59_000,
);
});
test('rejects widened commands and maps database failures to unavailable', async () => {
const repository = new PostgresPluginPackageManagementQuotaRepository({
async query() {
throw new Error('secret database diagnostic');
},
});
await assert.rejects(repository.consume(command({ extra: true })), TypeError);
await assert.rejects(
repository.consume(
command({ subject: { type: 'api_app', id: 'cluster-automation' } }),
),
TypeError,
);
await assert.rejects(
repository.consume(command()),
PluginPackageManagementUnavailableError,
);
assert.throws(
() =>
new PostgresPluginPackageManagementQuotaRepository(
{ query() {} },
{ limits: { 'plugin-package.inspect': 1_001 } },
),
TypeError,
);
});
@@ -0,0 +1,116 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
registerPluginPackageMaterializedRevisionRepositoryContract,
} = require('../../../test/contracts/pluginPackageMaterializedRevisionRepositoryContract.cjs');
const {
PluginPackageResourceMaterializationConflictError,
PluginPackageResourceMaterializationUnavailableError,
} = require('@qinglong/runtime-core/plugin-package-resource-materialization');
const {
PostgresPluginPackageMaterializedRevisionRepository,
} = require('../dist/plugin-package/installation/pluginPackageMaterializedRevisionRepository');
function fakePool() {
let row;
const queries = [];
return {
queries,
pool: {
async query(text, values = []) {
queries.push({ text, values });
if (text.startsWith('INSERT')) {
if (row) return { rows: [] };
row = {
generationDigest: values[0],
projectId: values[1],
packageName: values[2],
generation: values[3],
lockDigest: values[4],
manifestDigest: values[5],
revisionDigest: values[6],
revisionJson: JSON.parse(values[7]),
createdAtMs: 300,
};
return { rows: [{ generation_digest: values[0] }] };
}
if (text.startsWith('SELECT')) {
return { rows: row ? [{ ...row }] : [] };
}
throw new Error(`unexpected SQL: ${text}`);
},
},
corrupt() {
row.revisionJson.resources[0].value.name = 'Changed';
},
};
}
registerPluginPackageMaterializedRevisionRepositoryContract({
name: 'PostgreSQL Plugin Package materialized revision repository',
namespace: 'postgres-materialized',
profile: 'cluster-control',
async createRepository(_t, fixture) {
const value = fakePool();
return {
repository: new PostgresPluginPackageMaterializedRevisionRepository(
value.pool,
fixture.registry,
),
};
},
});
test('uses database time and fails closed on corrupted JSON', async () => {
const {
materializedRevisionFixture,
} = require('../../../test/contracts/pluginPackageMaterializedRevisionRepositoryContract.cjs');
const fixture = materializedRevisionFixture(
'postgres-corrupt',
'cluster-control',
);
const value = fakePool();
const repository = new PostgresPluginPackageMaterializedRevisionRepository(
value.pool,
fixture.registry,
);
await repository.publish(fixture.revision);
assert.match(
value.queries.find(({ text }) => text.startsWith('INSERT')).text,
/clock_timestamp\(\)/,
);
value.corrupt();
await assert.rejects(
repository.find(fixture.revision.generation.generationDigest),
PluginPackageResourceMaterializationUnavailableError,
);
});
test('maps PostgreSQL uniqueness rejection to a semantic conflict', async () => {
const repository =
new PostgresPluginPackageMaterializedRevisionRepository({
async query() {
const error = new Error('duplicate');
error.code = '23505';
throw error;
},
});
await assert.rejects(
repository.find('a'.repeat(64)),
PluginPackageResourceMaterializationConflictError,
);
});
test('publishes storage through package-executor and the explicit subpath', () => {
assert.equal(
require('@qinglong/cluster-postgres/plugin-package-materialized-revision')
.PostgresPluginPackageMaterializedRevisionRepository,
PostgresPluginPackageMaterializedRevisionRepository,
);
assert.equal(
require('../dist/entrypoints/packageExecutor')
.PostgresPluginPackageMaterializedRevisionRepository,
PostgresPluginPackageMaterializedRevisionRepository,
);
});
@@ -0,0 +1,204 @@
const assert = require('node:assert/strict');
const { generateKeyPairSync } = require('node:crypto');
const { test } = require('node:test');
const {
createPluginPackagePublisherRevocationProposal,
PluginPackagePublisherRevocationProposalConflictError,
} = require('@qinglong/runtime-core/plugin-package-publisher-revocation-proposal');
const {
createPluginPackagePublisherTrustSnapshot,
} = require('@qinglong/runtime-core/plugin-package-publisher-trust');
const {
PostgresPluginPackagePublisherRevocationProposalRepository,
} = require('../dist/plugin-package/publisher/pluginPackagePublisherRevocationProposalRepository');
const SUBJECT = Object.freeze({ type: 'user', id: 'usr_owner' });
const FENCE = Object.freeze({ projectVersion: 4, bindingVersion: 7 });
function proposal() {
const { publicKey } = generateKeyPairSync('ed25519');
const trustSnapshot = createPluginPackagePublisherTrustSnapshot([
{
publisher: 'publisher-a.example',
keyId: 'key-a',
publicKeyPem: publicKey.export({ type: 'spki', format: 'pem' }),
notBeforeMs: 1,
notAfterMs: 10_000,
},
]);
return createPluginPackagePublisherRevocationProposal({
actionRef: 'publisher-revoke:publisher-a.example:key-a',
authorityProjectId: 'cluster-trust-authority',
trustAuthorityId: 'cluster',
trustGeneration: 1,
trustSnapshot,
publisher: 'publisher-a.example',
keyId: 'key-a',
authorizationMode: 'dual_control',
reasonCode: 'suspected_key_compromise',
proposedBy: SUBJECT,
proposerAssurance: 'multi_factor',
proposalFence: FENCE,
createdAtMs: 100,
});
}
function audit(candidate, overrides = {}) {
return {
eventId: '32000000-0000-4000-8000-000000000001',
requestId: candidate.actionRef,
operationId: 'plugin_package.publisher_revocation.propose',
projectId: candidate.projectId,
subject: candidate.proposedBy,
authenticationId: 'auth-owner',
outcome: 'allowed',
reasons: ['publisher_revocation_proposal'],
fence: candidate.proposalFence,
occurredAtMs: candidate.createdAtMs,
...overrides,
};
}
function fixture(candidate, trustOverrides = {}) {
let storedProposal = null;
let storedAudit = null;
let signerLocks = 0;
const query = async (text, values = []) => {
if (text.includes('pg_advisory_xact_lock(hashtextextended')) {
signerLocks += 1;
assert.deepEqual(values, [
JSON.stringify([
candidate.actionInput.publisher,
candidate.actionInput.keyId,
]),
774635229,
]);
return { rows: [{}], rowCount: 1 };
}
if (
text.includes(
'FROM "ql3"."plugin_package_publisher_revocation_proposals"',
)
) {
return {
rows: storedProposal
? [
{
proposalJson: storedProposal,
proposalDigest: storedProposal.proposalDigest,
},
]
: [],
};
}
if (
text.includes(
'FROM "ql3"."plugin_package_publisher_trust_heads"',
)
) {
return {
rows: [
{
generation:
trustOverrides.generation ??
candidate.actionInput.trustGeneration,
effectiveTrustDigest:
trustOverrides.effectiveTrustDigest ??
candidate.actionInput.previousTrustDigest,
},
],
};
}
if (text.includes('lock_approval_policy_fence')) {
return { rows: [{ matches: true }] };
}
if (
text.includes(
'INSERT INTO\n "ql3"."plugin_package_publisher_revocation_proposals"',
)
) {
storedProposal = JSON.parse(values[20]);
return { rows: [], rowCount: 1 };
}
if (text.includes('INSERT INTO "ql3"."security_audit_events"')) {
storedAudit = audit(candidate);
return { rows: [], rowCount: 1 };
}
if (text.includes('FROM "ql3"."security_audit_events"')) {
return {
rows: storedAudit
? [
{
eventId: storedAudit.eventId,
requestId: storedAudit.requestId,
operationId: storedAudit.operationId,
projectId: storedAudit.projectId,
subjectType: storedAudit.subject.type,
subjectId: storedAudit.subject.id,
authenticationId: storedAudit.authenticationId,
outcome: storedAudit.outcome,
reasons: storedAudit.reasons,
projectVersion: storedAudit.fence.projectVersion,
bindingVersion: storedAudit.fence.bindingVersion,
occurredAtMs: storedAudit.occurredAtMs,
},
]
: [],
};
}
return { rows: [], rowCount: 0 };
};
const client = { query, release() {} };
const repository =
new PostgresPluginPackagePublisherRevocationProposalRepository({
query,
async connect() {
return client;
},
});
repository.signerLocks = () => signerLocks;
return repository;
}
test('persists and exactly replays a generation-fenced revocation proposal', async () => {
const candidate = proposal();
const repository = fixture(candidate);
const created = await repository.createProposal({
proposal: candidate,
audit: audit(candidate),
});
assert.equal(created.status, 'created');
assert.equal(repository.signerLocks(), 1);
const replay = await repository.createProposal({
proposal: candidate,
audit: audit(candidate),
});
assert.equal(replay.status, 'existing');
assert.equal(repository.signerLocks(), 1);
assert.deepEqual(
await repository.findProposalByActionRef(candidate.actionRef),
candidate,
);
});
test('rejects stale trust generations and mismatched audit authority', async () => {
const candidate = proposal();
await assert.rejects(
fixture(candidate, { generation: 2 }).createProposal({
proposal: candidate,
audit: audit(candidate),
}),
PluginPackagePublisherRevocationProposalConflictError,
);
assert.throws(
() =>
fixture(candidate).createProposal({
proposal: candidate,
audit: audit(candidate, {
reasons: ['client_supplied_transition'],
}),
}),
PluginPackagePublisherRevocationProposalConflictError,
);
});
@@ -0,0 +1,146 @@
const assert = require('node:assert/strict');
const { generateKeyPairSync } = require('node:crypto');
const { test } = require('node:test');
const {
createPluginPackagePublisherTrustSnapshot,
} = require('@qinglong/runtime-core/plugin-package-publisher-trust');
const {
PostgresPluginPackagePublisherTrustAuthorityRepository,
} = require('../dist/plugin-package/publisher/pluginPackagePublisherTrustAuthorityRepository');
function snapshot(publisher, keyId) {
const { publicKey } = generateKeyPairSync('ed25519');
return createPluginPackagePublisherTrustSnapshot([
{
publisher,
keyId,
publicKeyPem: publicKey.export({ type: 'spki', format: 'pem' }),
notBeforeMs: 1,
notAfterMs: 10_000,
},
]);
}
function fixture() {
const snapshots = new Map();
let head = null;
const queries = [];
const query = async (text, values = []) => {
queries.push({ text, values });
if (
text.includes(
'INSERT INTO "ql3"."plugin_package_publisher_trust_snapshots"',
)
) {
snapshots.set(values[0], JSON.parse(values[4]));
return { rows: [], rowCount: 1 };
}
if (
text.includes(
'FROM "ql3"."plugin_package_publisher_trust_heads" AS head',
)
) {
if (!head || head.authorityId !== values[0]) return { rows: [] };
const effective = snapshots.get(head.effectiveTrustDigest);
return {
rows: [
{
headJson: head,
headDigest: head.headDigest,
snapshotJson: effective,
snapshotDigest: effective.snapshotDigest,
},
],
};
}
if (
text.includes(
'INSERT INTO "ql3"."plugin_package_publisher_trust_heads"',
)
) {
if (head && head.authorityId === values[0]) {
return { rows: [], rowCount: 0 };
}
head = JSON.parse(values[6]);
return { rows: [], rowCount: 1 };
}
return { rows: [], rowCount: 0 };
};
const client = {
query,
release() {},
};
const repository =
new PostgresPluginPackagePublisherTrustAuthorityRepository({
query,
async connect() {
return client;
},
});
return { repository, queries, head: () => head };
}
test('observes one base snapshot and returns the durable effective head', async () => {
const value = fixture();
const base = snapshot('publisher-a.example', 'key-a');
const created = await value.repository.observeSnapshot({
authorityId: 'cluster',
snapshot: base,
observedBy: 'package-manager-1',
observedAtMs: 100,
});
assert.equal(created.status, 'created');
assert.equal(created.head.generation, 1);
assert.equal(created.head.baseSnapshotDigest, base.snapshotDigest);
assert.equal(
created.head.effectiveTrustDigest,
base.snapshotDigest,
);
const replay = await value.repository.observeSnapshot({
authorityId: 'cluster',
snapshot: base,
observedBy: 'package-manager-2',
observedAtMs: 200,
});
assert.equal(replay.status, 'existing');
assert.deepEqual(
await value.repository.findAuthority('cluster'),
{
head: value.head(),
effectiveSnapshot: base,
},
);
assert.equal(
value.queries.filter(({ text }) =>
text.includes(
'INSERT INTO "ql3"."plugin_package_publisher_trust_heads"',
),
).length,
2,
);
});
test('observes changed material as a candidate without advancing the head', async () => {
const value = fixture();
const base = snapshot('publisher-a.example', 'key-a');
await value.repository.observeSnapshot({
authorityId: 'cluster',
snapshot: base,
observedBy: 'package-manager-1',
observedAtMs: 100,
});
const candidate = snapshot('publisher-b.example', 'key-b');
const observed = await value.repository.observeSnapshot({
authorityId: 'cluster',
snapshot: candidate,
observedBy: 'package-manager-1',
observedAtMs: 200,
});
assert.equal(observed.status, 'candidate');
assert.equal(observed.head.generation, 1);
assert.equal(observed.head.baseSnapshotDigest, base.snapshotDigest);
assert.equal(observed.head.effectiveTrustDigest, base.snapshotDigest);
assert.deepEqual(observed.effectiveSnapshot, base);
});
@@ -0,0 +1,330 @@
const assert = require('node:assert/strict');
const { generateKeyPairSync } = require('node:crypto');
const { test } = require('node:test');
const {
approvalRequestDigest,
createApprovalRequest,
decideApprovalRequest,
} = require('@qinglong/runtime-core/approved-action');
const {
createPluginPackagePublisherTrustSnapshot,
} = require('@qinglong/runtime-core/plugin-package-publisher-trust');
const {
createPluginPackagePublisherTrustTransitionProposal,
PluginPackagePublisherTrustTransitionConflictError,
} = require('@qinglong/runtime-core/plugin-package-publisher-trust-transition-proposal');
const {
PostgresPluginPackagePublisherTrustTransitionProposalRepository,
} = require('../dist/plugin-package/publisher/pluginPackagePublisherTrustTransitionProposalRepository');
const SUBJECT = Object.freeze({ type: 'user', id: 'usr_owner' });
const FENCE = Object.freeze({ projectVersion: 4, bindingVersion: 7 });
function transition() {
const oldPair = generateKeyPairSync('ed25519');
const newPair = generateKeyPairSync('ed25519');
const oldDefinition = {
publisher: 'publisher-a.example',
keyId: 'key-old',
publicKeyPem: oldPair.publicKey.export({
type: 'spki',
format: 'pem',
}),
notBeforeMs: 1,
notAfterMs: 10_000,
};
const newDefinition = {
publisher: 'publisher-a.example',
keyId: 'key-new',
publicKeyPem: newPair.publicKey.export({
type: 'spki',
format: 'pem',
}),
notBeforeMs: 1,
notAfterMs: 20_000,
};
const currentSnapshot =
createPluginPackagePublisherTrustSnapshot([oldDefinition]);
const materialSnapshot =
createPluginPackagePublisherTrustSnapshot([
oldDefinition,
newDefinition,
]);
const created =
createPluginPackagePublisherTrustTransitionProposal({
actionRef: 'publisher-overlap:publisher-a.example:key-new',
authorityProjectId: 'cluster-trust-authority',
trustAuthorityId: 'cluster',
trustGeneration: 1,
mode: 'overlap_add',
trustSnapshot: currentSnapshot,
materialSnapshot,
publisher: 'publisher-a.example',
keyId: 'key-new',
proposedBy: SUBJECT,
proposerAssurance: 'multi_factor',
proposalFence: FENCE,
createdAtMs: 100,
});
return {
...created,
currentSnapshot,
};
}
function audit(proposal, overrides = {}) {
return {
eventId: '33000000-0000-4000-8000-000000000001',
requestId: proposal.actionRef,
operationId: 'plugin_package.publisher_trust_transition.propose',
projectId: proposal.projectId,
subject: proposal.proposedBy,
authenticationId: 'auth-owner',
outcome: 'allowed',
reasons: ['publisher_trust_transition_proposal'],
fence: proposal.proposalFence,
occurredAtMs: proposal.createdAtMs,
...overrides,
};
}
function fixture(value, trustOverrides = {}) {
const snapshots = new Map([
[value.currentSnapshot.snapshotDigest, value.currentSnapshot],
]);
let storedProposal = null;
let storedAudit = null;
let signerLocks = 0;
const query = async (text, values = []) => {
if (text.includes('pg_advisory_xact_lock(hashtextextended')) {
signerLocks += 1;
assert.deepEqual(values, [
JSON.stringify([
value.proposal.actionInput.publisher,
value.proposal.actionInput.keyId,
]),
774635229,
]);
return { rows: [{}], rowCount: 1 };
}
if (
text.includes(
'FROM "ql3"."plugin_package_publisher_trust_transition_proposals"',
)
) {
return {
rows: storedProposal
? [
{
proposalJson: storedProposal,
proposalDigest: storedProposal.proposalDigest,
},
]
: [],
};
}
if (
text.includes(
'FROM "ql3"."plugin_package_publisher_trust_heads" AS head',
)
) {
return {
rows: [
{
generation:
trustOverrides.generation ??
value.proposal.actionInput.trustGeneration,
effectiveTrustDigest:
trustOverrides.effectiveTrustDigest ??
value.proposal.actionInput.previousTrustDigest,
snapshotJson: value.currentSnapshot,
snapshotDigest: value.currentSnapshot.snapshotDigest,
},
],
};
}
if (
text.includes(
'INSERT INTO "ql3"."plugin_package_publisher_trust_snapshots"',
)
) {
const existed = snapshots.has(values[0]);
if (!existed) snapshots.set(values[0], JSON.parse(values[4]));
return { rows: [], rowCount: existed ? 0 : 1 };
}
if (
text.includes(
'FROM "ql3"."plugin_package_publisher_trust_snapshots"',
)
) {
const snapshot = snapshots.get(values[0]);
return {
rows: snapshot
? [
{
snapshotJson: snapshot,
snapshotDigest: snapshot.snapshotDigest,
},
]
: [],
};
}
if (text.includes('lock_approval_policy_fence')) {
return { rows: [{ matches: true }] };
}
if (
text.includes(
'"ql3"."plugin_package_publisher_trust_transition_proposals" (',
)
) {
storedProposal = JSON.parse(values[19]);
return { rows: [], rowCount: 1 };
}
if (text.includes('INSERT INTO "ql3"."security_audit_events"')) {
storedAudit = audit(value.proposal);
return { rows: [], rowCount: 1 };
}
if (text.includes('FROM "ql3"."security_audit_events"')) {
return {
rows: storedAudit
? [
{
eventId: storedAudit.eventId,
requestId: storedAudit.requestId,
operationId: storedAudit.operationId,
projectId: storedAudit.projectId,
subjectType: storedAudit.subject.type,
subjectId: storedAudit.subject.id,
authenticationId: storedAudit.authenticationId,
outcome: storedAudit.outcome,
reasons: storedAudit.reasons,
projectVersion: storedAudit.fence.projectVersion,
bindingVersion: storedAudit.fence.bindingVersion,
occurredAtMs: storedAudit.occurredAtMs,
},
]
: [],
};
}
return { rows: [], rowCount: 0 };
};
const client = { query, release() {} };
const repository =
new PostgresPluginPackagePublisherTrustTransitionProposalRepository({
query,
async connect() {
return client;
},
});
repository.signerLocks = () => signerLocks;
repository.snapshots = () => snapshots;
return repository;
}
test('persists candidate material and exactly replays an overlap proposal', async () => {
const value = transition();
const repository = fixture(value);
const command = {
proposal: value.proposal,
candidateSnapshot: value.candidateSnapshot,
audit: audit(value.proposal),
};
const created = await repository.createProposal(command);
assert.equal(created.status, 'created');
assert.equal(repository.signerLocks(), 1);
assert.deepEqual(
repository.snapshots().get(value.candidateSnapshot.snapshotDigest),
value.candidateSnapshot,
);
const replay = await repository.createProposal(command);
assert.equal(replay.status, 'existing');
assert.equal(repository.signerLocks(), 1);
assert.deepEqual(
await repository.findProposalByActionRef(value.proposal.actionRef),
value.proposal,
);
});
test('rejects stale trust generations and client-shaped candidate drift', async () => {
const value = transition();
await assert.rejects(
fixture(value, { generation: 2 }).createProposal({
proposal: value.proposal,
candidateSnapshot: value.candidateSnapshot,
audit: audit(value.proposal),
}),
PluginPackagePublisherTrustTransitionConflictError,
);
assert.throws(
() =>
fixture(value).createProposal({
proposal: value.proposal,
candidateSnapshot: value.currentSnapshot,
audit: audit(value.proposal),
}),
PluginPackagePublisherTrustTransitionConflictError,
);
});
test('lists only separation-of-duty approvals from the durable JSON record', async () => {
const value = transition();
const pending = createApprovalRequest({
id: 'approval-trust-transition-list',
projectId: value.proposal.projectId,
action: {
permission: value.proposal.permission,
actionType: value.proposal.actionType,
actionRef: value.proposal.actionRef,
actionDigest: value.proposal.actionDigest,
previewDigest: value.proposal.previewDigest,
},
risk: 'critical',
decisionMode: 'separation_of_duty',
requestedBy: SUBJECT,
requestedAtMs: 100,
expiresAtMs: 10_000,
requestFence: FENCE,
});
const approved = decideApprovalRequest(pending, {
expectedVersion: 1,
decisionId: 'decision-trust-transition-list',
decision: 'approved',
reasonCode: 'reviewed',
principal: {
subject: { type: 'user', id: 'usr_reviewer' },
authenticationId: 'auth-reviewer',
authenticatedAtMs: 100,
expiresAtMs: 10_000,
assurance: 'multi_factor',
},
decidedAtMs: 101,
authorizationFence: FENCE,
});
let inspectedSql = '';
const repository =
new PostgresPluginPackagePublisherTrustTransitionProposalRepository({
async query(text) {
inspectedSql = text;
return {
rows: [
{
requestJson: approved,
requestDigest: approvalRequestDigest(approved),
},
],
rowCount: 1,
};
},
async connect() {
throw new Error('listApprovedRequests must not open a transaction');
},
});
assert.deepEqual(await repository.listApprovedRequests(4), [approved]);
assert.match(
inspectedSql,
/request\.request_json ->> 'decisionMode'/,
);
assert.doesNotMatch(inspectedSql, /request\.decision_mode/);
});
@@ -0,0 +1,292 @@
const assert = require('node:assert/strict');
const { generateKeyPairSync } = require('node:crypto');
const { test } = require('node:test');
const {
approvalRequestDigest,
approvedActionDispatchDigest,
consumeApprovalRequest,
createApprovalRequest,
decideApprovalRequest,
} = require('@qinglong/runtime-core/approved-action');
const {
createPluginPackagePublisherTrustHead,
createPluginPackagePublisherTrustSnapshot,
} = require('@qinglong/runtime-core/plugin-package-publisher-trust');
const {
createPluginPackagePublisherTrustTransitionProposal,
PluginPackagePublisherTrustTransitionConflictError,
} = require('@qinglong/runtime-core/plugin-package-publisher-trust-transition-proposal');
const {
PostgresPluginPackagePublisherTrustTransitionRepository,
} = require('../dist/plugin-package/publisher/pluginPackagePublisherTrustTransitionRepository');
const REQUESTER = Object.freeze({ type: 'user', id: 'usr_owner' });
const REVIEWER = Object.freeze({ type: 'user', id: 'usr_security' });
const SYSTEM = Object.freeze({
type: 'system',
id: 'cluster_package_executor',
});
const FENCE = Object.freeze({ projectVersion: 4, bindingVersion: 7 });
function authority(mode = 'overlap_add') {
const oldPair = generateKeyPairSync('ed25519');
const newPair = generateKeyPairSync('ed25519');
const oldDefinition = {
publisher: 'publisher-a.example',
keyId: 'key-old',
publicKeyPem: oldPair.publicKey.export({
type: 'spki',
format: 'pem',
}),
notBeforeMs: 1,
notAfterMs: 10_000,
};
const newDefinition = {
publisher: 'publisher-a.example',
keyId: 'key-new',
publicKeyPem: newPair.publicKey.export({
type: 'spki',
format: 'pem',
}),
notBeforeMs: 1,
notAfterMs: 20_000,
};
const currentSnapshot =
createPluginPackagePublisherTrustSnapshot(
mode === 'overlap_add'
? [oldDefinition]
: [oldDefinition, newDefinition],
);
const created =
createPluginPackagePublisherTrustTransitionProposal({
actionRef:
mode === 'overlap_add'
? 'publisher-overlap:publisher-a.example:key-new'
: 'publisher-retire:publisher-a.example:key-old',
authorityProjectId: 'cluster-trust-authority',
trustAuthorityId: 'cluster',
trustGeneration: 1,
mode,
trustSnapshot: currentSnapshot,
...(mode === 'overlap_add'
? {
materialSnapshot:
createPluginPackagePublisherTrustSnapshot([
oldDefinition,
newDefinition,
]),
}
: {}),
publisher: 'publisher-a.example',
keyId: mode === 'overlap_add' ? 'key-new' : 'key-old',
proposedBy: REQUESTER,
proposerAssurance: 'multi_factor',
proposalFence: FENCE,
createdAtMs: 100,
});
const action = {
permission: created.proposal.permission,
actionType: created.proposal.actionType,
actionRef: created.proposal.actionRef,
actionDigest: created.proposal.actionDigest,
previewDigest: created.proposal.previewDigest,
};
const pending = createApprovalRequest({
id: `approval-${mode}`,
projectId: created.proposal.projectId,
action,
risk: 'critical',
decisionMode: 'separation_of_duty',
requestedBy: REQUESTER,
requestedAtMs: 100,
expiresAtMs: 1_000,
requestFence: FENCE,
});
const approved = decideApprovalRequest(pending, {
expectedVersion: 1,
decisionId: `decision-${mode}`,
decision: 'approved',
reasonCode: 'reviewed',
principal: {
subject: REVIEWER,
authenticationId: 'auth-reviewer',
authenticatedAtMs: 101,
expiresAtMs: 900,
assurance: 'multi_factor',
},
decidedAtMs: 110,
authorizationFence: FENCE,
});
const consumed = consumeApprovalRequest(approved, {
expectedVersion: 2,
consumptionId: `consume-${mode}`,
dispatchId: `dispatch-${mode}`,
action,
requestedBy: REQUESTER,
consumedBy: SYSTEM,
consumedAtMs: 120,
authorizationFence: FENCE,
});
return {
...created,
currentSnapshot,
approval: consumed.request,
dispatch: consumed.dispatch,
initialHead: createPluginPackagePublisherTrustHead(
'cluster',
currentSnapshot,
50,
),
};
}
function fixture(value, options = {}) {
let head = value.initialHead;
let receipt = null;
let signerLocks = 0;
const query = async (text, values = []) => {
if (
text.includes(
'FROM "ql3"."approved_action_dispatches" AS dispatch',
)
) {
return {
rows: [
{
proposalJson: value.proposal,
proposalDigest: value.proposal.proposalDigest,
dispatchJson: value.dispatch,
dispatchDigest: approvedActionDispatchDigest(value.dispatch),
approvalJson: value.approval,
approvalDigest: approvalRequestDigest(value.approval),
},
],
};
}
if (text.includes('pg_advisory_xact_lock(hashtextextended')) {
signerLocks += 1;
return { rows: [{}], rowCount: 1 };
}
if (
text.includes(
'FROM "ql3"."plugin_package_publisher_trust_transition_receipts"',
)
) {
return {
rows: receipt
? [
{
receiptJson: receipt,
receiptDigest: receipt.receiptDigest,
},
]
: [],
};
}
if (
text.includes(
'FROM "ql3"."plugin_package_publisher_trust_heads" AS head',
)
) {
return {
rows: [
{
headJson: head,
headDigest: head.headDigest,
effectiveSnapshotJson: value.currentSnapshot,
effectiveSnapshotDigest:
value.currentSnapshot.snapshotDigest,
candidateSnapshotJson: value.candidateSnapshot,
candidateSnapshotDigest:
value.candidateSnapshot.snapshotDigest,
},
],
};
}
if (
text.includes(
'FROM "ql3"."plugin_package_publisher_trust_heads"',
)
) {
return {
rows: [{ headJson: head, headDigest: head.headDigest }],
};
}
if (
text.includes(
'FROM "ql3"."plugin_package_publisher_provenance" AS provenance',
)
) {
return {
rows: options.matchingInstall
? [{ installationId: 'install-old' }]
: [],
};
}
if (
text.includes(
'UPDATE "ql3"."plugin_package_publisher_trust_heads"',
)
) {
head = JSON.parse(values[5]);
return { rows: [], rowCount: 1 };
}
if (
text.includes(
'"ql3"."plugin_package_publisher_trust_transition_receipts" (',
)
) {
receipt = JSON.parse(values[16]);
return { rows: [], rowCount: 1 };
}
return { rows: [], rowCount: 0 };
};
const client = { query, release() {} };
const repository =
new PostgresPluginPackagePublisherTrustTransitionRepository({
query,
async connect() {
return client;
},
});
repository.signerLocks = () => signerLocks;
repository.head = () => head;
return repository;
}
test('atomically advances overlap trust and exactly replays its receipt', async () => {
const value = authority();
const repository = fixture(value);
const created = await repository.applyApprovedTransition({
dispatch: value.dispatch,
executedAtMs: 130,
});
assert.equal(created.status, 'created');
assert.equal(created.receipt.mode, 'overlap_add');
assert.equal(created.receipt.retirementMatchingInstallations, null);
assert.equal(created.head.generation, 2);
assert.equal(
created.head.effectiveTrustDigest,
value.candidateSnapshot.snapshotDigest,
);
const replay = await repository.applyApprovedTransition({
dispatch: value.dispatch,
executedAtMs: 130,
});
assert.equal(replay.status, 'existing');
assert.equal(replay.receipt.receiptDigest, created.receipt.receiptDigest);
assert.equal(repository.signerLocks(), 2);
});
test('blocks safe retirement while a current installation uses the signer', async () => {
const value = authority('safe_retire');
await assert.rejects(
fixture(value, { matchingInstall: true }).applyApprovedTransition({
dispatch: value.dispatch,
executedAtMs: 130,
}),
PluginPackagePublisherTrustTransitionConflictError,
);
});
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,435 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
createInitialPluginPackageAutomationPublication,
} = require('@qinglong/runtime-core/plugin-package-automation-publication');
const {
createPluginPackageWorkflowAdmissionBundle,
createPluginPackageWorkflowExecutionPlan,
PluginPackageWorkflowAdmissionNotAllowedError,
PluginPackageWorkflowAdmissionUnavailableError,
} = require('@qinglong/runtime-core/plugin-package-workflow-execution-plan');
const {
transitionStepRunMutation,
} = require('@qinglong/runtime-core/step-run');
const {
pluginPackageTaskReconciliationFixture,
} = require('../../../test/contracts/pluginPackageTaskReconciliationRepositoryContract.cjs');
const {
PostgresPluginPackageWorkflowAdmissionRepository,
} = require('../dist/plugin-package/workflow/pluginPackageWorkflowAdmissionRepository');
function fixture(namespace = 'postgres-workflow-admission') {
const value = pluginPackageTaskReconciliationFixture(namespace, {
workflows: [
{
schema: 'qinglong/plugin-package-workflow-resource@v1',
id: 'daily',
name: 'Daily workflow',
enabled: true,
steps: [
{ id: 'collect', task: 'alpha', needs: [] },
{ id: 'summarize', task: 'beta', needs: ['collect'] },
],
},
],
});
const publication = createInitialPluginPackageAutomationPublication(
value.revision,
value.registry,
2_000,
);
const plan = createPluginPackageWorkflowExecutionPlan({
planId: `workflow-plan-${namespace}`,
runId: `run-${namespace}`,
workflowId: 'daily',
stepRunIds: {
collect: `step-collect-${namespace}`,
summarize: `step-summarize-${namespace}`,
},
publication,
revision: value.revision,
taskSpecSemanticRegistry: value.registry,
plannedAtMs: 3_000,
});
return { ...value, publication, plan };
}
function poolWithClient(client) {
return {
async query(text, values) {
return client.query(text, values);
},
async connect() {
return client;
},
};
}
function admissionRow(bundle) {
const { plan, receipt } = bundle;
return {
planDigest: plan.planDigest,
planId: plan.planId,
runId: plan.runId,
projectId: plan.target.projectId,
packageName: plan.target.packageName,
installationId: plan.target.installationId,
lockDigest: plan.target.lockDigest,
generation: plan.target.generation,
generationDigest: plan.target.generationDigest,
materializedRevisionDigest: plan.target.materializedRevisionDigest,
publicationDigest: plan.target.publicationDigest,
workflowId: plan.target.workflowId,
workflowDefinitionDigest: plan.target.workflowDefinitionDigest,
stepCount: plan.steps.length,
admittedAtMs: receipt.admittedAtMs,
finalRunVersion: receipt.finalRunVersion,
finalRunEventSequence: receipt.finalRunEventSequence,
receiptDigest: receipt.receiptDigest,
planJson: plan,
receiptJson: receipt,
};
}
function runRow(run) {
return {
projectId: run.projectId,
taskId: run.taskId,
taskRevision: run.taskRevision,
taskSnapshotRef: run.taskSnapshotRef ?? null,
triggerType: run.triggerType,
executionOrigin: run.executionOrigin,
executionOwner: run.executionOwner,
requestId: run.requestId ?? null,
status: run.status,
version: run.version,
eventSequence: run.eventSequence,
priority: run.priority,
idempotencyKey: run.idempotencyKey ?? null,
createdAtMs: run.createdAtMs,
startedAtMs: run.startedAtMs ?? null,
};
}
function eventRow(event) {
return {
id: event.id,
sequence: event.sequence,
type: event.type,
dedupeKey: event.dedupeKey ?? null,
actorType: event.actorType,
actorId: event.actorId ?? null,
stepRunId: event.stepRunId ?? null,
payload: event.payload,
createdAtMs: event.createdAtMs,
};
}
function stepEvidenceRow(bundle, stepKey, currentStepRun) {
const mutation = bundle.stepMutations.find(
({ stepRun }) => stepRun.stepKey === stepKey,
);
const step = bundle.plan.steps.find(
(candidate) => candidate.stepKey === stepKey,
);
assert.ok(mutation);
assert.ok(step);
return {
stepRunId: mutation.stepRun.id,
taskId: step.taskId,
taskDefinitionRef: step.taskDefinitionRef,
taskDefinitionDigest: step.taskDefinitionDigest,
needsJson: step.needs,
initialStatus: step.initialStatus,
mutationId: mutation.mutationId,
eventId: mutation.event.id,
currentStepKey: (currentStepRun ?? mutation.stepRun).stepKey,
currentKind: (currentStepRun ?? mutation.stepRun).kind,
currentDefinitionRef: (currentStepRun ?? mutation.stepRun).definitionRef,
currentDefinitionDigest:
(currentStepRun ?? mutation.stepRun).definitionDigest,
currentRequired: (currentStepRun ?? mutation.stepRun).required,
currentStatus: (currentStepRun ?? mutation.stepRun).status,
currentVersion: (currentStepRun ?? mutation.stepRun).version,
currentLastMutationId: (currentStepRun ?? mutation.stepRun).lastMutationId,
currentStepRunDigest: (currentStepRun ?? mutation.stepRun).stepRunDigest,
currentStepRunJson: currentStepRun ?? mutation.stepRun,
mutationDigest: mutation.mutationDigest,
eventSequence: mutation.event.sequence,
runVersion: mutation.expectedRunVersion + 1,
initialStepRunDigest: mutation.stepRun.stepRunDigest,
initialStepRunJson: mutation.stepRun,
};
}
test('admits the complete Workflow evidence in one SERIALIZABLE transaction', async () => {
const value = fixture();
const queries = [];
let released = false;
const client = {
async query(text, values) {
queries.push({ text, values });
if (
text.includes('plugin_package_workflow_admissions') &&
text.includes('WHERE plan_id')
) {
return { rows: [] };
}
if (text.includes('plugin_package_workflow_admission_snapshot')) {
return {
rows: [
{
publicationJson: value.publication,
revisionJson: value.revision,
},
],
};
}
return { rows: [] };
},
release() {
released = true;
},
};
const repository = new PostgresPluginPackageWorkflowAdmissionRepository(
poolWithClient(client),
);
const result = await repository.admit(value.plan);
assert.equal(result.status, 'created');
assert.equal(result.receipt.finalRunVersion, 3);
assert.equal(queries[0].text, 'BEGIN ISOLATION LEVEL SERIALIZABLE');
assert.equal(queries.at(-1).text, 'COMMIT');
assert.equal(released, true);
assert.equal(
queries.some(
({ text }) =>
text.includes('plugin_package_workflow_admissions') &&
text.includes('FOR SHARE'),
),
false,
'append-only replay must not require UPDATE authority',
);
assert.equal(
queries.filter(({ text }) => /INSERT INTO "ql3"\."runs"/.test(text)).length,
1,
);
assert.equal(
queries.filter(({ text }) => /INSERT INTO "ql3"\."step_runs"/.test(text))
.length,
2,
);
assert.equal(
queries.filter(({ text }) => /INSERT INTO "ql3"\."run_events"/.test(text))
.length,
3,
);
assert.equal(
queries.filter(({ text }) =>
/INSERT INTO "ql3"\."step_run_mutations"/.test(text),
).length,
2,
);
assert.equal(
queries.filter(({ text }) =>
/INSERT INTO "ql3"\."plugin_package_workflow_admissions"/.test(text),
).length,
1,
);
});
test('exactly replays durable admission after current StepRun progression', async () => {
const value = fixture('postgres-workflow-replay');
const bundle = createPluginPackageWorkflowAdmissionBundle(value.plan);
const collect = bundle.stepMutations.find(
({ stepRun }) => stepRun.stepKey === 'collect',
).stepRun;
const running = transitionStepRunMutation(
collect,
{
expectedVersion: collect.version,
expectedDigest: collect.stepRunDigest,
mutationId: 'postgres-workflow-progress-running',
to: 'running',
atMs: 4_000,
},
{
expectedRunVersion: bundle.run.version,
expectedRunEventSequence: bundle.run.eventSequence,
eventId: 'postgres-workflow-running-event',
dedupeKey: 'postgres-workflow-running-event',
actor: { type: 'executor' },
},
);
const succeeded = transitionStepRunMutation(
running.stepRun,
{
expectedVersion: running.stepRun.version,
expectedDigest: running.stepRun.stepRunDigest,
mutationId: 'postgres-workflow-progress-success',
to: 'succeeded',
atMs: 5_000,
},
{
expectedRunVersion: bundle.run.version + 1,
expectedRunEventSequence: bundle.run.eventSequence + 1,
eventId: 'postgres-workflow-success-event',
dedupeKey: 'postgres-workflow-success-event',
actor: { type: 'executor' },
},
);
const queries = [];
const client = {
async query(text, values) {
queries.push({ text, values });
if (
text.includes('plugin_package_workflow_admissions') &&
text.includes('WHERE plan_id')
) {
return {
rows: [admissionRow(bundle)],
};
}
if (text.includes('FROM "ql3"."runs" WHERE id')) {
return {
rows: [
runRow({
...bundle.run,
version: bundle.run.version + 2,
eventSequence: bundle.run.eventSequence + 2,
}),
],
};
}
if (text.includes('FROM "ql3"."run_events"')) {
return {
rows: [
eventRow(bundle.admissionEvent),
...bundle.stepMutations.map(({ event }) => eventRow(event)),
],
};
}
if (
text.includes('FROM "ql3"."plugin_package_workflow_admission_steps"')
) {
return {
rows: [
stepEvidenceRow(
bundle,
values[1],
values[1] === 'collect' ? succeeded.stepRun : undefined,
),
],
};
}
return { rows: [] };
},
release() {},
};
const repository = new PostgresPluginPackageWorkflowAdmissionRepository(
poolWithClient(client),
);
assert.deepEqual(await repository.findPlanByPlanId(value.plan.planId), value.plan);
assert.deepEqual(await repository.admit(value.plan), {
status: 'existing',
receipt: bundle.receipt,
});
assert.equal(
queries.some(({ text }) =>
text.includes('plugin_package_workflow_admission_snapshot'),
),
false,
);
const initialEventsQuery = queries.find(({ text }) =>
text.includes('FROM "ql3"."run_events"'),
);
assert.match(initialEventsQuery.text, /sequence <= \$2/);
assert.deepEqual(initialEventsQuery.values, [
value.plan.runId,
bundle.receipt.finalRunEventSequence,
]);
assert.equal(queries.at(-1).text, 'COMMIT');
});
test('rolls back without Run evidence when the database guard denies admission', async () => {
const value = fixture('postgres-workflow-denied');
const queries = [];
const client = {
async query(text, values) {
queries.push({ text, values });
if (
text.includes('plugin_package_workflow_admissions') &&
text.includes('WHERE plan_id')
) {
return { rows: [] };
}
if (text.includes('plugin_package_workflow_admission_snapshot')) {
return { rows: [] };
}
return { rows: [] };
},
release() {},
};
const repository = new PostgresPluginPackageWorkflowAdmissionRepository(
poolWithClient(client),
);
await assert.rejects(
repository.admit(value.plan),
PluginPackageWorkflowAdmissionNotAllowedError,
);
assert.equal(queries.at(-1).text, 'ROLLBACK');
assert.equal(
queries.some(({ text }) => /INSERT INTO "ql3"\."runs"/.test(text)),
false,
);
});
test('surfaces serialization failure as bounded retryable unavailability', async () => {
const value = fixture('postgres-workflow-serialization');
const serializationFailure = Object.assign(
new Error('could not serialize access'),
{ code: '40001' },
);
const client = {
async query(text) {
if (
text.includes('plugin_package_workflow_admissions') &&
text.includes('WHERE plan_id')
) {
throw serializationFailure;
}
return { rows: [] };
},
release() {},
};
const repository = new PostgresPluginPackageWorkflowAdmissionRepository(
poolWithClient(client),
);
await assert.rejects(
repository.admit(value.plan),
PluginPackageWorkflowAdmissionUnavailableError,
);
});
test('publishes Workflow admission only through its explicit cluster subpath', () => {
const authority = require('@qinglong/cluster-postgres/plugin-package-workflow-admission');
const root = require('../dist');
const runtime = require('@qinglong/cluster-postgres/runtime');
assert.equal(
authority.PostgresPluginPackageWorkflowAdmissionRepository,
PostgresPluginPackageWorkflowAdmissionRepository,
);
assert.equal(
root.PostgresPluginPackageWorkflowAdmissionRepository,
undefined,
);
assert.equal(
runtime.PostgresPluginPackageWorkflowAdmissionRepository,
undefined,
);
});
@@ -0,0 +1,497 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
createInitialPluginPackageAutomationPublication,
} = require('@qinglong/runtime-core/plugin-package-automation-publication');
const {
createPluginPackageWorkflowAdmissionBundle,
createPluginPackageWorkflowExecutionPlan,
} = require('@qinglong/runtime-core/plugin-package-workflow-execution-plan');
const {
InvalidPluginPackageWorkflowFrontierError,
PluginPackageWorkflowFrontierUnavailableError,
resolvePluginPackageWorkflowFrontier,
} = require('@qinglong/runtime-core/plugin-package-workflow-frontier');
const {
transitionStepRunMutation,
} = require('@qinglong/runtime-core/step-run');
const {
pluginPackageTaskReconciliationFixture,
} = require('../../../test/contracts/pluginPackageTaskReconciliationRepositoryContract.cjs');
const {
PostgresPluginPackageWorkflowFrontierRepository,
} = require('../dist/plugin-package/workflow/pluginPackageWorkflowFrontierRepository');
function fixture(namespace) {
const value = pluginPackageTaskReconciliationFixture(namespace, {
workflows: [
{
schema: 'qinglong/plugin-package-workflow-resource@v1',
id: 'daily',
name: 'Daily workflow',
enabled: true,
steps: [
{ id: 'collect', task: 'alpha', needs: [] },
{ id: 'summarize', task: 'beta', needs: ['collect'] },
],
},
],
});
const publication = createInitialPluginPackageAutomationPublication(
value.revision,
value.registry,
2_000,
);
const plan = createPluginPackageWorkflowExecutionPlan({
planId: `wf-plan-${namespace}`,
runId: `wf-run-${namespace}`,
workflowId: 'daily',
stepRunIds: {
collect: `wf-collect-${namespace}`,
summarize: `wf-summary-${namespace}`,
},
publication,
revision: value.revision,
taskSpecSemanticRegistry: value.registry,
plannedAtMs: 3_000,
});
const bundle = createPluginPackageWorkflowAdmissionBundle(plan);
return { ...value, publication, plan, bundle };
}
function transition(stepRun, to, runVersion, atMs) {
return transitionStepRunMutation(
stepRun,
{
expectedVersion: stepRun.version,
expectedDigest: stepRun.stepRunDigest,
mutationId: `${stepRun.stepKey}-${to}-${stepRun.version}`,
to,
atMs,
...(to === 'failed' ? { resultCode: 'task_failed' } : {}),
},
{
expectedRunVersion: runVersion,
expectedRunEventSequence: runVersion,
eventId: `${stepRun.stepKey}-${to}-event-${stepRun.version}`,
dedupeKey: `${stepRun.stepKey}-${to}-event-${stepRun.version}`,
actor: { type: 'executor' },
},
).stepRun;
}
function progressedSnapshot(value, collectTerminal = 'succeeded') {
const collectInitial = value.bundle.stepMutations.find(
({ stepRun }) => stepRun.stepKey === 'collect',
).stepRun;
const summarize = value.bundle.stepMutations.find(
({ stepRun }) => stepRun.stepKey === 'summarize',
).stepRun;
const collectRunning = transition(collectInitial, 'running', 3, 4_000);
const collect = transition(
collectRunning,
collectTerminal,
4,
5_000,
);
return {
run: Object.freeze({
...value.bundle.run,
version: 5,
eventSequence: 5,
}),
stepRuns: Object.freeze([collect, summarize]),
};
}
function runRow(run) {
return {
id: run.id,
projectId: run.projectId,
taskId: run.taskId,
taskRevision: run.taskRevision,
taskName: run.taskName ?? null,
taskSnapshotRef: run.taskSnapshotRef ?? null,
legacyCronId: run.legacyCronId ?? null,
parentRunId: run.parentRunId ?? null,
retryOfRunId: run.retryOfRunId ?? null,
triggerId: run.triggerId ?? null,
triggerType: run.triggerType,
executionOrigin: run.executionOrigin,
executionOwner: run.executionOwner,
triggeredBy: run.triggeredBy ?? null,
requestId: run.requestId ?? null,
scheduledForMs: run.scheduledForMs ?? null,
status: run.status,
version: run.version,
eventSequence: run.eventSequence,
priority: run.priority,
idempotencyKey: run.idempotencyKey ?? null,
inputRef: run.inputRef ?? null,
outputRef: run.outputRef ?? null,
createdAtMs: run.createdAtMs,
queuedAtMs: run.queuedAtMs ?? null,
startedAtMs: run.startedAtMs ?? null,
finishedAtMs: run.finishedAtMs ?? null,
cancelRequestedAtMs: run.cancelRequestedAtMs ?? null,
cancelReason: run.cancelReason ?? null,
errorCode: run.errorCode ?? null,
errorSummary: run.errorSummary ?? null,
};
}
function stepRunRow(stepRun) {
return {
id: stepRun.id,
runId: stepRun.runId,
parentStepRunId: stepRun.parentStepRunId,
stepKey: stepRun.stepKey,
kind: stepRun.kind,
definitionRef: stepRun.definitionRef,
definitionDigest: stepRun.definitionDigest,
required: stepRun.required,
status: stepRun.status,
version: stepRun.version,
attemptCount: stepRun.attemptCount,
inputRef: stepRun.inputRef ?? null,
outputRef: stepRun.outputRef ?? null,
approvalRequestId: stepRun.approvalRequestId ?? null,
readyAtMs: stepRun.readyAtMs ?? null,
startedAtMs: stepRun.startedAtMs ?? null,
finishedAtMs: stepRun.finishedAtMs ?? null,
resultCode: stepRun.resultCode ?? null,
errorSummary: stepRun.errorSummary ?? null,
createdAtMs: stepRun.createdAtMs,
updatedAtMs: stepRun.updatedAtMs,
lastMutationId: stepRun.lastMutationId,
stepRunDigest: stepRun.stepRunDigest,
stepRunJson: stepRun,
};
}
function transactionPool(value, snapshot, options = {}) {
const queries = [];
let connections = 0;
let releases = 0;
const pool = {
async query(text, values) {
queries.push({ connection: 0, text, values });
if (text.includes('plugin_package_workflow_admissions')) {
return {
rows: options.candidateRows ?? [
{
runId: value.plan.runId,
planDigest: value.plan.planDigest,
admittedAtMs: 3_000,
},
],
};
}
return { rows: [] };
},
async connect() {
connections += 1;
const connection = connections;
let serializationFailureRemaining =
connection <= (options.serializationFailures ?? 0) ? 1 : 0;
return {
async query(text, values) {
queries.push({ connection, text, values });
if (
serializationFailureRemaining > 0 &&
text.includes('plugin_package_workflow_admissions')
) {
serializationFailureRemaining -= 1;
throw Object.assign(new Error('could not serialize access'), {
code: '40001',
});
}
if (
text.includes('plugin_package_workflow_admissions') &&
text.includes('plan_json')
) {
return {
rows: [
{
planDigest: value.plan.planDigest,
planJson: value.plan,
},
],
};
}
if (text.includes('FROM "ql3"."runs"')) {
return { rows: [runRow(snapshot.run)] };
}
if (text.includes('FROM "ql3"."step_runs"')) {
return { rows: snapshot.stepRuns.map(stepRunRow) };
}
if (text.includes('transaction_timestamp()')) {
return { rows: [{ observedAtMs: options.observedAtMs ?? 6_000 }] };
}
return { rows: [], rowCount: 1 };
},
release() {
releases += 1;
queries.push({ connection, text: 'RELEASE' });
},
};
},
};
return {
pool,
queries,
get connections() {
return connections;
},
get releases() {
return releases;
},
};
}
test('bounds and keyset-pages only actionable Workflow candidates', async () => {
const value = fixture('pg-frontier-page');
const snapshot = progressedSnapshot(value);
const secondDigest = 'f'.repeat(64);
const db = transactionPool(value, snapshot, {
candidateRows: [
{
runId: value.plan.runId,
planDigest: value.plan.planDigest,
admittedAtMs: 3_000,
},
{
runId: 'wf-run-next',
planDigest: secondDigest,
admittedAtMs: 3_001,
},
],
});
const repository =
new PostgresPluginPackageWorkflowFrontierRepository(db.pool);
assert.deepEqual(
await repository.listCandidates({
limit: 1,
after: {
admittedAtMs: 2_999,
planDigest: '0'.repeat(64),
},
}),
{
candidates: [
{
runId: value.plan.runId,
planDigest: value.plan.planDigest,
admittedAtMs: 3_000,
},
],
truncated: true,
next: {
admittedAtMs: 3_000,
planDigest: value.plan.planDigest,
},
},
);
const candidateQuery = db.queries[0];
assert.deepEqual(candidateQuery.values, [2_999, '0'.repeat(64), 2]);
assert.match(candidateQuery.text, /jsonb_array_elements_text/);
assert.match(
candidateQuery.text,
/run\.cancel_requested_at_ms IS NULL/,
);
assert.match(candidateQuery.text, /current\.status = 'pending'/);
assert.match(candidateQuery.text, /dependency\.status IN/);
assert.match(candidateQuery.text, /LIMIT \$3/);
await assert.rejects(
repository.listCandidates({ limit: 65 }),
InvalidPluginPackageWorkflowFrontierError,
);
assert.equal(db.queries.length, 1);
});
test('advances a ready dependency with locked rows and one aggregate Run CAS', async () => {
const value = fixture('pg-frontier-advance');
const snapshot = progressedSnapshot(value);
const db = transactionPool(value, snapshot);
const repository =
new PostgresPluginPackageWorkflowFrontierRepository(db.pool);
const result = await repository.advance(value.plan.runId);
assert.equal(result.status, 'advanced');
assert.equal(result.stepMutationCount, 1);
assert.equal(result.terminalStatus, null);
assert.equal(result.runVersion, 6);
assert.deepEqual(result.readyStepRunIds, [
value.plan.steps.find(({ stepKey }) => stepKey === 'summarize').stepRunId,
]);
const sql = db.queries.map(({ text }) => text);
assert.equal(sql[0], 'BEGIN ISOLATION LEVEL SERIALIZABLE');
assert.match(
sql.find((text) => text.includes('FROM "ql3"."runs"')),
/FOR UPDATE/,
);
assert.match(
sql.find((text) => text.includes('FROM "ql3"."step_runs"')),
/FOR UPDATE/,
);
assert.equal(
sql.filter((text) => text.includes('UPDATE "ql3"."runs"')).length,
1,
);
assert.equal(
sql.filter((text) => text.includes('UPDATE "ql3"."step_runs"')).length,
1,
);
assert.equal(
sql.filter((text) => text.includes('INSERT INTO "ql3"."run_events"'))
.length,
1,
);
assert.equal(
sql.filter((text) =>
text.includes('INSERT INTO "ql3"."step_run_mutations"'),
).length,
1,
);
assert.equal(sql.at(-2), 'COMMIT');
assert.equal(sql.at(-1), 'RELEASE');
});
test('skips blocked work and terminalizes the aggregate in one transaction', async () => {
const value = fixture('pg-frontier-terminal');
const snapshot = progressedSnapshot(value, 'failed');
const db = transactionPool(value, snapshot);
const repository =
new PostgresPluginPackageWorkflowFrontierRepository(db.pool);
const result = await repository.advance(value.plan.runId);
assert.equal(result.status, 'terminal');
assert.equal(result.stepMutationCount, 1);
assert.equal(result.terminalStatus, 'failed');
assert.equal(result.runVersion, 7);
const runUpdate = db.queries.find(({ text }) =>
text.includes('UPDATE "ql3"."runs"'),
);
assert.deepEqual(runUpdate.values.slice(0, 4), [
'failed',
2,
6_000,
'workflow_step_failed',
]);
assert.equal(
db.queries.filter(({ text }) =>
text.includes('INSERT INTO "ql3"."run_events"'),
).length,
2,
);
assert.equal(db.queries.at(-2).text, 'COMMIT');
});
test('retries a serialization race with a fresh client and converges', async () => {
const value = fixture('pg-frontier-retry');
const snapshot = progressedSnapshot(value);
const db = transactionPool(value, snapshot, {
serializationFailures: 1,
});
const repository =
new PostgresPluginPackageWorkflowFrontierRepository(db.pool);
assert.equal((await repository.advance(value.plan.runId)).status, 'advanced');
assert.equal(db.connections, 2);
assert.equal(db.releases, 2);
assert.equal(
db.queries.filter(({ text }) => text === 'ROLLBACK').length,
1,
);
const unavailableDb = transactionPool(value, snapshot, {
serializationFailures: 99,
});
await assert.rejects(
new PostgresPluginPackageWorkflowFrontierRepository(
unavailableDb.pool,
).advance(value.plan.runId),
PluginPackageWorkflowFrontierUnavailableError,
);
assert.equal(unavailableDb.connections, 3);
assert.equal(unavailableDb.releases, 3);
});
test('returns settled for an already terminal aggregate without writes', async () => {
const value = fixture('pg-frontier-settled');
const progressed = progressedSnapshot(value);
const frontier = resolvePluginPackageWorkflowFrontier({
plan: value.plan,
run: progressed.run,
stepRuns: progressed.stepRuns,
observedAtMs: 6_000,
});
const summarizeReady = frontier.stepMutations[0].stepRun;
const summarizeRunning = transition(summarizeReady, 'running', 6, 7_000);
const summarizeSucceeded = transition(
summarizeRunning,
'succeeded',
7,
8_000,
);
const terminalResolution = resolvePluginPackageWorkflowFrontier({
plan: value.plan,
run: {
...progressed.run,
version: 8,
eventSequence: 8,
},
stepRuns: [progressed.stepRuns[0], summarizeSucceeded],
observedAtMs: 9_000,
});
const terminal = terminalResolution.terminalTransition;
assert.ok(terminal);
const snapshot = {
run: Object.freeze({
...progressed.run,
status: terminal.status,
version: 9,
eventSequence: 9,
finishedAtMs: terminal.finishedAtMs,
errorCode: terminal.errorCode ?? undefined,
}),
stepRuns: Object.freeze([
progressed.stepRuns[0],
summarizeSucceeded,
]),
};
const db = transactionPool(value, snapshot, { observedAtMs: 9_000 });
const result =
await new PostgresPluginPackageWorkflowFrontierRepository(
db.pool,
).advance(value.plan.runId);
assert.equal(result.status, 'settled');
assert.equal(result.terminalStatus, 'succeeded');
assert.equal(
db.queries.some(({ text }) => text.includes('UPDATE "ql3"')),
false,
);
assert.equal(db.queries.at(-2).text, 'COMMIT');
});
test('publishes Workflow frontier only through its explicit cluster subpath', () => {
const authority = require('@qinglong/cluster-postgres/plugin-package-workflow-frontier');
const root = require('../dist');
const runtime = require('@qinglong/cluster-postgres/runtime');
assert.equal(
authority.PostgresPluginPackageWorkflowFrontierRepository,
PostgresPluginPackageWorkflowFrontierRepository,
);
assert.equal(
root.PostgresPluginPackageWorkflowFrontierRepository,
undefined,
);
assert.equal(
runtime.PostgresPluginPackageWorkflowFrontierRepository,
undefined,
);
});
@@ -0,0 +1,501 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
compileClusterCommandTaskDefinition,
} = require('@qinglong/runtime-core/cluster-execution-revision');
const {
createInitialPluginPackageAutomationPublication,
} = require('@qinglong/runtime-core/plugin-package-automation-publication');
const {
planPluginPackageTaskReconciliation,
pluginPackageTaskReconciliationTaskIds,
} = require('@qinglong/runtime-core/plugin-package-task-reconciliation');
const {
createPluginPackageWorkflowAdmissionBundle,
createPluginPackageWorkflowExecutionPlan,
} = require('@qinglong/runtime-core/plugin-package-workflow-execution-plan');
const {
InvalidPluginPackageWorkflowTaskAttemptAdmissionError,
PluginPackageWorkflowTaskAttemptAdmissionConflictError,
} = require('@qinglong/runtime-core/plugin-package-workflow-task-attempt-admission');
const {
pluginPackageTaskReconciliationFixture,
} = require('../../../test/contracts/pluginPackageTaskReconciliationRepositoryContract.cjs');
const {
PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository,
} = require('../dist/plugin-package/workflow/pluginPackageWorkflowTaskAttemptAdmissionRepository');
function fixture(namespace) {
const value = pluginPackageTaskReconciliationFixture(namespace, {
workflows: [
{
schema: 'qinglong/plugin-package-workflow-resource@v1',
id: 'daily',
name: 'Daily workflow',
enabled: true,
steps: [
{ id: 'collect', task: 'alpha', needs: [] },
{ id: 'summarize', task: 'beta', needs: ['collect'] },
],
},
],
});
const publication = createInitialPluginPackageAutomationPublication(
value.revision,
value.registry,
2_000,
);
const plan = createPluginPackageWorkflowExecutionPlan({
planId: `wf-attempt-plan-${namespace}`,
runId: `wf-attempt-run-${namespace}`,
workflowId: 'daily',
stepRunIds: {
collect: `wf-attempt-collect-${namespace}`,
summarize: `wf-attempt-summary-${namespace}`,
},
publication,
revision: value.revision,
taskSpecSemanticRegistry: value.registry,
plannedAtMs: 3_000,
});
const admission = createPluginPackageWorkflowAdmissionBundle(plan);
const reconciliationPlan = planPluginPackageTaskReconciliation({
revision: value.revision,
previousReceipt: null,
facts: pluginPackageTaskReconciliationTaskIds(
value.revision,
null,
value.registry,
).map((taskId) => ({
taskId,
packageName: null,
current: null,
})),
committedAtMs: 2_500,
taskSpecSemanticRegistry: value.registry,
});
const taskDefinition = reconciliationPlan.writes.find(
({ definition }) =>
definition.taskId === `pkg:${value.packageName}:alpha`,
).definition;
const execution = compileClusterCommandTaskDefinition(
taskDefinition,
value.registry,
);
const stepRun = admission.stepMutations.find(
({ stepRun: candidate }) => candidate.stepKey === 'collect',
).stepRun;
return {
...value,
plan,
admission,
reconciliation: reconciliationPlan.receipt,
execution,
stepRun,
};
}
function runRow(run) {
return {
id: run.id,
projectId: run.projectId,
taskId: run.taskId,
taskRevision: run.taskRevision,
taskName: run.taskName ?? null,
taskSnapshotRef: run.taskSnapshotRef ?? null,
legacyCronId: run.legacyCronId ?? null,
parentRunId: run.parentRunId ?? null,
retryOfRunId: run.retryOfRunId ?? null,
triggerId: run.triggerId ?? null,
triggerType: run.triggerType,
executionOrigin: run.executionOrigin,
executionOwner: run.executionOwner,
triggeredBy: run.triggeredBy ?? null,
requestId: run.requestId ?? null,
scheduledForMs: run.scheduledForMs ?? null,
status: run.status,
version: run.version,
eventSequence: run.eventSequence,
priority: run.priority,
idempotencyKey: run.idempotencyKey ?? null,
inputRef: run.inputRef ?? null,
outputRef: run.outputRef ?? null,
createdAtMs: run.createdAtMs,
queuedAtMs: run.queuedAtMs ?? null,
startedAtMs: run.startedAtMs ?? null,
finishedAtMs: run.finishedAtMs ?? null,
cancelRequestedAtMs: run.cancelRequestedAtMs ?? null,
cancelReason: run.cancelReason ?? null,
errorCode: run.errorCode ?? null,
errorSummary: run.errorSummary ?? null,
};
}
function stepRunRow(stepRun) {
return {
id: stepRun.id,
runId: stepRun.runId,
parentStepRunId: stepRun.parentStepRunId ?? null,
stepKey: stepRun.stepKey,
kind: stepRun.kind,
definitionRef: stepRun.definitionRef,
definitionDigest: stepRun.definitionDigest,
required: stepRun.required,
status: stepRun.status,
version: stepRun.version,
attemptCount: stepRun.attemptCount,
inputRef: stepRun.inputRef ?? null,
outputRef: stepRun.outputRef ?? null,
approvalRequestId: stepRun.approvalRequestId ?? null,
readyAtMs: stepRun.readyAtMs ?? null,
startedAtMs: stepRun.startedAtMs ?? null,
finishedAtMs: stepRun.finishedAtMs ?? null,
resultCode: stepRun.resultCode ?? null,
errorSummary: stepRun.errorSummary ?? null,
createdAtMs: stepRun.createdAtMs,
updatedAtMs: stepRun.updatedAtMs,
lastMutationId: stepRun.lastMutationId,
stepRunDigest: stepRun.stepRunDigest,
stepRunJson: stepRun,
};
}
function snapshotRow(value) {
const execution = value.execution;
return {
planJson: value.plan,
reconciliationJson: value.reconciliation,
executionProjectId: execution.projectId,
executionTaskId: execution.taskId,
executionSourceRevision: execution.sourceRevision,
executionTaskRevision: execution.taskRevision,
executionSourceContentDigest: execution.sourceContentDigest,
executionExecutorType: execution.executorType,
executionPlanSchema: execution.planSchema,
executionPlanJson: {
command: execution.command,
environment: execution.environment,
...(execution.workingDirectory === undefined
? {}
: { workingDirectory: execution.workingDirectory }),
...(execution.timeoutMs === undefined
? {}
: { timeoutMs: execution.timeoutMs }),
...(execution.placement === undefined
? {}
: { placement: execution.placement }),
},
executionContentDigest: execution.contentDigest,
executionCreatedAtMs: execution.createdAtMs,
};
}
function mockPool(value, options = {}) {
const queries = [];
let connections = 0;
let releases = 0;
return {
pool: {
async connect() {
connections += 1;
const connection = connections;
let retryFailure =
connection <= (options.serializationFailures ?? 0);
return {
async query(text, values) {
queries.push({ connection, text, values });
if (
retryFailure &&
text.includes(
'plugin_package_workflow_task_attempt_snapshot',
)
) {
retryFailure = false;
throw Object.assign(new Error('serialization failure'), {
code: '40001',
});
}
if (
text.includes('FROM "ql3"."step_runs" AS current')
) {
return {
rows: options.candidateRows ?? [
{
runId: value.plan.runId,
stepRunId: value.stepRun.id,
readyAtMs: value.stepRun.readyAtMs,
planDigest: value.plan.planDigest,
},
],
};
}
if (text.includes('FROM "ql3"."runs"')) {
return {
rows: [
runRow(options.run ?? value.admission.run),
],
};
}
if (
text.includes('FROM "ql3"."step_runs"') &&
text.includes('FOR UPDATE')
) {
return { rows: [stepRunRow(value.stepRun)] };
}
if (
text.includes(
'plugin_package_workflow_task_attempt_admissions',
) &&
text.includes('receipt_json') &&
text.includes('WHERE run_id')
) {
return {
rows: options.existingReceipt
? [
{
receiptDigest:
options.existingReceipt.receiptDigest,
receiptJson: options.existingReceipt,
},
]
: [],
};
}
if (
text.includes(
'plugin_package_workflow_task_attempt_snapshot',
)
) {
return {
rows: options.snapshotMissing
? []
: [snapshotRow(value)],
};
}
if (text.includes('transaction_timestamp()')) {
return {
rows: [{ admittedAtMs: options.admittedAtMs ?? 4_000 }],
};
}
if (text.includes('MAX(attempt)')) {
return {
rows: [
{ attemptNumber: options.attemptNumber ?? 1 },
],
};
}
return { rows: [], rowCount: 1 };
},
release() {
releases += 1;
queries.push({ connection, text: 'RELEASE' });
},
};
},
},
queries,
get connections() {
return connections;
},
get releases() {
return releases;
},
};
}
test('bounds and keyset-pages only ready uncancelled Task attempts', async () => {
const value = fixture('pg-wta-page');
const db = mockPool(value, {
candidateRows: [
{
runId: value.plan.runId,
stepRunId: value.stepRun.id,
readyAtMs: value.stepRun.readyAtMs,
planDigest: value.plan.planDigest,
},
{
runId: 'wf-attempt-run-next',
stepRunId: 'wf-attempt-step-next',
readyAtMs: value.stepRun.readyAtMs + 1,
planDigest: 'f'.repeat(64),
},
],
});
const repository =
new PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository(
db.pool,
);
assert.deepEqual(
await repository.listCandidates({
limit: 1,
after: {
readyAtMs: value.stepRun.readyAtMs - 1,
stepRunId: 'previous-step',
},
}),
{
candidates: [
{
runId: value.plan.runId,
stepRunId: value.stepRun.id,
readyAtMs: value.stepRun.readyAtMs,
planDigest: value.plan.planDigest,
},
],
truncated: true,
next: {
readyAtMs: value.stepRun.readyAtMs,
stepRunId: value.stepRun.id,
},
},
);
const query = db.queries[0];
assert.deepEqual(query.values, [
'previous-step',
value.stepRun.readyAtMs - 1,
2,
]);
assert.match(query.text, /run\.cancel_requested_at_ms IS NULL/);
assert.match(query.text, /current\.status = 'ready'/);
assert.match(query.text, /NOT EXISTS/);
assert.match(query.text, /LIMIT \$3/);
await assert.rejects(
repository.listCandidates({ limit: 65 }),
InvalidPluginPackageWorkflowTaskAttemptAdmissionError,
);
});
test('atomically admits the exact cluster execution and replays the epoch', async () => {
const value = fixture('pg-wta-create');
const db = mockPool(value);
const repository =
new PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository(
db.pool,
);
const created = await repository.admit(
value.plan.runId,
value.stepRun.id,
);
assert.equal(created.status, 'created');
assert.equal(created.receipt.resourceTaskId, 'alpha');
assert.equal(
created.receipt.taskId,
`pkg:${value.packageName}:alpha`,
);
assert.equal(created.receipt.executorType, 'remote_worker');
assert.equal(created.receipt.executionDigest, value.execution.contentDigest);
assert.equal(created.receipt.attemptNumber, 1);
assert.equal(created.receipt.runVersion, value.admission.run.version + 1);
const sql = db.queries.map(({ text }) => text);
assert.equal(sql[0], 'BEGIN ISOLATION LEVEL SERIALIZABLE');
assert.match(
sql.find((text) => text.includes('FROM "ql3"."runs"')),
/FOR UPDATE/,
);
assert.match(
sql.find(
(text) =>
text.includes('FROM "ql3"."step_runs"') &&
text.includes('FOR UPDATE'),
),
/FOR UPDATE/,
);
assert.ok(
sql.some((text) =>
text.includes('plugin_package_workflow_task_attempt_snapshot'),
),
);
assert.equal(
sql.filter((text) => text.includes('UPDATE "ql3"."runs"')).length,
1,
);
assert.ok(
sql.some((text) => text.includes('INSERT INTO "ql3"."run_attempts"')),
);
assert.ok(
sql.some((text) => text.includes('INSERT INTO "ql3"."run_events"')),
);
assert.ok(
sql.some((text) =>
text.includes(
'"ql3"."plugin_package_workflow_task_attempt_admissions"',
),
),
);
assert.equal(sql.at(-2), 'COMMIT');
assert.equal(sql.at(-1), 'RELEASE');
const replayDb = mockPool(value, {
existingReceipt: created.receipt,
run: {
...value.admission.run,
version: created.receipt.runVersion,
eventSequence: created.receipt.runEventSequence,
},
});
const replay =
new PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository(
replayDb.pool,
);
assert.deepEqual(
await replay.admit(value.plan.runId, value.stepRun.id),
{ status: 'existing', receipt: created.receipt },
);
assert.equal(
replayDb.queries.some(({ text }) =>
text.includes('plugin_package_workflow_task_attempt_snapshot'),
),
false,
);
});
test('retries serialization once and fails closed without a snapshot', async () => {
const value = fixture('pg-wta-retry');
const retried = mockPool(value, { serializationFailures: 1 });
const repository =
new PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository(
retried.pool,
);
assert.equal(
(
await repository.admit(value.plan.runId, value.stepRun.id)
).status,
'created',
);
assert.equal(retried.connections, 2);
assert.equal(retried.releases, 2);
assert.ok(retried.queries.some(({ text }) => text === 'ROLLBACK'));
const missing = mockPool(value, { snapshotMissing: true });
await assert.rejects(
new PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository(
missing.pool,
).admit(value.plan.runId, value.stepRun.id),
PluginPackageWorkflowTaskAttemptAdmissionConflictError,
);
assert.ok(missing.queries.some(({ text }) => text === 'ROLLBACK'));
assert.equal(
missing.queries.some(({ text }) =>
text.includes('INSERT INTO "ql3"."run_attempts"'),
),
false,
);
});
test('is available only through the explicit package subpath', () => {
assert.match(
require.resolve(
'@qinglong/cluster-postgres/plugin-package-workflow-task-attempt-admission',
),
/pluginPackageWorkflowTaskAttemptAdmissionRepository\.js$/,
);
const root = require('@qinglong/cluster-postgres');
assert.equal(
root.PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository,
undefined,
);
});
@@ -0,0 +1,281 @@
const assert = require('node:assert/strict');
const { EventEmitter } = require('node:events');
const test = require('node:test');
const { types } = require('pg');
const {
POSTGRES_AVAILABILITY_SQLSTATE_CLASSES,
POSTGRES_AVAILABILITY_SQLSTATES,
POSTGRES_AVAILABILITY_SYSTEM_ERROR_CODES,
PgPoolBinding,
createPostgresDatabaseOpener,
isPostgresAvailabilityError,
} = require('../dist');
function options(overrides = {}) {
return {
role: 'runtime',
connection: {
connectionString: 'postgresql://ql3:secret@127.0.0.1:5432/ql3',
tls: { mode: 'disable' },
},
onPoolError() {},
...overrides,
};
}
test('creates and closes a pg.Pool without eagerly opening a connection', async () => {
let passwordReads = 0;
const openDatabase = createPostgresDatabaseOpener(
options({
connection: {
host: '127.0.0.1',
database: 'ql3',
user: 'ql3',
password() {
passwordReads += 1;
return 'secret';
},
tls: { mode: 'disable' },
},
}),
);
assert.equal(passwordReads, 0);
const database = await openDatabase();
assert.equal(passwordReads, 0);
await Promise.all([database.close(), database.close()]);
assert.equal(passwordReads, 0);
});
test('rejects connection-string TLS overrides and unsafe application names', () => {
assert.throws(
() =>
createPostgresDatabaseOpener(
options({
connection: {
connectionString:
'postgresql://ql3:secret@127.0.0.1/ql3?sslmode=disable',
tls: { mode: 'verify-full' },
},
}),
),
/explicit tls option/,
);
assert.throws(
() =>
createPostgresDatabaseOpener(
options({ pool: { applicationName: 'ql3 runtime -c role=admin' } }),
),
/safe identifier/,
);
assert.throws(
() =>
createPostgresDatabaseOpener(
options({
connection: {
connectionString: 'postgresql://ql3:secret@database.internal/ql3',
tls: { mode: 'verify-full' },
},
}),
),
/explicit DNS servername/,
);
});
test('enforces role-specific bounded pool sizes', () => {
assert.throws(
() =>
createPostgresDatabaseOpener(options({ pool: { maxConnections: 65 } })),
/between 1 and 64/,
);
assert.throws(
() =>
createPostgresDatabaseOpener(
options({
role: 'migration',
pool: { maxConnections: 5 },
}),
),
/between 1 and 4/,
);
for (const role of [
'ai-maintenance',
'ai-credential-tester',
'automation-manager',
'approval-manager',
'worker-credential-manager',
'worker-credential-executor',
]) {
assert.doesNotThrow(() => createPostgresDatabaseOpener(options({ role })));
assert.throws(
() =>
createPostgresDatabaseOpener(
options({ role, pool: { maxConnections: 5 } }),
),
/between 1 and 4/,
);
}
assert.throws(
() =>
createPostgresDatabaseOpener(
options({
role: 'worker-ingress',
pool: { maxConnections: 17 },
}),
),
/between 1 and 16/,
);
assert.throws(
() =>
createPostgresDatabaseOpener(
options({
role: 'admin',
pool: { maxConnections: 5 },
}),
),
/between 1 and 4/,
);
assert.throws(
() => createPostgresDatabaseOpener(options({ role: 'unknown' })),
/database role is invalid/,
);
});
test('keeps PostgreSQL bigint parsing as strings and omits pg-native', () => {
assert.equal(types.getTypeParser(20)('9007199254740993'), '9007199254740993');
assert.throws(() => require.resolve('pg-native'), {
code: 'MODULE_NOT_FOUND',
});
});
test('classifies only explicit PostgreSQL and transport availability codes', () => {
assert.deepEqual(POSTGRES_AVAILABILITY_SQLSTATE_CLASSES, ['08']);
for (const code of [
...POSTGRES_AVAILABILITY_SQLSTATES,
...POSTGRES_AVAILABILITY_SYSTEM_ERROR_CODES,
]) {
assert.equal(
isPostgresAvailabilityError(Object.assign(new Error(code), { code })),
true,
code,
);
}
assert.equal(
isPostgresAvailabilityError(
Object.assign(new Error('vendor connection exception'), {
code: '08999',
}),
),
true,
);
for (const code of ['23505', '40001', '40P01', '55P03', '57014']) {
assert.equal(
isPostgresAvailabilityError(Object.assign(new Error(code), { code })),
false,
code,
);
}
assert.equal(isPostgresAvailabilityError(new Error('uncoded')), false);
assert.equal(isPostgresAvailabilityError({ code: '08006' }), false);
});
test('reports query availability without replacing the original rejection', async () => {
const connectionLost = Object.assign(new Error('connection lost'), {
code: '08006',
});
const duplicate = Object.assign(new Error('duplicate'), { code: '23505' });
const observed = [];
let currentError = connectionLost;
const pool = new PgPoolBinding(
{
async query() {
throw currentError;
},
},
(error) => {
observed.push(error);
throw new Error('availability listener failure');
},
);
await assert.rejects(
pool.query('SELECT 1'),
(error) => error === connectionLost,
);
assert.deepEqual(observed, [connectionLost]);
currentError = duplicate;
await assert.rejects(pool.query('INSERT'), (error) => error === duplicate);
assert.deepEqual(observed, [connectionLost]);
});
test('reports availability from connect and bound-client query failures', async () => {
const connectError = Object.assign(new Error('network unreachable'), {
code: 'ENETUNREACH',
});
const readOnlyError = Object.assign(new Error('read-only transaction'), {
code: '25006',
});
const observed = [];
let connectFails = true;
const driverClient = Object.assign(new EventEmitter(), {
async query() {
throw readOnlyError;
},
release() {},
});
const pool = new PgPoolBinding(
{
async connect() {
if (connectFails) throw connectError;
return driverClient;
},
},
(error) => observed.push(error),
);
await assert.rejects(pool.connect(), (error) => error === connectError);
connectFails = false;
const client = await pool.connect();
await assert.rejects(
client.query('SELECT 1'),
(error) => error === readOnlyError,
);
client.release();
assert.deepEqual(observed, [connectError, readOnlyError]);
});
test('contains checked-out client error events and removes its listener on release', async () => {
const administratorShutdown = Object.assign(
new Error('terminating connection due to administrator command'),
{ code: '57P01' },
);
const observed = [];
let releases = 0;
const driverClient = Object.assign(new EventEmitter(), {
async query() {
return { rows: [], rowCount: 0 };
},
release() {
releases += 1;
},
});
const pool = new PgPoolBinding(
{
async connect() {
return driverClient;
},
},
(error) => observed.push(error),
);
const client = await pool.connect();
assert.equal(driverClient.listenerCount('error'), 1);
assert.equal(driverClient.emit('error', administratorShutdown), true);
assert.deepEqual(observed, [administratorShutdown]);
client.release();
assert.equal(releases, 1);
assert.equal(driverClient.listenerCount('error'), 0);
});
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,174 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
PostgresMigrationLeaderUnavailableError,
PostgresMigrationStreamStore,
} = require('../dist');
const { runMigrationStream } = require('@qinglong/runtime-core');
const CHECKSUM = 'a'.repeat(64);
function createPool(lockResults = [true, true]) {
const records = new Map();
const calls = [];
let released = 0;
let lockIndex = 0;
function selectRecord(values, source) {
const record = source.get(values[0]);
return { rows: record ? [{ ...record }] : [] };
}
return {
calls,
records,
get released() {
return released;
},
pool: {
async query(text, values = []) {
calls.push({ scope: 'pool', text, values: [...values] });
if (text.startsWith('SELECT\n stream_id')) {
if (values.length === 0) {
return {
rows: [...records.values()].map((record) => ({ ...record })),
};
}
return selectRecord(values, records);
}
throw new Error(`unexpected pool query: ${text}`);
},
async connect() {
let staged = null;
return {
async query(text, values = []) {
calls.push({ scope: 'client', text, values: [...values] });
if (text === 'BEGIN') {
staged = new Map(
[...records].map(([id, record]) => [id, { ...record }]),
);
return { rows: [] };
}
if (text === 'COMMIT') {
records.clear();
for (const [id, record] of staged) records.set(id, record);
staged = null;
return { rows: [] };
}
if (text === 'ROLLBACK') {
staged = null;
return { rows: [] };
}
if (text.startsWith('SELECT set_config')) return { rows: [] };
if (text.startsWith('SELECT pg_try_advisory_xact_lock')) {
return { rows: [{ acquired: lockResults[lockIndex++] ?? true }] };
}
if (text.startsWith('CREATE SCHEMA')) return { rows: [] };
if (text.startsWith('CREATE TABLE')) return { rows: [] };
if (text.startsWith('SELECT\n stream_id')) {
return selectRecord(values, staged);
}
if (text.startsWith('INSERT INTO')) {
const [migrationId, streamId, dialect, checksum, appliedAtMs] =
values;
staged.set(migrationId, {
migrationId,
streamId,
dialect,
checksum,
appliedAtMs: String(appliedAtMs),
});
return { rows: [], rowCount: 1 };
}
if (text === 'CREATE TABLE ql3.run_probe(id integer)') {
return { rows: [] };
}
throw new Error(`unexpected client query: ${text}`);
},
release() {
released += 1;
},
};
},
},
};
}
function stream(up) {
return {
id: 'postgresql-main',
dialect: 'postgresql',
migrationIdScheme: 'postgres-prefixed',
checksumScheme: 'sha256',
migrations: [
{
id: 'pg-0001-schema-history',
checksum: CHECKSUM,
up,
},
],
};
}
test('serializes PostgreSQL history bootstrap and migration in advisory-lock transactions', async () => {
const state = createPool();
let upCalls = 0;
await runMigrationStream({
stream: stream(async (context) => {
upCalls += 1;
await context.query('CREATE TABLE ql3.run_probe(id integer)');
}),
store: new PostgresMigrationStreamStore(state.pool),
clock: () => 123,
});
assert.equal(upCalls, 1);
assert.equal(state.released, 2);
assert.deepEqual(state.records.get('pg-0001-schema-history'), {
migrationId: 'pg-0001-schema-history',
streamId: 'postgresql-main',
dialect: 'postgresql',
checksum: CHECKSUM,
appliedAtMs: '123',
});
assert.equal(
state.calls.filter(({ text }) =>
text.startsWith('SELECT pg_try_advisory_xact_lock'),
).length,
2,
);
assert.equal(state.calls.filter(({ text }) => text === 'COMMIT').length, 2);
});
test('fails closed before migration work when another migration leader owns the lock', async () => {
const state = createPool([true, false]);
let upCalls = 0;
await assert.rejects(
runMigrationStream({
stream: stream(async () => {
upCalls += 1;
}),
store: new PostgresMigrationStreamStore(state.pool),
}),
PostgresMigrationLeaderUnavailableError,
);
assert.equal(upCalls, 0);
assert.equal(state.records.size, 0);
assert.equal(state.calls.filter(({ text }) => text === 'ROLLBACK').length, 1);
assert.equal(state.released, 2);
});
test('rolls PostgreSQL migration work and history back together', async () => {
const state = createPool();
await assert.rejects(
runMigrationStream({
stream: stream(async () => {
throw new Error('migration work failed');
}),
store: new PostgresMigrationStreamStore(state.pool),
}),
/migration work failed/,
);
assert.equal(state.records.size, 0);
assert.equal(state.calls.filter(({ text }) => text === 'ROLLBACK').length, 1);
});
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,222 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
ProjectPolicyUnavailableError,
ProjectRoleBindingMutationConflictError,
ProjectRoleBindingVersionConflictError,
} = require('@qinglong/runtime-core/project-policy');
const {
PostgresProjectPolicyRepository,
} = require('../dist/security/projectPolicyRepository');
const BINDING = Object.freeze({
projectId: 'default',
subject: Object.freeze({ type: 'user', id: 'usr_primary' }),
version: 1,
state: 'active',
role: 'owner',
mutationId: 'grant-owner-1',
changedBy: Object.freeze({ type: 'system', id: 'bootstrap' }),
createdAtMs: 1,
});
function bindingRow(overrides = {}) {
return {
projectId: 'default',
subjectType: 'user',
subjectId: 'usr_primary',
version: 1,
state: 'active',
role: 'owner',
mutationId: 'grant-owner-1',
changedByType: 'system',
changedById: 'bootstrap',
createdAtMs: '1',
...overrides,
};
}
function snapshotRow(overrides = {}) {
return {
projectId: 'default',
projectName: 'Default',
projectSlug: 'default',
projectStatus: 'active',
projectVersion: 1,
projectCreatedAtMs: '0',
projectUpdatedAtMs: '0',
bindingProjectId: 'default',
bindingSubjectType: 'user',
bindingSubjectId: 'usr_primary',
bindingVersion: 1,
bindingState: 'active',
bindingRole: 'owner',
bindingMutationId: 'grant-owner-1',
bindingChangedByType: 'system',
bindingChangedById: 'bootstrap',
bindingCreatedAtMs: '1',
...overrides,
};
}
test('resolves one normalized Project and latest RoleBinding snapshot', async () => {
const queries = [];
const repository = new PostgresProjectPolicyRepository({
async query(text, values) {
queries.push({ text, values });
return { rows: [snapshotRow()] };
},
async connect() {
throw new Error('not used');
},
});
assert.deepEqual(
await repository.resolve('default', { type: 'user', id: 'usr_primary' }),
{
project: {
id: 'default',
name: 'Default',
slug: 'default',
status: 'active',
version: 1,
createdAtMs: 0,
updatedAtMs: 0,
},
binding: BINDING,
},
);
assert.deepEqual(queries[0].values, ['default', 'user', 'usr_primary']);
assert.match(queries[0].text, /LEFT JOIN LATERAL/);
assert.match(queries[0].text, /ORDER BY candidate\.version DESC/);
await assert.rejects(
repository.resolve('x'.repeat(129), {
type: 'user',
id: 'usr_primary',
}),
);
assert.equal(queries.length, 1);
});
function appendPool(options = {}) {
const events = [];
let connection = 0;
return {
events,
pool: {
async query() {
throw new Error('not used');
},
async connect() {
connection += 1;
let inserted = false;
return {
async query(text) {
events.push(text.split('\n', 1)[0]);
if (text.startsWith('SELECT id FROM'))
return { rows: [{ id: 'default' }] };
if (text.includes('mutation_id = $2')) {
return {
rows: options.replay ? [bindingRow(options.replay)] : [],
};
}
if (text.startsWith('SELECT version FROM')) {
return {
rows:
options.currentVersion === undefined
? []
: [{ version: options.currentVersion }],
};
}
if (text.startsWith('INSERT INTO')) {
if (options.retryOnce && connection === 1 && !inserted) {
inserted = true;
throw Object.assign(new Error('serialization'), {
code: '40001',
});
}
return { rows: [], rowCount: 1 };
}
return { rows: [] };
},
release() {
events.push(`release:${connection}`);
},
};
},
},
};
}
test('appends under a serializable Project lock and replays exactly', async () => {
const insertedFixture = appendPool();
const inserted = await new PostgresProjectPolicyRepository(
insertedFixture.pool,
).append({ expectedCurrentVersion: 0, binding: BINDING });
assert.deepEqual(inserted, { status: 'inserted', binding: BINDING });
assert.equal(
insertedFixture.events.some((event) =>
event.startsWith('BEGIN ISOLATION LEVEL SERIALIZABLE'),
),
true,
);
assert.ok(
insertedFixture.events.findIndex((event) => event.startsWith('SELECT id')) <
insertedFixture.events.findIndex((event) => event.startsWith('INSERT')),
);
assert.ok(insertedFixture.events.includes('COMMIT'));
const replayFixture = appendPool({ replay: {} });
const replay = await new PostgresProjectPolicyRepository(
replayFixture.pool,
).append({ expectedCurrentVersion: 0, binding: BINDING });
assert.deepEqual(replay, { status: 'existing', binding: BINDING });
assert.equal(
replayFixture.events.some((event) => event.startsWith('INSERT')),
false,
);
});
test('rejects stale versions and conflicting mutation replay', async () => {
await assert.rejects(
new PostgresProjectPolicyRepository(
appendPool({ currentVersion: 1 }).pool,
).append({ expectedCurrentVersion: 0, binding: BINDING }),
ProjectRoleBindingVersionConflictError,
);
await assert.rejects(
new PostgresProjectPolicyRepository(
appendPool({ replay: { role: 'viewer' } }).pool,
).append({ expectedCurrentVersion: 0, binding: BINDING }),
ProjectRoleBindingMutationConflictError,
);
});
test('retries serialization failures but fails closed on corrupt rows', async () => {
const retryFixture = appendPool({ retryOnce: true });
assert.equal(
(
await new PostgresProjectPolicyRepository(retryFixture.pool).append({
expectedCurrentVersion: 0,
binding: BINDING,
})
).status,
'inserted',
);
assert.equal(
retryFixture.events.filter((event) => event.startsWith('release:')).length,
2,
);
const repository = new PostgresProjectPolicyRepository({
async query() {
return { rows: [snapshotRow({ projectVersion: 'not-a-number' })] };
},
async connect() {
throw new Error('not used');
},
});
await assert.rejects(
repository.resolve('default', { type: 'user', id: 'usr_primary' }),
ProjectPolicyUnavailableError,
);
});
@@ -0,0 +1,170 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
createProjectToolDefinitionSnapshot,
ProjectToolDefinitionSnapshotConflictError,
} = require('@qinglong/runtime-core/project-tool-definition-snapshot');
const {
PostgresProjectToolDefinitionSnapshotRepository,
} = require('../dist/tool-execution/projectToolDefinitionSnapshotRepository');
function fakePool() {
const queries = [];
const snapshots = new Map();
const client = {
async query(text, values = []) {
queries.push({ text, values });
if (
text.startsWith('BEGIN') ||
text.startsWith('SELECT set_config') ||
text === 'COMMIT' ||
text === 'ROLLBACK'
) {
return { rows: [] };
}
if (text.includes('FROM "ql3"."plugin_package_install_heads"')) {
return { rows: [] };
}
if (
text.startsWith('INSERT INTO "ql3"."project_tool_definition_snapshots"')
) {
const key = `${values[0]}:${values[1]}`;
if (snapshots.has(key)) return { rows: [] };
snapshots.set(key, {
projectId: values[0],
activeVectorDigest: values[1],
definitionsDigest: values[2],
snapshotDigest: values[3],
snapshotJson: JSON.parse(values[4]),
committedAtMs: 500,
});
return { rows: [{ activeVectorDigest: values[1] }] };
}
if (text.includes('FROM "ql3"."project_tool_definition_snapshots"')) {
const row = snapshots.get(`${values[0]}:${values[1]}`);
return { rows: row ? [{ ...row }] : [] };
}
if (
text.includes('FROM "ql3"."project_tool_definition_snapshot_sources"')
) {
return { rows: [] };
}
throw new Error(`unexpected SQL: ${text}`);
},
release() {},
};
return {
queries,
pool: {
query: (...args) => client.query(...args),
async connect() {
return client;
},
},
corrupt(snapshot) {
snapshots.get(
`${snapshot.projectId}:${snapshot.activeVectorDigest}`,
).snapshotJson.snapshotDigest = 'f'.repeat(64);
},
};
}
test('publishes and exactly replays one empty PostgreSQL snapshot', async () => {
const value = fakePool();
const repository = new PostgresProjectToolDefinitionSnapshotRepository(
value.pool,
);
const snapshot = createProjectToolDefinitionSnapshot({
projectId: 'project-postgres-snapshot',
contributions: [],
});
assert.equal(await repository.findCurrent(snapshot.projectId), null);
const created = await repository.publish(snapshot);
assert.equal(created.status, 'created');
assert.deepEqual(created.record.snapshot, snapshot);
assert.equal((await repository.publish(snapshot)).status, 'existing');
assert.deepEqual(
await repository.findCurrent(snapshot.projectId),
created.record,
);
assert.match(
value.queries.find(({ text }) =>
text.startsWith('INSERT INTO "ql3"."project_tool_definition_snapshots"'),
).text,
/clock_timestamp\(\)/,
);
assert.equal(
value.queries.some(({ text }) =>
text.startsWith('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY'),
),
true,
);
});
test('fails closed for corrupted PostgreSQL snapshot JSON', async () => {
const value = fakePool();
const repository = new PostgresProjectToolDefinitionSnapshotRepository(
value.pool,
);
const snapshot = createProjectToolDefinitionSnapshot({
projectId: 'project-postgres-corrupt',
contributions: [],
});
await repository.publish(snapshot);
value.corrupt(snapshot);
await assert.rejects(
repository.findCurrent(snapshot.projectId),
/snapshot is unavailable/,
);
});
test('maps PostgreSQL constraint rejection to snapshot conflict', async () => {
const repository = new PostgresProjectToolDefinitionSnapshotRepository({
async query() {
return { rows: [] };
},
async connect() {
return {
async query(text) {
if (
text.startsWith('BEGIN') ||
text.startsWith('SELECT set_config')
) {
return { rows: [] };
}
const error = new Error('constraint');
error.code = '23505';
throw error;
},
release() {},
};
},
});
const snapshot = createProjectToolDefinitionSnapshot({
projectId: 'project-postgres-conflict',
contributions: [],
});
await assert.rejects(
repository.publish(snapshot),
ProjectToolDefinitionSnapshotConflictError,
);
});
test('publishes PostgreSQL snapshot storage only through executor subpaths', () => {
assert.equal(
require('@qinglong/cluster-postgres/project-tool-definition-snapshot')
.PostgresProjectToolDefinitionSnapshotRepository,
PostgresProjectToolDefinitionSnapshotRepository,
);
assert.equal(
require('../dist/entrypoints/packageExecutor')
.PostgresProjectToolDefinitionSnapshotRepository,
PostgresProjectToolDefinitionSnapshotRepository,
);
assert.equal(
require('../dist').PostgresProjectToolDefinitionSnapshotRepository,
undefined,
);
});
@@ -0,0 +1,384 @@
const assert = require('node:assert/strict');
const { createHash } = require('node:crypto');
const { test } = require('node:test');
const {
RemoteRunActivationUnavailableError,
} = require('@qinglong/runtime-core/remote-activation');
const {
createStepRunRecord,
} = require('@qinglong/runtime-core/step-run');
const {
PostgresRemoteRunActivationRepository,
} = require('../dist/entrypoints/runtime');
const SESSION_ID = '018f5c64-9b9d-7f1a-8c2d-1234567890ac';
const LEASE_TOKEN = 'worker_generated_lease_capability_0000000000000001';
const LEASE_DIGEST = createHash('sha256').update(LEASE_TOKEN).digest('hex');
function command() {
return {
runId: 'run-1',
attemptId: 'attempt-1',
workerId: 'edge-1',
workerSessionId: SESSION_ID,
workerGeneration: 2,
offerId: 'offer-1',
leaseGeneration: 3,
leaseToken: LEASE_TOKEN,
expectedLeaseVersion: 4,
eventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a1',
};
}
function fixture({ failEvent = false, timeoutMs = 5_000, omitTimeout = false } = {}) {
const calls = [];
const client = {
async query(sql, params = []) {
const normalized = sql.replace(/\s+/g, ' ').trim();
calls.push({ sql: normalized, params });
if (
normalized === 'BEGIN' ||
normalized.startsWith('SET LOCAL') ||
normalized === 'COMMIT' ||
normalized === 'ROLLBACK' ||
normalized.startsWith('SELECT pg_advisory_xact_lock')
) return { rows: [], rowCount: 0 };
if (normalized.includes('FROM "ql3"."worker_sessions"')) {
return {
rows: [{
workerId: 'edge-1',
sessionId: SESSION_ID,
workerGeneration: 2,
workerStatus: 'online',
workerLeaseExpiresAtMs: 2_000,
}],
rowCount: 1,
};
}
if (normalized.includes('INNER JOIN "ql3"."run_attempts"')) {
return {
rows: [{
runId: 'run-1',
runStatus: 'dispatching',
executionOwner: 'runtime',
cancelRequestedAtMs: null,
cancelReason: null,
runErrorCode: null,
runVersion: 3,
eventSequence: 7,
planJson: {
command: { kind: 'argv', file: '/bin/true', args: [] },
environment: [],
...(omitTimeout ? {} : { timeoutMs }),
},
attemptId: 'attempt-1',
attemptRunId: 'run-1',
attemptStatus: 'claimed',
executorType: 'remote_worker',
attemptWorkerId: 'edge-1',
attemptWorkerSessionId: SESSION_ID,
attemptWorkerGeneration: 2,
attemptLeaseGeneration: 3,
attemptLeaseVersion: 4,
attemptLeaseTokenDigest: LEASE_DIGEST,
attemptOfferId: 'offer-1',
callbackSequence: 0,
callbackTokenDigest: null,
executorHandle: null,
logArtifactId: null,
deadlineAtMs: null,
startedAtMs: null,
finishedAtMs: null,
attemptErrorCode: null,
}],
rowCount: 1,
};
}
if (normalized.includes('FROM "ql3"."run_dispatch_leases"')) {
return {
rows: [{
attemptId: 'attempt-1',
runId: 'run-1',
leaseStatus: 'leased',
leaseVersion: 4,
leaseGeneration: 3,
workerId: 'edge-1',
workerSessionId: SESSION_ID,
workerGeneration: 2,
leaseTokenDigest: LEASE_DIGEST,
offerId: 'offer-1',
leaseExpiresAtMs: 2_000,
}],
rowCount: 1,
};
}
if (normalized.includes('statement_timestamp()')) {
return { rows: [{ nowMs: 1_000 }], rowCount: 1 };
}
if (normalized.startsWith('UPDATE "ql3"."run_attempts"')) {
return { rows: [], rowCount: 1 };
}
if (normalized.startsWith('UPDATE "ql3"."runs"')) {
return { rows: [], rowCount: 1 };
}
if (normalized.startsWith('INSERT INTO "ql3"."run_events"')) {
if (failEvent) throw new Error('injected event failure');
return { rows: [], rowCount: 1 };
}
throw new Error(`Unexpected SQL: ${normalized}`);
},
release() {
calls.push({ sql: 'RELEASE', params: [] });
},
};
const pool = {
async connect() { return client; },
};
return { repository: new PostgresRemoteRunActivationRepository(pool), calls };
}
test('locks every authority before database time and persists digest-only starting', async () => {
const { repository, calls } = fixture();
const activation = await repository.acknowledgeStarting(command());
assert.equal(activation.status, 'applied');
assert.equal(activation.snapshot.attemptStatus, 'starting');
assert.equal(activation.snapshot.deadlineAtMs, 6_000);
const index = (needle) => calls.findIndex(({ sql }) => sql.includes(needle));
assert.ok(index('pg_advisory_xact_lock') < index('worker_sessions'));
assert.ok(index('worker_sessions') < index('INNER JOIN "ql3"."run_attempts"'));
assert.ok(index('INNER JOIN "ql3"."run_attempts"') < index('run_dispatch_leases'));
assert.ok(index('run_dispatch_leases') < index('statement_timestamp()'));
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), true);
assert.equal(JSON.stringify(calls).includes(LEASE_TOKEN), false);
assert.equal(JSON.stringify(calls).includes(LEASE_DIGEST), true);
const attemptUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."run_attempts"'));
assert.equal(attemptUpdate.params.at(-1), 6_000);
});
test('keeps the durable deadline absent when the immutable revision has no timeout', async () => {
const { repository, calls } = fixture({ omitTimeout: true });
const activation = await repository.acknowledgeStarting(command());
assert.equal(activation.snapshot.deadlineAtMs, undefined);
const attemptUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."run_attempts"'));
assert.equal(attemptUpdate.params.at(-1), null);
});
test('rolls the transaction back and exposes storage failure as unavailable', async () => {
const { repository, calls } = fixture({ failEvent: true });
await assert.rejects(
repository.acknowledgeStarting(command()),
(error) =>
error instanceof RemoteRunActivationUnavailableError &&
error.cause?.message === 'injected event failure',
);
assert.equal(calls.some(({ sql }) => sql === 'ROLLBACK'), true);
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), false);
});
function workflowFixture() {
const stepRun = createStepRunRecord({
id: 'workflow-step-1',
runId: 'run-1',
stepKey: 'collect',
kind: 'task',
definitionRef: 'pkg:demo:alpha',
definitionDigest: 'a'.repeat(64),
required: true,
initialStatus: 'ready',
mutationId: 'workflow-step-created',
createdAtMs: 500,
});
const calls = [];
const client = {
async query(sql, params = []) {
const normalized = sql.replace(/\s+/g, ' ').trim();
calls.push({ sql: normalized, params });
if (
normalized === 'BEGIN' ||
normalized.startsWith('SET LOCAL') ||
normalized === 'COMMIT' ||
normalized === 'ROLLBACK' ||
normalized.startsWith('SELECT pg_advisory_xact_lock')
) {
return { rows: [], rowCount: 0 };
}
if (normalized.includes('FROM "ql3"."worker_sessions"')) {
return {
rows: [{
workerId: 'edge-1',
sessionId: SESSION_ID,
workerGeneration: 2,
workerStatus: 'online',
workerLeaseExpiresAtMs: 2_000,
}],
rowCount: 1,
};
}
if (normalized.includes('INNER JOIN "ql3"."run_attempts"')) {
return {
rows: [{
runId: 'run-1',
runStatus: 'running',
executionOwner: 'runtime',
cancelRequestedAtMs: null,
cancelReason: null,
runErrorCode: null,
runVersion: 4,
eventSequence: 4,
planJson: {
command: { kind: 'argv', file: '/bin/true', args: [] },
environment: [],
timeoutMs: 5_000,
},
attemptId: 'attempt-1',
attemptRunId: 'run-1',
attemptStepRunId: stepRun.id,
attemptStatus: 'starting',
executorType: 'remote_worker',
attemptWorkerId: 'edge-1',
attemptWorkerSessionId: SESSION_ID,
attemptWorkerGeneration: 2,
attemptLeaseGeneration: 3,
attemptLeaseVersion: 4,
attemptLeaseTokenDigest: LEASE_DIGEST,
attemptOfferId: 'offer-1',
callbackSequence: 0,
callbackTokenDigest: null,
executorHandle: null,
logArtifactId: null,
deadlineAtMs: 6_000,
startedAtMs: null,
finishedAtMs: null,
attemptErrorCode: null,
workflowAttemptId: 'attempt-1',
workflowStepRunId: stepRun.id,
admittedWorkflowStepVersion: stepRun.version,
admittedWorkflowStepDigest: stepRun.stepRunDigest,
}],
rowCount: 1,
};
}
if (normalized.includes('FROM "ql3"."step_runs"')) {
return {
rows: [{
workflowStepVersion: stepRun.version,
workflowStepDigest: stepRun.stepRunDigest,
workflowStepJson: stepRun,
}],
rowCount: 1,
};
}
if (normalized.includes('FROM "ql3"."run_dispatch_leases"')) {
return {
rows: [{
attemptId: 'attempt-1',
runId: 'run-1',
leaseStatus: 'leased',
leaseVersion: 4,
leaseGeneration: 3,
workerId: 'edge-1',
workerSessionId: SESSION_ID,
workerGeneration: 2,
leaseTokenDigest: LEASE_DIGEST,
offerId: 'offer-1',
leaseExpiresAtMs: 2_000,
}],
rowCount: 1,
};
}
if (normalized.includes('statement_timestamp()')) {
return { rows: [{ nowMs: 1_000 }], rowCount: 1 };
}
if (
normalized.startsWith('UPDATE ') ||
normalized.startsWith('INSERT INTO ')
) {
return { rows: [], rowCount: 1 };
}
throw new Error(`Unexpected SQL: ${normalized}`);
},
release() {
calls.push({ sql: 'RELEASE', params: [] });
},
};
return {
calls,
repository: new PostgresRemoteRunActivationRepository({
async connect() {
return client;
},
}),
stepRun,
};
}
test('activates a Workflow Task Attempt by advancing StepRun, not the aggregate Run', async () => {
const { repository, calls, stepRun } = workflowFixture();
const result = await repository.acknowledgeRunning({
...command(),
attemptEventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a2',
runEventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a3',
executorHandle: 'worker-handle-1',
callbackSequence: 1,
callbackTokenDigest: 'b'.repeat(64),
});
assert.equal(result.status, 'applied');
assert.equal(result.snapshot.runStatus, 'running');
assert.equal(result.snapshot.attemptStatus, 'running');
const runUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."runs"'));
assert.equal(runUpdate.params[1], 'running');
assert.equal(runUpdate.params.at(-1), 'running');
const stepUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."step_runs"'));
assert.ok(stepUpdate);
assert.equal(stepUpdate.params[0], 'running');
assert.equal(stepUpdate.params[2], stepRun.attemptCount + 1);
assert.ok(
calls.some(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."step_run_mutations"')),
);
const attemptEvent = calls.find(
({ sql, params }) =>
sql.startsWith('INSERT INTO "ql3"."run_events"') &&
params[3] === 'workflow.task_attempt.running',
);
assert.equal(attemptEvent.params[7], stepRun.id);
assert.match(attemptEvent.params[8], /"execution_scope":"workflow_task"/);
});
test('fails a Workflow Task before StepRun start without terminalizing its aggregate Run', async () => {
const { repository, calls, stepRun } = workflowFixture();
const result = await repository.failStart({
...command(),
attemptEventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a2',
runEventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a3',
});
assert.equal(result.status, 'applied');
assert.equal(result.snapshot.runStatus, 'running');
assert.equal(result.snapshot.attemptStatus, 'failed');
assert.equal(result.snapshot.errorCode, 'EXECUTOR_START_FAILED');
const runUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."runs"'));
assert.equal(runUpdate.params[1], 'running');
assert.equal(runUpdate.params[2], null);
assert.equal(runUpdate.params[3], null);
assert.equal(runUpdate.params[4], null);
const stepUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."step_runs"'));
assert.ok(stepUpdate);
assert.equal(stepUpdate.params[0], 'failed');
assert.equal(stepUpdate.params[2], stepRun.attemptCount);
assert.equal(stepUpdate.params[5], stepRun.readyAtMs);
assert.equal(stepUpdate.params[6], null);
assert.equal(stepUpdate.params[7], 1_000);
assert.ok(
calls.some(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."step_run_mutations"')),
);
});
@@ -0,0 +1,130 @@
const assert = require('node:assert/strict');
const { createHash } = require('node:crypto');
const { test } = require('node:test');
const {
PostgresRemoteRunActivationRepository,
} = require('@qinglong/cluster-postgres/runtime');
test('pins the shared running status parameter to PostgreSQL varchar', async () => {
const token = 'worker_generated_lease_capability_0000000000000001';
const digest = createHash('sha256').update(token).digest('hex');
const client = {
async query(text) {
const sql = String(text);
if (sql === 'BEGIN' || sql === 'COMMIT' || sql === 'ROLLBACK') {
return { rows: [] };
}
if (sql.startsWith('SET LOCAL')) return { rows: [] };
if (sql.includes('pg_advisory_xact_lock')) return { rows: [{}] };
if (sql.includes('FROM "ql3"."worker_sessions"')) {
return {
rows: [{
workerId: 'worker-live',
sessionId: '019f7094-a853-72f3-82ab-dfa08e6bd1c1',
workerGeneration: 1,
workerStatus: 'online',
workerLeaseExpiresAtMs: '9000',
}],
};
}
if (sql.includes('FROM "ql3"."runs" AS run')) {
return {
rows: [{
runId: 'run-live',
runStatus: 'dispatching',
executionOwner: 'runtime',
cancelRequestedAtMs: null,
cancelReason: null,
runErrorCode: null,
runVersion: 2,
eventSequence: 2,
planJson: { command: {}, environment: [] },
attemptId: 'attempt-live',
attemptRunId: 'run-live',
attemptStepRunId: null,
attemptStatus: 'starting',
executorType: 'remote_worker',
attemptWorkerId: 'worker-live',
attemptWorkerSessionId: '019f7094-a853-72f3-82ab-dfa08e6bd1c1',
attemptWorkerGeneration: 1,
attemptLeaseGeneration: 1,
attemptLeaseVersion: 0,
attemptLeaseTokenDigest: digest,
attemptOfferId: 'offer-live',
callbackSequence: 0,
callbackTokenDigest: null,
executorHandle: null,
logArtifactId: null,
deadlineAtMs: null,
startedAtMs: null,
finishedAtMs: null,
attemptErrorCode: null,
workflowAttemptId: null,
workflowStepRunId: null,
admittedWorkflowStepVersion: null,
admittedWorkflowStepDigest: null,
}],
};
}
if (sql.includes('FROM "ql3"."run_dispatch_leases"')) {
return {
rows: [{
attemptId: 'attempt-live',
runId: 'run-live',
leaseStatus: 'leased',
leaseVersion: 0,
leaseGeneration: 1,
workerId: 'worker-live',
workerSessionId: '019f7094-a853-72f3-82ab-dfa08e6bd1c1',
workerGeneration: 1,
leaseTokenDigest: digest,
offerId: 'offer-live',
leaseExpiresAtMs: '9000',
}],
};
}
if (sql.includes('statement_timestamp()')) {
return { rows: [{ nowMs: '1000' }] };
}
if (sql.includes('UPDATE "ql3"."run_attempts"')) {
return { rowCount: 1, rows: [] };
}
if (sql.includes('UPDATE "ql3"."runs"')) {
assert.match(sql, /SET status = \$2::varchar/);
assert.match(sql, /WHEN \$2::varchar = 'running'/);
return { rowCount: 1, rows: [] };
}
if (sql.includes('INSERT INTO "ql3"."run_events"')) {
return { rowCount: 1, rows: [] };
}
throw new Error(`unexpected query: ${sql}`);
},
release() {},
};
const repository = new PostgresRemoteRunActivationRepository({
async connect() {
return client;
},
async query() {
throw new Error('not used');
},
});
const result = await repository.acknowledgeRunning({
runId: 'run-live',
attemptId: 'attempt-live',
workerId: 'worker-live',
workerSessionId: '019f7094-a853-72f3-82ab-dfa08e6bd1c1',
workerGeneration: 1,
offerId: 'offer-live',
leaseGeneration: 1,
leaseToken: token,
expectedLeaseVersion: 0,
attemptEventId: '019f7094-a853-72f3-82ab-dfa08e6bd1c2',
runEventId: '019f7094-a853-72f3-82ab-dfa08e6bd1c3',
executorHandle: 'ql3lp1.test',
logArtifactId: 'wlog-0123456789abcdef0123456789abcd',
callbackSequence: 1,
callbackTokenDigest: 'b'.repeat(64),
});
assert.equal(result.status, 'applied');
});
@@ -0,0 +1,94 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
PostgresRemoteWorkerAttestationEvidenceProvider,
} = require('../dist');
function target() {
return {
runId: 'run-1',
attemptId: 'attempt-1',
attemptNumber: 1,
executorType: 'remote-worker',
executorHandle: 'remote:handle-1',
callbackSequence: 5,
workerId: 'edge-1',
workerSessionId: '018f5c64-9b9d-7f1a-8c2d-1234567890ac',
workerGeneration: 2,
leaseTokenDigest: 'a'.repeat(64),
leaseGeneration: 3,
leaseVersion: 4,
offerId: 'offer-1',
};
}
function attestation(state, receivedAtMs = 9_000) {
return {
attestationId: '018f5c64-9b9d-7f1a-8c2d-1234567890ab',
...target(),
sequence: 1,
state,
journalRevision: 6,
receivedAtMs,
};
}
function provider(value, observedAtMs = '10000') {
return new PostgresRemoteWorkerAttestationEvidenceProvider(
{ async query() { return { rows: [{ observedAtMs }] }; } },
{
async findLatestExact(observed) {
assert.deepEqual(observed, {
runId: 'run-1', attemptId: 'attempt-1', workerId: 'edge-1',
workerSessionId: target().workerSessionId, workerGeneration: 2,
leaseTokenDigest: 'a'.repeat(64), leaseGeneration: 3,
leaseVersion: 4, offerId: 'offer-1', callbackSequence: 5,
executorHandle: 'remote:handle-1',
});
return value;
},
async submit() { throw new Error('not used'); },
},
{ runningFreshnessMs: 2_000 },
);
}
test('treats only exact stopped attestation as authoritative not-running', async () => {
const context = { signal: new AbortController().signal };
assert.deepEqual(await provider(attestation('stopped')).inspect(target(), context), {
status: 'not_running',
});
assert.deepEqual(await provider(attestation('running')).inspect(target(), context), {
status: 'running',
});
});
test('keeps missing, stale, malformed-time and cancelled evidence fail-closed', async () => {
const context = { signal: new AbortController().signal };
for (const candidate of [
provider(null),
provider(attestation('running', 1_000)),
provider(attestation('running'), 'corrupt'),
]) {
assert.deepEqual(await candidate.inspect(target(), context), {
status: 'unknown',
reason: 'provider_unavailable',
});
}
const controller = new AbortController();
controller.abort();
assert.deepEqual(await provider(attestation('running')).inspect(target(), {
signal: controller.signal,
}), { status: 'unknown', reason: 'provider_unavailable' });
});
test('rejects an unbounded attestation freshness window', () => {
assert.throws(
() => new PostgresRemoteWorkerAttestationEvidenceProvider(
{ async query() { return { rows: [] }; } },
{ async findLatestExact() { return null; }, async submit() {} },
{ runningFreshnessMs: 300_001 },
),
/freshness is invalid/,
);
});
@@ -0,0 +1,606 @@
'use strict';
const assert = require('node:assert/strict');
const { createHash } = require('node:crypto');
const { test } = require('node:test');
const {
RemoteWorkerCompletionFenceRejectedError,
RemoteWorkerCompletionUnavailableError,
} = require('@qinglong/runtime-core/remote-worker-completion');
const {
createStepRunRecord,
transitionStepRunRecord,
} = require('@qinglong/runtime-core/step-run');
const { PostgresRemoteWorkerCompletionRepository } = require('../dist/entrypoints/runtime');
const SESSION_ID = '018f5c64-9b9d-7f1a-8c2d-1234567890ac';
const LEASE_TOKEN = 'worker_generated_lease_capability_0000000000000001';
const LEASE_DIGEST = createHash('sha256').update(LEASE_TOKEN).digest('hex');
const LOG_ARTIFACT_ID = `wlog-${'a'.repeat(30)}`;
const ARTIFACT_SHA256 = 'b'.repeat(64);
const CALLBACK_DIGEST = 'c'.repeat(64);
function fence() {
return {
workerId: 'worker-1',
workerSessionId: SESSION_ID,
workerGeneration: 2,
projectId: 'project-1',
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
leaseGeneration: 3,
leaseToken: LEASE_TOKEN,
expectedLeaseVersion: 4,
};
}
function uploadCommand() {
return {
...fence(),
logArtifactId: LOG_ARTIFACT_ID,
byteLength: 10,
truncated: false,
};
}
function completionCommand(overrides = {}) {
return {
...fence(),
callbackSequence: 1,
callbackTokenDigest: CALLBACK_DIGEST,
result: {
outcome: 'succeeded',
startedAtMs: 100,
finishedAtMs: 200,
exitCode: 0,
},
artifact: {
logArtifactId: LOG_ARTIFACT_ID,
byteLength: 10,
sha256: ARTIFACT_SHA256,
truncated: false,
},
attemptEventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a1',
runEventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a2',
...overrides,
};
}
function aggregate(overrides = {}) {
return {
runId: 'run-1',
projectId: 'project-1',
runStatus: 'dispatching',
executionOwner: 'runtime',
cancelRequestedAtMs: null,
cancelReason: null,
runErrorCode: null,
runVersion: 3,
eventSequence: 7,
runCreatedAtMs: 40,
runStartedAtMs: null,
attemptId: 'attempt-1',
attemptRunId: 'run-1',
attemptStatus: 'starting',
executorType: 'remote_worker',
attemptWorkerId: 'worker-1',
attemptWorkerSessionId: SESSION_ID,
attemptWorkerGeneration: 2,
attemptLeaseGeneration: 3,
attemptLeaseVersion: 4,
attemptLeaseTokenDigest: LEASE_DIGEST,
attemptOfferId: 'offer-1',
callbackSequence: 0,
callbackTokenDigest: null,
logArtifactId: null,
attemptCreatedAtMs: 50,
startedAtMs: null,
finishedAtMs: null,
exitCode: null,
attemptErrorCode: null,
...overrides,
};
}
function lease(overrides = {}) {
return {
attemptId: 'attempt-1',
runId: 'run-1',
leaseStatus: 'leased',
leaseVersion: 4,
leaseGeneration: 3,
workerId: 'worker-1',
workerSessionId: SESSION_ID,
workerGeneration: 2,
leaseTokenDigest: LEASE_DIGEST,
offerId: 'offer-1',
leaseExpiresAtMs: 2_000,
...overrides,
};
}
function replayPayload(command, overrides = {}) {
return {
attempt_id: command.attemptId,
lease_generation: command.leaseGeneration,
from_status: 'running',
to_status: 'succeeded',
callback_sequence: command.callbackSequence,
callback_token_digest: command.callbackTokenDigest,
worker_started_at_ms: command.result.startedAtMs,
worker_finished_at_ms: command.result.finishedAtMs,
exit_code: command.result.exitCode,
log_artifact_id: command.artifact.logArtifactId,
artifact_byte_length: command.artifact.byteLength,
artifact_sha256: command.artifact.sha256,
artifact_truncated: command.artifact.truncated,
error_code: null,
...overrides,
};
}
function workflowReadyStep() {
return createStepRunRecord({
id: 'workflow-step-1',
runId: 'run-1',
stepKey: 'collect',
kind: 'task',
definitionRef: 'pkg:demo:alpha',
definitionDigest: 'a'.repeat(64),
required: true,
initialStatus: 'ready',
mutationId: 'workflow-step-created',
createdAtMs: 60,
});
}
function workflowAggregate(stepRun, overrides = {}) {
return aggregate({
runStatus: 'running',
runVersion: 4,
eventSequence: 9,
runStartedAtMs: 50,
attemptStepRunId: stepRun.id,
workflowAttemptId: 'attempt-1',
workflowStepRunId: stepRun.id,
admittedWorkflowStepVersion: 1,
admittedWorkflowStepDigest: workflowReadyStep().stepRunDigest,
...overrides,
});
}
function fixture(options = {}) {
const calls = [];
const client = {
async query(sql, params = []) {
const normalized = sql.replace(/\s+/g, ' ').trim();
calls.push({ sql: normalized, params });
if (
normalized === 'BEGIN' ||
normalized.startsWith('SET LOCAL') ||
normalized === 'COMMIT' ||
normalized === 'ROLLBACK' ||
normalized.startsWith('SELECT pg_advisory_xact_lock')
)
return { rows: [], rowCount: 0 };
if (normalized.includes('FROM "ql3"."worker_sessions"')) {
return {
rows: [
options.worker ?? {
workerId: 'worker-1',
sessionId: SESSION_ID,
workerGeneration: 2,
workerStatus: 'online',
workerLeaseExpiresAtMs: 2_000,
},
],
rowCount: 1,
};
}
if (normalized.includes('INNER JOIN "ql3"."run_attempts"')) {
return { rows: [options.aggregate ?? aggregate()], rowCount: 1 };
}
if (normalized.includes('FROM "ql3"."step_runs"')) {
return {
rows: options.stepRun
? [
{
workflowStepVersion: options.stepRun.version,
workflowStepDigest: options.stepRun.stepRunDigest,
workflowStepJson: options.stepRun,
},
]
: [],
rowCount: options.stepRun ? 1 : 0,
};
}
if (normalized.includes('FROM "ql3"."run_dispatch_leases"')) {
return { rows: [options.lease ?? lease()], rowCount: 1 };
}
if (normalized.includes('statement_timestamp()')) {
return { rows: [{ nowMs: options.nowMs ?? 1_000 }], rowCount: 1 };
}
if (
normalized.startsWith('SELECT payload') &&
normalized.includes('FROM "ql3"."run_events"')
) {
return {
rows: options.replayPayload
? [{ payload: options.replayPayload }]
: [],
rowCount: options.replayPayload ? 1 : 0,
};
}
if (
normalized.startsWith('UPDATE "ql3"."run_dispatch_leases"') ||
normalized.startsWith('UPDATE "ql3"."run_attempts"') ||
normalized.startsWith('UPDATE "ql3"."runs"') ||
normalized.startsWith('UPDATE "ql3"."step_runs"')
)
return { rows: [], rowCount: options.updateRowCount ?? 1 };
if (
normalized.startsWith('INSERT INTO "ql3"."run_events"') ||
normalized.startsWith('INSERT INTO "ql3"."step_run_mutations"')
) {
if (options.failEvent) throw new Error('injected event failure');
return { rows: [], rowCount: 1 };
}
throw new Error(`Unexpected SQL: ${normalized}`);
},
release() {
calls.push({ sql: 'RELEASE', params: [] });
},
};
return {
repository: new PostgresRemoteWorkerCompletionRepository({
async connect() {
return client;
},
}),
calls,
};
}
test('authorizes Artifact upload only after the shared Attempt lock and DB clock', async () => {
const { repository, calls } = fixture();
await repository.authorizeArtifactUpload(uploadCommand());
const index = (needle) => calls.findIndex(({ sql }) => sql.includes(needle));
assert.ok(index('pg_advisory_xact_lock') < index('worker_sessions'));
assert.ok(
index('worker_sessions') < index('INNER JOIN "ql3"."run_attempts"'),
);
assert.ok(
index('INNER JOIN "ql3"."run_attempts"') < index('run_dispatch_leases'),
);
assert.ok(index('run_dispatch_leases') < index('statement_timestamp()'));
assert.equal(
calls.some(({ sql }) => sql === 'COMMIT'),
true,
);
assert.equal(JSON.stringify(calls).includes(LEASE_TOKEN), false);
assert.equal(JSON.stringify(calls).includes(LEASE_DIGEST), false);
});
test('completes directly from the durable starting crash window in one transaction', async () => {
const { repository, calls } = fixture();
assert.deepEqual(await repository.complete(completionCommand()), {
status: 'applied',
runId: 'run-1',
attemptId: 'attempt-1',
callbackSequence: 1,
});
const attemptUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."run_attempts"'),
);
assert.equal(attemptUpdate.params[9], 'succeeded');
assert.equal(attemptUpdate.params[10], 5);
assert.equal(attemptUpdate.params[11], 1);
assert.equal(attemptUpdate.params[12], CALLBACK_DIGEST);
assert.equal(attemptUpdate.params[13], LOG_ARTIFACT_ID);
assert.equal(attemptUpdate.params[14], 100);
assert.equal(attemptUpdate.params[15], 1_000);
assert.equal(attemptUpdate.params[16], 0);
assert.equal(attemptUpdate.params[19], 'starting');
assert.equal(
calls.filter(({ sql }) => sql.startsWith('INSERT INTO "ql3"."run_events"'))
.length,
2,
);
assert.equal(
calls.some(({ sql }) => sql === 'COMMIT'),
true,
);
assert.equal(JSON.stringify(calls).includes(LEASE_TOKEN), false);
assert.equal(JSON.stringify(calls).includes(ARTIFACT_SHA256), true);
});
test('completes a Workflow Task from the starting crash window without terminalizing its parent Run', async () => {
const stepRun = workflowReadyStep();
const { repository, calls } = fixture({
stepRun,
aggregate: workflowAggregate(stepRun),
});
assert.equal(
(await repository.complete(completionCommand())).status,
'applied',
);
const runUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."runs"'),
);
assert.match(runUpdate.sql, /SET version = \$2, event_sequence = \$3/);
assert.equal(runUpdate.sql.includes('SET status ='), false);
assert.deepEqual(runUpdate.params, ['run-1', 7, 12, 4]);
const stepUpdates = calls.filter(({ sql }) =>
sql.startsWith('UPDATE "ql3"."step_runs"'),
);
assert.equal(stepUpdates.length, 2);
assert.equal(stepUpdates[0].params[0], 'running');
assert.equal(stepUpdates[0].params[2], 1);
assert.equal(stepUpdates[1].params[0], 'succeeded');
assert.equal(stepUpdates[1].params[2], 1);
assert.equal(
calls.filter(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."step_run_mutations"'),
).length,
2,
);
const attemptEvent = calls.find(
({ sql, params }) =>
sql.startsWith('INSERT INTO "ql3"."run_events"') &&
params[3] === 'workflow.task_attempt.succeeded',
);
assert.equal(attemptEvent.params[2], 11);
assert.equal(attemptEvent.params[7], stepRun.id);
assert.match(attemptEvent.params[8], /"execution_scope":"workflow_task"/);
assert.match(attemptEvent.params[8], /"step_run_id":"workflow-step-1"/);
});
test('makes parent Workflow cancellation win over a successful stopped Worker completion', async () => {
const ready = workflowReadyStep();
const running = transitionStepRunRecord(ready, {
expectedVersion: ready.version,
expectedDigest: ready.stepRunDigest,
mutationId: 'workflow-step-running-before-stop',
to: 'running',
atMs: 100,
});
const { repository, calls } = fixture({
stepRun: running,
aggregate: workflowAggregate(running, {
attemptStatus: 'running',
callbackSequence: 1,
callbackTokenDigest: CALLBACK_DIGEST,
logArtifactId: LOG_ARTIFACT_ID,
cancelRequestedAtMs: 150,
cancelReason: 'user',
startedAtMs: 100,
}),
});
assert.equal(
(await repository.complete(completionCommand())).status,
'applied',
);
const attemptUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."run_attempts"'),
);
assert.equal(attemptUpdate.params[9], 'cancelled');
assert.equal(attemptUpdate.params[17], 'EXECUTION_CANCELLED');
const runUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."runs"'),
);
assert.match(runUpdate.sql, /SET version = \$2, event_sequence = \$3/);
assert.equal(runUpdate.sql.includes('SET status ='), false);
const stepUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."step_runs"'),
);
assert.equal(stepUpdate.params[0], 'cancelled');
const attemptEvent = calls.find(
({ sql, params }) =>
sql.startsWith('INSERT INTO "ql3"."run_events"') &&
params[3] === 'workflow.task_attempt.cancelled',
);
assert.equal(attemptEvent.params[7], running.id);
assert.match(attemptEvent.params[8], /"error_code":"EXECUTION_CANCELLED"/);
assert.equal(
calls.some(({ sql }) => sql === 'COMMIT'),
true,
);
});
test('makes timeout intent win over a successful Worker exit', async () => {
const running = aggregate({
runStatus: 'running',
attemptStatus: 'running',
cancelRequestedAtMs: 900,
cancelReason: 'timeout',
callbackSequence: 1,
callbackTokenDigest: CALLBACK_DIGEST,
logArtifactId: LOG_ARTIFACT_ID,
runStartedAtMs: 100,
startedAtMs: 100,
});
const { repository, calls } = fixture({ aggregate: running });
await repository.complete(completionCommand());
const attemptUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."run_attempts"'),
);
assert.equal(attemptUpdate.params[9], 'timed_out');
assert.equal(attemptUpdate.params[17], 'EXECUTION_TIMED_OUT');
const runUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."runs"'),
);
assert.equal(runUpdate.params[1], 'timed_out');
});
test('makes a Workflow Task deadline win without timing out its parent Run', async () => {
const ready = workflowReadyStep();
const running = transitionStepRunRecord(ready, {
expectedVersion: ready.version,
expectedDigest: ready.stepRunDigest,
mutationId: 'workflow-step-running',
to: 'running',
atMs: 100,
});
const { repository, calls } = fixture({
stepRun: running,
aggregate: workflowAggregate(running, {
attemptStatus: 'running',
callbackSequence: 1,
callbackTokenDigest: CALLBACK_DIGEST,
logArtifactId: LOG_ARTIFACT_ID,
deadlineAtMs: 950,
startedAtMs: 100,
}),
});
await repository.complete(completionCommand());
const attemptUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."run_attempts"'),
);
assert.equal(attemptUpdate.params[9], 'timed_out');
assert.equal(attemptUpdate.params[17], 'EXECUTION_TIMED_OUT');
const stepUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."step_runs"'),
);
assert.equal(stepUpdate.params[0], 'timed_out');
assert.equal(stepUpdate.params[8], 'execution_timed_out');
const runUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."runs"'),
);
assert.equal(runUpdate.sql.includes('SET status ='), false);
});
test('replays a completed Workflow Task from immutable Attempt state after its parent Run is cancelled', async () => {
const command = completionCommand();
const ready = workflowReadyStep();
const running = transitionStepRunRecord(ready, {
expectedVersion: ready.version,
expectedDigest: ready.stepRunDigest,
mutationId: 'workflow-step-running',
to: 'running',
atMs: 100,
});
const succeeded = transitionStepRunRecord(running, {
expectedVersion: running.version,
expectedDigest: running.stepRunDigest,
mutationId: command.runEventId,
to: 'succeeded',
atMs: 1_000,
});
const aggregateRow = workflowAggregate(succeeded, {
runStatus: 'cancelled',
cancelRequestedAtMs: 1_200,
cancelReason: 'user',
runErrorCode: 'EXECUTION_CANCELLED',
attemptStatus: 'succeeded',
attemptLeaseVersion: 5,
callbackSequence: 1,
callbackTokenDigest: CALLBACK_DIGEST,
logArtifactId: LOG_ARTIFACT_ID,
startedAtMs: 100,
finishedAtMs: 1_000,
exitCode: 0,
});
const exact = fixture({
aggregate: aggregateRow,
stepRun: succeeded,
lease: lease({ leaseStatus: 'completed', leaseVersion: 5 }),
replayPayload: replayPayload(command, {
execution_scope: 'workflow_task',
step_run_id: succeeded.id,
}),
});
assert.equal(
(await exact.repository.complete(command)).status,
'already_completed',
);
});
test('accepts only an event-authenticated exact completed replay', async () => {
const command = completionCommand();
const terminal = aggregate({
runStatus: 'succeeded',
attemptStatus: 'succeeded',
attemptLeaseVersion: 5,
callbackSequence: 1,
callbackTokenDigest: CALLBACK_DIGEST,
logArtifactId: LOG_ARTIFACT_ID,
runStartedAtMs: 100,
startedAtMs: 100,
finishedAtMs: 1_000,
exitCode: 0,
});
const exact = fixture({
aggregate: terminal,
lease: lease({ leaseStatus: 'completed', leaseVersion: 5 }),
replayPayload: replayPayload(command),
});
assert.equal(
(await exact.repository.complete(command)).status,
'already_completed',
);
const drifted = fixture({
aggregate: terminal,
lease: lease({ leaseStatus: 'completed', leaseVersion: 5 }),
replayPayload: replayPayload(command, { artifact_sha256: 'd'.repeat(64) }),
});
await assert.rejects(
drifted.repository.complete(command),
(error) =>
error instanceof RemoteWorkerCompletionFenceRejectedError &&
error.reason === 'replay_mismatch',
);
const invalidOrigin = fixture({
aggregate: terminal,
lease: lease({ leaseStatus: 'completed', leaseVersion: 5 }),
replayPayload: replayPayload(command, { from_status: 'queued' }),
});
await assert.rejects(
invalidOrigin.repository.complete(command),
(error) =>
error instanceof RemoteWorkerCompletionFenceRejectedError &&
error.reason === 'replay_mismatch',
);
});
test('fences future Worker timestamps and rolls storage failures back', async () => {
const future = fixture();
await assert.rejects(
future.repository.complete(
completionCommand({
result: {
outcome: 'succeeded',
startedAtMs: 100,
finishedAtMs: 302_000,
exitCode: 0,
},
}),
),
(error) =>
error instanceof RemoteWorkerCompletionFenceRejectedError &&
error.reason === 'state_mismatch',
);
const failed = fixture({ failEvent: true });
await assert.rejects(
failed.repository.complete(completionCommand()),
(error) =>
error instanceof RemoteWorkerCompletionUnavailableError &&
error.cause?.message === 'injected event failure',
);
assert.equal(
failed.calls.some(({ sql }) => sql === 'ROLLBACK'),
true,
);
assert.equal(
failed.calls.some(({ sql }) => sql === 'COMMIT'),
false,
);
});
@@ -0,0 +1,324 @@
'use strict';
const assert = require('node:assert/strict');
const { createHash } = require('node:crypto');
const { test } = require('node:test');
const {
RemoteWorkerLeaseControlFenceRejectedError,
} = require('@qinglong/runtime-core/remote-worker-lease-control');
const {
createStepRunRecord,
transitionStepRunRecord,
} = require('@qinglong/runtime-core/step-run');
const {
PostgresRemoteWorkerLeaseControlRepository,
} = require('../dist/entrypoints/runtime');
const SESSION_ID = '018f5c64-9b9d-7f1a-8c2d-1234567890ac';
const LEASE_TOKEN = 'worker_generated_lease_capability_0000000000000001';
const LEASE_DIGEST = createHash('sha256').update(LEASE_TOKEN).digest('hex');
function command() {
return {
workerId: 'worker-1',
workerSessionId: SESSION_ID,
workerGeneration: 2,
projectId: 'project-1',
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
leaseGeneration: 3,
leaseToken: LEASE_TOKEN,
expectedLeaseVersion: 4,
leaseDurationMs: 30_000,
timeoutEventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a1',
};
}
function aggregate(overrides = {}) {
return {
runId: 'run-1',
projectId: 'project-1',
runStatus: 'running',
executionOwner: 'runtime',
cancelRequestedAtMs: null,
cancelReason: null,
runVersion: 3,
eventSequence: 7,
attemptId: 'attempt-1',
attemptRunId: 'run-1',
attemptStatus: 'running',
executorType: 'remote_worker',
attemptWorkerId: 'worker-1',
attemptWorkerSessionId: SESSION_ID,
attemptWorkerGeneration: 2,
attemptLeaseGeneration: 3,
attemptLeaseVersion: 4,
attemptLeaseTokenDigest: LEASE_DIGEST,
attemptOfferId: 'offer-1',
deadlineAtMs: null,
...overrides,
};
}
function lease(overrides = {}) {
return {
runId: 'run-1',
leaseStatus: 'leased',
leaseVersion: 4,
leaseGeneration: 3,
workerId: 'worker-1',
workerSessionId: SESSION_ID,
workerGeneration: 2,
leaseTokenDigest: LEASE_DIGEST,
offerId: 'offer-1',
leaseExpiresAtMs: 2_000,
...overrides,
};
}
function workflowSteps() {
const ready = createStepRunRecord({
id: 'workflow-step-1',
runId: 'run-1',
stepKey: 'collect',
kind: 'task',
definitionRef: 'pkg:demo:alpha',
definitionDigest: 'a'.repeat(64),
required: true,
initialStatus: 'ready',
mutationId: 'workflow-step-created',
createdAtMs: 100,
});
const running = transitionStepRunRecord(ready, {
expectedVersion: ready.version,
expectedDigest: ready.stepRunDigest,
mutationId: 'workflow-step-running',
to: 'running',
atMs: 200,
});
return { ready, running };
}
function fixture(options = {}) {
const calls = [];
const client = {
async query(sql, params = []) {
const normalized = sql.replace(/\s+/g, ' ').trim();
calls.push({ sql: normalized, params });
if (
normalized === 'BEGIN' || normalized === 'COMMIT' ||
normalized === 'ROLLBACK' || normalized.startsWith('SET LOCAL') ||
normalized.startsWith('SELECT pg_advisory_xact_lock')
) return { rows: [], rowCount: 0 };
if (normalized.includes('FROM "ql3"."worker_sessions"')) {
return {
rows: [options.worker ?? {
workerId: 'worker-1',
sessionId: SESSION_ID,
workerGeneration: 2,
workerStatus: 'online',
workerLeaseExpiresAtMs: 2_000,
}],
rowCount: 1,
};
}
if (normalized.includes('INNER JOIN "ql3"."run_attempts"')) {
return { rows: [options.aggregate ?? aggregate()], rowCount: 1 };
}
if (normalized.includes('FROM "ql3"."step_runs"')) {
return {
rows: options.stepRun
? [{
workflowStepVersion: options.stepRun.version,
workflowStepDigest: options.stepRun.stepRunDigest,
workflowStepJson: options.stepRun,
}]
: [],
rowCount: options.stepRun ? 1 : 0,
};
}
if (normalized.includes('FROM "ql3"."run_dispatch_leases"')) {
return { rows: [options.lease ?? lease()], rowCount: 1 };
}
if (normalized.includes('statement_timestamp()')) {
return { rows: [{ nowMs: options.nowMs ?? 1_000 }], rowCount: 1 };
}
if (
normalized.startsWith('SELECT created_at_ms') &&
normalized.includes('FROM "ql3"."run_events"')
) {
return {
rows: options.existingTimeoutAt === undefined
? []
: [{ createdAtMs: options.existingTimeoutAt }],
rowCount: options.existingTimeoutAt === undefined ? 0 : 1,
};
}
if (
normalized.startsWith('UPDATE "ql3"."run_dispatch_leases"') &&
normalized.includes('RETURNING')
) {
return {
rows: [{ renewedAtMs: options.nowMs ?? 1_000, expiresAtMs: 31_000 }],
rowCount: 1,
};
}
if (normalized.startsWith('UPDATE "ql3"."runs"')) {
return { rows: [], rowCount: options.runUpdateCount ?? 1 };
}
if (normalized.startsWith('UPDATE "ql3"."run_attempts"')) {
return { rows: [], rowCount: options.attemptUpdateCount ?? 1 };
}
if (normalized.startsWith('INSERT INTO "ql3"."run_events"')) {
return { rows: [], rowCount: 1 };
}
throw new Error(`Unexpected SQL: ${normalized}`);
},
release() { calls.push({ sql: 'RELEASE', params: [] }); },
};
return {
repository: new PostgresRemoteWorkerLeaseControlRepository({
async connect() { return client; },
}),
calls,
};
}
test('renews one exact live Lease without creating control intent', async () => {
const { repository, calls } = fixture();
assert.deepEqual(await repository.control(command()), {
status: 'renewed',
projectId: 'project-1',
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
leaseGeneration: 3,
leaseVersion: 5,
renewedAtMs: 1_000,
expiresAtMs: 31_000,
});
assert.equal(calls.some(({ sql }) =>
sql.startsWith('UPDATE "ql3"."runs"')), false);
assert.equal(calls.some(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."run_events"')), false);
assert.equal(JSON.stringify(calls).includes(LEASE_TOKEN), false);
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), true);
});
test('projects an existing durable cancellation while renewing', async () => {
const { repository } = fixture({
aggregate: aggregate({ cancelRequestedAtMs: 900, cancelReason: 'user' }),
});
const result = await repository.control(command());
assert.equal(result.status, 'stop_requested');
assert.deepEqual(result.stop, { reason: 'user', requestedAtMs: 900 });
assert.equal(result.leaseVersion, 5);
});
test('persists one due deadline as timeout intent before returning stop', async () => {
const { repository, calls } = fixture({
aggregate: aggregate({ deadlineAtMs: 950 }),
});
const result = await repository.control(command());
assert.equal(result.status, 'stop_requested');
assert.deepEqual(result.stop, { reason: 'timeout', requestedAtMs: 1_000 });
const cancel = calls.find(({ sql }) => sql.startsWith('UPDATE "ql3"."runs"'));
assert.deepEqual(cancel.params, ['run-1', 1_000, 4, 8, 3]);
const event = calls.find(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."run_events"'));
assert.equal(event.params[0], command().timeoutEventId);
assert.equal(event.params[3], 'remote-timeout:attempt-1:3');
});
test('stops one timed-out Workflow Task without cancelling its parent Run', async () => {
const { ready, running } = workflowSteps();
const { repository, calls } = fixture({
stepRun: running,
aggregate: aggregate({
deadlineAtMs: 950,
attemptStepRunId: running.id,
workflowAttemptId: 'attempt-1',
workflowStepRunId: running.id,
admittedWorkflowStepVersion: ready.version,
admittedWorkflowStepDigest: ready.stepRunDigest,
}),
});
const result = await repository.control(command());
assert.equal(result.status, 'stop_requested');
assert.deepEqual(result.stop, {
reason: 'timeout',
requestedAtMs: 1_000,
});
const runUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."runs"'));
assert.match(runUpdate.sql, /SET version = \$2, event_sequence = \$3/);
assert.equal(runUpdate.sql.includes('cancel_requested_at_ms ='), false);
assert.deepEqual(runUpdate.params, ['run-1', 4, 8, 3]);
const event = calls.find(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."run_events"'));
assert.match(event.sql, /workflow\.task_timeout_requested/);
assert.equal(event.params[5], running.id);
assert.match(event.params[6], /"execution_scope":"workflow_task"/);
});
test('returns an exact terminal projection without renewing', async () => {
const { repository, calls } = fixture({
aggregate: aggregate({
runStatus: 'cancelled',
attemptStatus: 'cancelled',
}),
lease: lease({ leaseStatus: 'completed', leaseVersion: 5 }),
});
assert.deepEqual(await repository.control(command()), {
status: 'terminal',
projectId: 'project-1',
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
leaseGeneration: 3,
terminalStatus: 'cancelled',
});
assert.equal(calls.some(({ sql }) =>
sql.startsWith('UPDATE "ql3"."run_dispatch_leases"')), false);
});
test('fences stale versions and rolls the transaction back', async () => {
const { repository, calls } = fixture({
lease: lease({ leaseVersion: 5 }),
});
await assert.rejects(
repository.control(command()),
(error) =>
error instanceof RemoteWorkerLeaseControlFenceRejectedError &&
error.reason === 'version_mismatch',
);
assert.equal(calls.some(({ sql }) => sql === 'ROLLBACK'), true);
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), false);
});
test('preserves granular identity fence reasons without exposing capability data', async () => {
const cases = [
[{ worker: { workerId: 'worker-1', sessionId: SESSION_ID,
workerGeneration: 3, workerStatus: 'online', workerLeaseExpiresAtMs: 2_000 } },
'worker_generation_mismatch'],
[{ aggregate: aggregate({ projectId: 'project-other' }) }, 'project_mismatch'],
[{ aggregate: aggregate({ executionOwner: 'legacy' }) }, 'execution_owner_mismatch'],
[{ lease: lease({ workerSessionId: '018f5c64-9b9d-7f1a-8c2d-1234567890ad' }) },
'worker_session_mismatch'],
[{ aggregate: aggregate({ attemptOfferId: 'offer-other' }) }, 'offer_mismatch'],
[{ lease: lease({ leaseTokenDigest: 'f'.repeat(64) }) }, 'lease_token_mismatch'],
];
for (const [options, reason] of cases) {
const { repository } = fixture(options);
await assert.rejects(
repository.control(command()),
(error) =>
error instanceof RemoteWorkerLeaseControlFenceRejectedError &&
error.reason === reason &&
!error.message.includes(LEASE_TOKEN),
);
}
});
@@ -0,0 +1,142 @@
'use strict';
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
createClusterTaskExecutionRevision,
} = require('@qinglong/runtime-core/cluster-execution-revision');
const {
digestRunDispatchLeaseToken,
} = require('@qinglong/runtime-core/run-dispatch-lease');
const { createSecretRef } = require('@qinglong/runtime-core/secret-reference');
const {
PostgresRemoteWorkerSecretDeliveryAuthorityRepository,
} = require('../dist/worker-credential/remoteWorkerSecretDeliveryRepository');
const SESSION_ID = '018f0000-0000-7000-8000-000000000001';
const SOURCE_DIGEST = 'a'.repeat(64);
const TASK_REVISION = `qltd:v1:1:${SOURCE_DIGEST}`;
const LEASE_TOKEN = 'worker_generated_lease_capability_0000000000000001';
const SECRET_REF = createSecretRef({ projectId: 'project-1', name: 'token' });
function revision() {
return createClusterTaskExecutionRevision({
projectId: 'project-1', taskId: 'task-1', taskRevision: TASK_REVISION,
sourceRevision: 1, sourceContentDigest: SOURCE_DIGEST,
executorType: 'remote_worker', planSchema: 'qinglong/command-execution@v1',
command: { kind: 'argv', file: '/bin/true', args: [] },
environment: [{ name: 'TOKEN', kind: 'secret', secretRef: SECRET_REF }],
createdAtMs: 1,
});
}
function command(executionDigest, overrides = {}) {
return {
workerId: 'edge-1', workerSessionId: SESSION_ID, workerGeneration: 2,
runId: 'run-1', attemptId: 'attempt-1', projectId: 'project-1',
taskId: 'task-1', taskRevision: TASK_REVISION, executionDigest,
offerId: 'offer-1', leaseGeneration: 3, leaseToken: LEASE_TOKEN,
expectedLeaseVersion: 4, secretRefs: [SECRET_REF], ...overrides,
};
}
function authorityRow(plan, overrides = {}) {
return {
observedAtMs: '1000', runId: 'run-1', runProjectId: 'project-1',
runTaskId: 'task-1', runTaskRevision: TASK_REVISION,
runStatus: 'dispatching', executionOwner: 'runtime',
cancelRequestedAtMs: null, attemptStatus: 'starting',
attemptExecutorType: 'remote_worker', attemptWorkerId: 'edge-1',
attemptWorkerSessionId: SESSION_ID, attemptWorkerGeneration: 2,
attemptLeaseTokenDigest: digestRunDispatchLeaseToken(LEASE_TOKEN),
attemptLeaseGeneration: 3, attemptLeaseVersion: 4,
attemptOfferId: 'offer-1', sessionId: SESSION_ID, sessionGeneration: 2,
sessionStatus: 'online', sessionExpiresAtMs: '5000',
leaseRunId: 'run-1', leaseStatus: 'leased', leaseVersion: 4,
leaseGeneration: 3, leaseWorkerId: 'edge-1',
leaseWorkerSessionId: SESSION_ID, leaseWorkerGeneration: 2,
leaseTokenDigest: digestRunDispatchLeaseToken(LEASE_TOKEN),
leaseOfferId: 'offer-1', leaseExpiresAtMs: '5000',
revisionProjectId: plan.projectId, revisionTaskId: plan.taskId,
sourceRevision: plan.sourceRevision,
revisionTaskRevision: plan.taskRevision,
sourceContentDigest: plan.sourceContentDigest,
revisionExecutorType: plan.executorType, planSchema: plan.planSchema,
planJson: {
command: plan.command,
environment: plan.environment,
...(plan.workingDirectory === undefined ? {} : { workingDirectory: plan.workingDirectory }),
...(plan.timeoutMs === undefined ? {} : { timeoutMs: plan.timeoutMs }),
...(plan.placement === undefined ? {} : { placement: plan.placement }),
},
revisionContentDigest: plan.contentDigest,
revisionCreatedAtMs: String(plan.createdAtMs),
...overrides,
};
}
function fixture(row) {
const queries = [];
let released = 0;
const client = {
async query(sql, params) {
queries.push({ sql, params });
if (sql.includes('FROM observation')) return { rows: row ? [row] : [] };
return { rows: [] };
},
release() { released += 1; },
};
return {
repository: new PostgresRemoteWorkerSecretDeliveryAuthorityRepository({
async connect() { return client; },
}),
queries,
released: () => released,
};
}
test('authorizes exact Session, Lease and immutable execution revision fences', async () => {
const plan = revision();
const { repository, queries, released } = fixture(authorityRow(plan));
const result = await repository.authorize(command(plan.contentDigest));
assert.deepEqual(result.secretRefs, [SECRET_REF]);
assert.equal(result.executionDigest, plan.contentDigest);
assert.equal('leaseToken' in result, false);
assert.equal(queries.some(({ sql }) => sql.includes('pg_advisory_xact_lock')), true);
assert.equal(queries.at(-1).sql, 'COMMIT');
assert.equal(released(), 1);
});
test('rejects expired or replayed Lease authority before returning Secret scope', async () => {
const plan = revision();
for (const overrides of [
{ leaseVersion: 5 },
{ leaseExpiresAtMs: '1000' },
{ sessionExpiresAtMs: '1000' },
{ attemptStatus: 'running' },
]) {
const { repository, queries } = fixture(authorityRow(plan, overrides));
await assert.rejects(
repository.authorize(command(plan.contentDigest)),
/authority_mismatch/,
);
assert.equal(queries.at(-1).sql, 'ROLLBACK');
}
});
test('rejects partial Secret scope and execution digest drift', async () => {
const plan = revision();
const extra = createSecretRef({ projectId: 'project-1', name: 'other' });
const partial = fixture(authorityRow(plan));
await assert.rejects(
partial.repository.authorize(command(plan.contentDigest, {
secretRefs: [extra],
})),
/secret_scope_mismatch/,
);
const digestDrift = fixture(authorityRow(plan));
await assert.rejects(
digestDrift.repository.authorize(command('b'.repeat(64))),
/secret_scope_mismatch/,
);
});
@@ -0,0 +1,221 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
DuplicateIdempotencyKeyError,
MAX_RUN_EVENT_PAYLOAD_BYTES,
RunEventPayloadTooLargeError,
RunRepositoryBusyError,
RunRepositoryConstraintError,
} = require('@qinglong/runtime-core');
const { PostgresRunRepository, PostgresRunTransaction } = require('../dist');
const RUN = Object.freeze({
id: '019f70b0-0000-7000-8000-000000000001',
projectId: 'default',
taskId: 'task-1',
taskRevision: 'revision-1',
taskName: 'test',
triggerType: 'manual',
executionOrigin: 'manual',
executionOwner: 'runtime',
triggeredBy: 'user:1',
status: 'created',
version: 0,
eventSequence: 0,
priority: 0,
idempotencyKey: 'request-1',
createdAtMs: 1_750_000_000_000,
});
const EVENT = Object.freeze({
id: '019f70b0-0000-7000-8000-000000000003',
runId: RUN.id,
sequence: 1,
type: 'run.created',
dedupeKey: 'run.created',
actorType: 'system',
payload: Object.freeze({ source: 'postgres-test' }),
createdAtMs: 1_750_000_000_002,
});
function driverError(code, constraint) {
return Object.assign(new Error('driver failure'), { code, constraint });
}
function clientHarness(handler = async () => ({ rows: [], rowCount: 0 })) {
const queries = [];
let released = 0;
const client = {
async query(text, values) {
queries.push({ text, values });
return handler(text, values, queries.length);
},
release() {
released += 1;
},
};
return {
queries,
released: () => released,
pool: {
query: (...args) => client.query(...args),
async connect() {
return client;
},
},
};
}
function databaseRunRow(overrides = {}) {
return {
...RUN,
createdAtMs: String(RUN.createdAtMs),
taskSnapshotRef: null,
legacyCronId: null,
parentRunId: null,
retryOfRunId: null,
triggerId: null,
requestId: null,
scheduledForMs: null,
queuedAtMs: null,
startedAtMs: null,
finishedAtMs: null,
cancelRequestedAtMs: null,
cancelReason: null,
inputRef: null,
outputRef: null,
errorCode: null,
errorSummary: null,
...overrides,
};
}
test('orders one bounded PostgreSQL transaction and releases its client', async () => {
const harness = clientHarness();
const result = await new PostgresRunRepository(harness.pool).transaction(
async () => 'committed',
);
assert.equal(result, 'committed');
assert.deepEqual(
harness.queries.map(({ text }) => text),
[
'BEGIN',
'SET TRANSACTION ISOLATION LEVEL READ COMMITTED',
"SELECT set_config('statement_timeout', $1, true)",
"SELECT set_config('lock_timeout', $1, true)",
"SELECT set_config('idle_in_transaction_session_timeout', $1, true)",
'COMMIT',
],
);
assert.equal(harness.released(), 1);
});
test('rolls back a serialization failure and exposes a stable domain error', async () => {
const harness = clientHarness(async (text) => {
if (text === 'COMMIT') throw driverError('40001');
return { rows: [], rowCount: 0 };
});
await assert.rejects(
new PostgresRunRepository(harness.pool).transaction(async () => 1),
RunRepositoryBusyError,
);
assert.equal(harness.queries.at(-1).text, 'ROLLBACK');
assert.equal(harness.released(), 1);
});
test('normalizes bigint rows and treats missing rows as null', async () => {
const row = databaseRunRow({ scheduledForMs: '1750000000100' });
const harness = clientHarness(async (text, values) => {
if (!text.includes('FROM "ql3"."runs"')) {
throw new Error(`unexpected query: ${text}`);
}
return values[0] === RUN.id
? { rows: [row], rowCount: 1 }
: { rows: [], rowCount: 0 };
});
const repository = new PostgresRunRepository(harness.pool);
assert.deepEqual(await repository.findRunById(RUN.id), {
...RUN,
scheduledForMs: 1_750_000_000_100,
});
assert.equal(await repository.findRunById('missing'), null);
});
test('lists one Project with the indexed descending Run keyset', async () => {
const harness = clientHarness(async () => ({
rows: [
databaseRunRow({ id: 'run-c', createdAtMs: '20' }),
databaseRunRow({ id: 'run-b', createdAtMs: '20' }),
],
rowCount: 2,
}));
const repository = new PostgresRunRepository(harness.pool);
const rows = await repository.listRunsByProject({
projectId: 'default',
limit: 2,
after: { createdAtMs: 30, runId: 'run-z' },
});
assert.deepEqual(
rows.map(({ id }) => id),
['run-c', 'run-b'],
);
assert.deepEqual(harness.queries[0].values, ['default', 'run-z', 30, 2]);
assert.match(
harness.queries[0].text,
/ORDER BY "created_at_ms" DESC, "id" DESC\s+LIMIT \$4/u,
);
await assert.rejects(
repository.listRunsByProject({ projectId: 'default', limit: 66 }),
TypeError,
);
});
test('rejects duplicate identity rows and corrupt enum data', async () => {
const duplicate = clientHarness(async () => ({
rows: [databaseRunRow(), databaseRunRow()],
rowCount: 2,
}));
await assert.rejects(
new PostgresRunRepository(duplicate.pool).findRunById(RUN.id),
RunRepositoryConstraintError,
);
const corrupt = clientHarness(async () => ({
rows: [databaseRunRow({ status: 'invented' })],
rowCount: 1,
}));
await assert.rejects(
new PostgresRunRepository(corrupt.pool).findRunById(RUN.id),
RunRepositoryConstraintError,
);
});
test('maps PostgreSQL constraint names without leaking driver errors', async () => {
const transaction = new PostgresRunTransaction({
async query() {
throw driverError('23505', 'ql3_runs_project_idempotency_uidx');
},
});
await assert.rejects(
transaction.insertRun(RUN),
DuplicateIdempotencyKeyError,
);
});
test('rejects oversized event payloads before issuing SQL', async () => {
let calls = 0;
const transaction = new PostgresRunTransaction({
async query() {
calls += 1;
return { rows: [], rowCount: 0 };
},
});
await assert.rejects(
transaction.appendEvent({
...EVENT,
payload: { value: 'x'.repeat(MAX_RUN_EVENT_PAYLOAD_BYTES) },
}),
RunEventPayloadTooLargeError,
);
assert.equal(calls, 0);
});
@@ -0,0 +1,67 @@
require('ts-node/register/transpile-only');
const assert = require('node:assert/strict');
const { test } = require('node:test');
const { getTableConfig } = require('drizzle-orm/pg-core');
const {
postgresqlControlSchemaContract,
ql3PostgresTables,
} = require('../dist');
function describeTable(table) {
const config = getTableConfig(table);
const primaryIndexes = config.columns
.filter((column) => column.primary)
.map(() => `${config.name}_pkey`);
return {
name: config.name,
schema: config.schema,
columns: config.columns.map((column) => column.name),
indexes: [
...primaryIndexes,
...config.primaryKeys.map((entry) => entry.getName()),
...config.indexes.map((entry) => entry.config.name),
],
checks: config.checks.map((entry) => entry.name),
foreignKeys: config.foreignKeys.map((entry) => entry.getName()),
};
}
test('Drizzle schema exactly matches the reviewed ql3 table and index contract', () => {
const drizzleTables = ql3PostgresTables.map(describeTable);
assert.deepEqual(
drizzleTables.map(({ name, schema, columns }) => ({
name,
schema,
columns,
})),
postgresqlControlSchemaContract.tables.map(({ name, columns }) => ({
name,
schema: postgresqlControlSchemaContract.schema,
columns: [...columns],
})),
);
assert.deepEqual(
drizzleTables.flatMap(({ indexes }) => indexes).sort(),
[...postgresqlControlSchemaContract.indexes].sort(),
);
});
test('Drizzle schema carries every named ql3 check and foreign-key boundary', () => {
const drizzleTables = ql3PostgresTables.map(describeTable);
const checks = drizzleTables.flatMap((table) => table.checks).sort();
const foreignKeys = drizzleTables
.flatMap((table) => table.foreignKeys)
.sort();
assert.deepEqual(checks, [...postgresqlControlSchemaContract.checks].sort());
assert.deepEqual(
foreignKeys,
[...postgresqlControlSchemaContract.foreignKeys].sort(),
);
assert.equal(
drizzleTables.some(({ name }) =>
['crontabs', 'envs', 'subscriptions', 'runninginstances'].includes(name),
),
false,
);
});
@@ -0,0 +1,77 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
SecurityAuditUnavailableError,
} = require('@qinglong/runtime-core/security-audit');
const {
PostgresSecurityAuditRepository,
} = require('@qinglong/cluster-postgres/runtime');
function record(overrides = {}) {
return {
eventId: '123e4567-e89b-42d3-a456-426614174000',
requestId: 'request-1',
operationId: 'run.create',
projectId: 'default',
subject: { type: 'user', id: 'usr_primary' },
authenticationId: 'api_credential:primary:1',
outcome: 'allowed',
reasons: ['role_grant'],
fence: { projectVersion: 2, bindingVersion: 3 },
occurredAtMs: 1000,
...overrides,
};
}
test('inserts one normalized low-sensitive audit fact without reading it back', async () => {
const calls = [];
const repository = new PostgresSecurityAuditRepository({
async query(sql, values) {
calls.push({ sql, values });
return { rows: [] };
},
});
await repository.record(record());
assert.equal(calls.length, 1);
assert.match(calls[0].sql, /^INSERT INTO "ql3"\."security_audit_events"/);
assert.deepEqual(calls[0].values, [
'123e4567-e89b-42d3-a456-426614174000',
'request-1',
'run.create',
'default',
'user',
'usr_primary',
'api_credential:primary:1',
'allowed',
'["role_grant"]',
2,
3,
1000,
]);
assert.equal(JSON.stringify(calls).includes('secret'), false);
});
test('maps invalid facts and database failures to low-sensitive unavailable', async () => {
let calls = 0;
const invalid = new PostgresSecurityAuditRepository({
async query() {
calls += 1;
return { rows: [] };
},
});
await assert.rejects(
invalid.record(record({ reasons: ['driver password leaked'] })),
SecurityAuditUnavailableError,
);
assert.equal(calls, 0);
const unavailable = new PostgresSecurityAuditRepository({
async query() {
throw new Error('driver detail');
},
});
await assert.rejects(
unavailable.record(record()),
SecurityAuditUnavailableError,
);
});
@@ -0,0 +1,308 @@
'use strict';
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
TaskStartFenceRejectedError,
TaskStartNotFoundError,
} = require('@qinglong/runtime-core/task-start');
const {
createTaskDefinitionRecord,
} = require('@qinglong/runtime-core/task-definition');
const {
createBuiltInTaskSpecSemanticRegistry,
} = require('@qinglong/runtime-core/task-spec-semantic');
const {
compileClusterCommandTaskDefinition,
} = require('@qinglong/runtime-core/cluster-execution-revision');
const {
PostgresTaskStartRepository,
} = require('@qinglong/cluster-postgres/task-start');
const IDS = [
'019f7300-0000-7000-8000-000000000801',
'019f7300-0000-7000-8000-000000000802',
'019f7300-0000-7000-8000-000000000803',
'019f7300-0000-7000-8000-000000000804',
];
const MUTATION_ID = '019f7300-0000-7000-8000-000000000800';
const TASK_SEMANTICS = createBuiltInTaskSpecSemanticRegistry();
const TASK_COMMAND = {
projectId: 'project-1',
taskId: 'task-1',
expectedRevision: null,
mutationId: '019f7300-0000-7000-8000-000000000899',
name: 'Task 1',
kind: 'command',
spec: {
schema: 'qinglong/command@v1',
config: {
command: { kind: 'argv', file: '/bin/echo', args: ['cluster'] },
},
},
labels: {},
enabled: true,
occurredAtMs: 20,
};
const DEFINITION = createTaskDefinitionRecord({
...TASK_COMMAND,
spec: TASK_SEMANTICS.normalize({
projectId: TASK_COMMAND.projectId,
taskId: TASK_COMMAND.taskId,
kind: TASK_COMMAND.kind,
spec: TASK_COMMAND.spec,
}),
}, 10);
const DISABLED_DEFINITION = createTaskDefinitionRecord({
...TASK_COMMAND,
enabled: false,
spec: TASK_SEMANTICS.normalize({
projectId: TASK_COMMAND.projectId,
taskId: TASK_COMMAND.taskId,
kind: TASK_COMMAND.kind,
spec: TASK_COMMAND.spec,
}),
}, 10);
const EXECUTION = compileClusterCommandTaskDefinition(
DEFINITION,
TASK_SEMANTICS,
);
function command(overrides = {}) {
return {
projectId: 'project-1',
taskId: 'task-1',
mutationId: MUTATION_ID,
expectedRevision: DEFINITION.revision,
expectedContentDigest: DEFINITION.contentDigest,
runId: IDS[0],
attemptId: IDS[1],
createdEventId: IDS[2],
queuedEventId: IDS[3],
subject: { type: 'user', id: 'user-1' },
policyFence: { projectVersion: 2, bindingVersion: 3 },
...overrides,
};
}
function taskRow(overrides = {}) {
return {
projectId: DEFINITION.projectId,
taskId: DEFINITION.taskId,
taskRevision: DEFINITION.revision,
definitionMutationId: DEFINITION.mutationId,
taskName: DEFINITION.name,
description: null,
taskKind: DEFINITION.kind,
specJson: DEFINITION.spec,
labelsJson: DEFINITION.labels,
enabled: DEFINITION.enabled,
taskContentDigest: DEFINITION.contentDigest,
taskCreatedAtMs: DEFINITION.createdAtMs,
taskUpdatedAtMs: DEFINITION.updatedAtMs,
...overrides,
};
}
function executionRow() {
return {
projectId: EXECUTION.projectId,
taskId: EXECUTION.taskId,
sourceRevision: EXECUTION.sourceRevision,
taskRevision: EXECUTION.taskRevision,
sourceContentDigest: EXECUTION.sourceContentDigest,
executorType: EXECUTION.executorType,
planSchema: EXECUTION.planSchema,
planJson: {
command: EXECUTION.command,
environment: EXECUTION.environment,
placement: EXECUTION.placement,
},
contentDigest: EXECUTION.contentDigest,
createdAtMs: EXECUTION.createdAtMs,
};
}
function fixture(options = {}) {
const calls = [];
const client = {
async query(sql, params = []) {
const normalized = sql.replace(/\s+/g, ' ').trim();
calls.push({ sql: normalized, params });
if (
normalized.startsWith('BEGIN') ||
normalized === 'COMMIT' ||
normalized === 'ROLLBACK' ||
normalized.startsWith('SELECT set_config') ||
normalized.startsWith('INSERT INTO')
) return { rows: [], rowCount: normalized.startsWith('INSERT') ? 1 : 0 };
if (normalized.includes('FROM "ql3"."projects"')) {
const rows = options.projectRows ?? [{
projectStatus: 'active',
projectVersion: 2,
}];
return { rows, rowCount: rows.length };
}
if (normalized.includes('FROM "ql3"."project_role_bindings"')) {
const rows = options.bindingRows ?? [{
bindingVersion: 3,
bindingState: 'active',
bindingRole: 'operator',
}];
return { rows, rowCount: rows.length };
}
if (normalized.includes('FROM "ql3"."runs"')) {
const rows = options.runRows ?? [];
return { rows, rowCount: rows.length };
}
if (normalized.includes('FROM "ql3"."task_definitions"')) {
const rows = options.taskRows ?? [taskRow()];
return { rows, rowCount: rows.length };
}
if (normalized.includes('FROM "ql3"."task_execution_revisions"')) {
const rows = options.executionRows ?? [executionRow()];
return { rows, rowCount: rows.length };
}
if (normalized.includes('statement_timestamp()')) {
return { rows: [{ nowMs: 1_000 }], rowCount: 1 };
}
if (normalized.includes('FROM "ql3"."run_attempts"')) {
const rows = options.attemptRows ?? [];
return { rows, rowCount: rows.length };
}
if (normalized.includes('FROM "ql3"."run_events"')) {
const rows = options.eventRows ?? [];
return { rows, rowCount: rows.length };
}
throw new Error(`Unexpected SQL: ${normalized}`);
},
release() { calls.push({ sql: 'RELEASE', params: [] }); },
};
return {
repository: new PostgresTaskStartRepository({
async connect() { return client; },
}),
calls,
};
}
test('revalidates Policy and Task/execution digests before one atomic Run aggregate', async () => {
const { repository, calls } = fixture();
assert.deepEqual(await repository.startTask(command()), {
status: 'accepted',
projectId: 'project-1',
taskId: 'task-1',
taskRevision: 1,
taskContentDigest: DEFINITION.contentDigest,
runId: IDS[0],
attemptId: IDS[1],
runStatus: 'queued',
runVersion: 2,
eventSequence: 2,
executorType: 'remote_worker',
executionRevisionDigest: EXECUTION.contentDigest,
createdAtMs: 1_000,
});
const project = calls.findIndex(({ sql }) => sql.includes('FROM "ql3"."projects"'));
const binding = calls.findIndex(({ sql }) => sql.includes('project_role_bindings'));
const task = calls.findIndex(({ sql }) => sql.includes('task_definitions'));
const execution = calls.findIndex(({ sql }) => sql.includes('task_execution_revisions'));
assert.ok(project < binding && binding < task && task < execution);
assert.equal(calls.filter(({ sql }) => sql.startsWith('INSERT INTO')).length, 4);
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), true);
});
test('returns the original durable identities for an exact replay', async () => {
const createdPayload = {
status: 'created',
version: 1,
execution_owner: 'runtime',
executor_type: 'remote_worker',
execution_revision_digest: EXECUTION.contentDigest,
task_revision: 1,
task_content_digest: DEFINITION.contentDigest,
mutation_id: MUTATION_ID,
policy_fence: { project_version: 2, binding_version: 3 },
};
const { repository, calls } = fixture({
runRows: [{
runId: IDS[0],
projectId: 'project-1',
taskId: 'task-1',
taskRevisionRef: EXECUTION.taskRevision,
triggerType: 'task_start',
executionOrigin: 'manual',
executionOwner: 'runtime',
triggeredBy: 'user-1',
requestId: MUTATION_ID,
priority: 0,
createdAtMs: 1_000,
}],
attemptRows: [{ attemptId: IDS[1], executorType: 'remote_worker' }],
eventRows: [
{
sequence: 1,
type: 'run.created',
actorType: 'user',
actorId: 'user-1',
payload: createdPayload,
createdAtMs: 1_000,
},
{
sequence: 2,
type: 'run.queued',
actorType: 'user',
actorId: 'user-1',
payload: { from_status: 'created', to_status: 'queued', version: 2 },
createdAtMs: 1_000,
},
],
});
const replay = await repository.startTask(command({
runId: '019f7300-0000-7000-8000-000000000901',
attemptId: '019f7300-0000-7000-8000-000000000902',
createdEventId: '019f7300-0000-7000-8000-000000000903',
queuedEventId: '019f7300-0000-7000-8000-000000000904',
}));
assert.equal(replay.status, 'existing');
assert.equal(replay.runId, IDS[0]);
assert.equal(replay.attemptId, IDS[1]);
assert.equal(calls.some(({ sql }) => sql.includes('task_definitions')), false);
assert.equal(calls.some(({ sql }) => sql.startsWith('INSERT INTO')), false);
});
test('rejects missing, authorization, definition and disabled fences', async () => {
await assert.rejects(
fixture({ projectRows: [] }).repository.startTask(command()),
TaskStartNotFoundError,
);
await assert.rejects(
fixture({ bindingRows: [{
bindingVersion: 4,
bindingState: 'revoked',
bindingRole: null,
}] }).repository.startTask(command()),
(error) =>
error instanceof TaskStartFenceRejectedError &&
error.reason === 'authorization_changed',
);
await assert.rejects(
fixture().repository.startTask(command({ expectedRevision: 2 })),
(error) =>
error instanceof TaskStartFenceRejectedError &&
error.reason === 'definition_changed',
);
await assert.rejects(
fixture({
taskRows: [taskRow({
enabled: false,
taskContentDigest: DISABLED_DEFINITION.contentDigest,
})],
}).repository.startTask(command({
expectedContentDigest: DISABLED_DEFINITION.contentDigest,
})),
(error) =>
error instanceof TaskStartFenceRejectedError &&
error.reason === 'task_disabled',
);
});
@@ -0,0 +1,311 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
createToolExecutionEvidenceBundle,
ToolExecutionEvidenceConflictError,
ToolExecutionEvidenceUnavailableError,
} = require('@qinglong/runtime-core/tool-execution-evidence');
const {
PostgresToolExecutionEvidenceRepository,
} = require('@qinglong/cluster-postgres/tool-execution-evidence');
function evidence(overrides = {}) {
const createdAtMs = overrides.createdAtMs ?? 1700000000000;
return createToolExecutionEvidenceBundle({
traceId: overrides.traceId ?? '1'.repeat(32),
spanId: overrides.spanId ?? '2'.repeat(16),
projectId: 'project-1',
runId: 'run-1',
stepRunId: 'step-1',
invocationPlanDigest: '3'.repeat(64),
bindingDigest: '4'.repeat(64),
adapterDigest: '5'.repeat(64),
redactionContractDigest: '6'.repeat(64),
auditContractDigest: '7'.repeat(64),
audit: {
eventId:
overrides.eventId ?? '12345678-1234-4234-9234-123456789abc',
requestId: 'request-1',
operationId: 'tool.invoke.start',
projectId: 'project-1',
subject: { type: 'user', id: 'user-1' },
authenticationId: 'authentication-1',
outcome: 'allowed',
reasons: ['policy_allowed'],
fence: { projectVersion: 4, bindingVersion: 7 },
occurredAtMs: createdAtMs,
},
createdAtMs,
});
}
function row(bundle, overrides = {}) {
return {
traceJson: bundle.trace,
auditJson: bundle.audit,
receiptJson: bundle.receipt,
storedTraceId: bundle.trace.traceId,
storedSpanId: bundle.trace.spanId,
storedParentSpanId: bundle.trace.parentSpanId,
traceProjectId: bundle.trace.projectId,
traceRunId: bundle.trace.runId,
traceStepRunId: bundle.trace.stepRunId,
traceInvocationPlanDigest: bundle.trace.invocationPlanDigest,
traceBindingDigest: bundle.trace.bindingDigest,
storedAdapterDigest: bundle.trace.adapterDigest,
storedRedactionContractDigest: bundle.trace.redactionContractDigest,
storedAuditContractDigest: bundle.trace.auditContractDigest,
traceCreatedAtMs: String(bundle.trace.createdAtMs),
storedTraceDigest: bundle.trace.traceDigest,
storedEventId: bundle.receipt.eventId,
receiptProjectId: bundle.receipt.projectId,
receiptRunId: bundle.receipt.runId,
receiptStepRunId: bundle.receipt.stepRunId,
receiptTraceId: bundle.receipt.traceId,
receiptSpanId: bundle.receipt.spanId,
receiptTraceDigest: bundle.receipt.traceDigest,
receiptInvocationPlanDigest: bundle.receipt.invocationPlanDigest,
receiptBindingDigest: bundle.receipt.bindingDigest,
storedAuditRecordDigest: bundle.receipt.auditRecordDigest,
receiptCreatedAtMs: String(bundle.receipt.createdAtMs),
storedReceiptDigest: bundle.receipt.receiptDigest,
...overrides,
};
}
function clientWith(handler) {
const calls = [];
let released = false;
return {
calls,
get released() {
return released;
},
async query(text, values = []) {
calls.push({ text, values });
if (
text.startsWith('BEGIN') ||
text.includes("set_config('") ||
text === 'COMMIT' ||
text === 'ROLLBACK'
) {
return { rows: [] };
}
return handler(text, values, calls);
},
release() {
released = true;
},
};
}
test('prepares audit, trace and receipt in one serializable transaction', async () => {
const bundle = evidence();
const client = clientWith(async (sql) => {
if (sql.includes('WHERE receipt.event_id')) return { rows: [] };
if (sql.includes('FROM "ql3"."step_runs" AS step')) {
return {
rows: [{ kind: 'tool', status: 'ready', projectId: 'project-1' }],
};
}
if (sql.startsWith('INSERT INTO')) return { rows: [], rowCount: 1 };
throw new Error(`unexpected SQL: ${sql}`);
});
const pool = {
async query() {
throw new Error('pool query is not used by prepare');
},
async connect() {
return client;
},
};
const repository = new PostgresToolExecutionEvidenceRepository(pool);
assert.deepEqual(await repository.prepare(bundle), {
status: 'created',
bundle,
});
assert.equal(client.released, true);
assert.deepEqual(
client.calls
.map(({ text }) => text)
.filter((sql) => sql.startsWith('INSERT INTO'))
.map((sql) => sql.match(/"ql3"\."([^"]+)"/)[1]),
[
'security_audit_events',
'tool_execution_trace_anchors',
'tool_execution_audit_receipts',
],
);
assert.equal(
client.calls.some(({ text }) => text === 'COMMIT'),
true,
);
assert.equal(
client.calls.some(({ text }) => text === 'ROLLBACK'),
false,
);
});
test('replays exact evidence and rejects identity reuse', async () => {
const bundle = evidence();
const stored = row(bundle);
const exactClient = clientWith(async (sql) => {
if (sql.includes('WHERE receipt.event_id')) return { rows: [stored] };
throw new Error(`unexpected SQL: ${sql}`);
});
const exact = new PostgresToolExecutionEvidenceRepository({
async query() {
throw new Error('unused');
},
async connect() {
return exactClient;
},
});
assert.deepEqual(await exact.prepare(bundle), {
status: 'existing',
bundle,
});
const changed = evidence({
createdAtMs: bundle.trace.createdAtMs + 1,
traceId: bundle.trace.traceId,
spanId: bundle.trace.spanId,
eventId: bundle.audit.eventId,
});
const conflictClient = clientWith(async (sql) => {
if (sql.includes('WHERE receipt.event_id')) return { rows: [stored] };
throw new Error(`unexpected SQL: ${sql}`);
});
const conflict = new PostgresToolExecutionEvidenceRepository({
async query() {
throw new Error('unused');
},
async connect() {
return conflictClient;
},
});
await assert.rejects(
conflict.prepare(changed),
ToolExecutionEvidenceConflictError,
);
assert.equal(
conflictClient.calls.some(({ text }) => text === 'ROLLBACK'),
true,
);
assert.equal(conflictClient.released, true);
});
test('reads bounded evidence without SELECT authority on the audit table', async () => {
const first = evidence();
const second = evidence({
createdAtMs: first.trace.createdAtMs + 1,
traceId: '8'.repeat(32),
spanId: '9'.repeat(16),
eventId: '87654321-4321-4321-8321-cba987654321',
});
const calls = [];
const repository = new PostgresToolExecutionEvidenceRepository({
async query(text, values) {
calls.push({ text, values });
if (text.includes('WHERE trace.trace_id')) {
return { rows: [row(first)] };
}
if (text.includes('WHERE receipt.event_id')) {
return { rows: [row(first)] };
}
if (text.includes('WHERE trace.run_id')) {
return { rows: [row(first), row(second)] };
}
throw new Error(`unexpected SQL: ${text}`);
},
async connect() {
throw new Error('unused');
},
});
assert.deepEqual(
await repository.findByTrace(first.trace.traceId, first.trace.spanId),
first,
);
assert.deepEqual(
await repository.findByAuditEventId(first.audit.eventId),
first,
);
assert.deepEqual(
await repository.listByRun({ runId: 'run-1', limit: 1 }),
{
bundles: [first],
truncated: true,
next: {
createdAtMs: first.trace.createdAtMs,
traceId: first.trace.traceId,
spanId: first.trace.spanId,
},
},
);
assert.equal(
calls.some(({ text }) =>
text.includes('JOIN "ql3"."security_audit_events"'),
),
false,
);
assert.deepEqual(calls.at(-1).values, ['run-1', null, 0, '', 2]);
});
test('rejects invalid lookups before SQL and fails closed on corrupt rows', async () => {
let queries = 0;
const repository = new PostgresToolExecutionEvidenceRepository({
async query() {
queries += 1;
return {
rows: [row(evidence(), { storedTraceDigest: '0'.repeat(64) })],
};
},
async connect() {
throw new Error('unused');
},
});
await assert.rejects(repository.findByTrace('bad', 'also-bad'), {
code: 'TOOL_EXECUTION_EVIDENCE_INVALID',
});
assert.equal(queries, 0);
await assert.rejects(
repository.findByTrace('1'.repeat(32), '2'.repeat(16)),
ToolExecutionEvidenceUnavailableError,
);
});
test('rolls back when the StepRun is not an admitted same-Project Tool step', async () => {
const bundle = evidence();
const client = clientWith(async (sql) => {
if (sql.includes('WHERE receipt.event_id')) return { rows: [] };
if (sql.includes('FROM "ql3"."step_runs" AS step')) {
return {
rows: [{ kind: 'task', status: 'ready', projectId: 'project-1' }],
};
}
throw new Error(`unexpected SQL: ${sql}`);
});
const repository = new PostgresToolExecutionEvidenceRepository({
async query() {
throw new Error('unused');
},
async connect() {
return client;
},
});
await assert.rejects(
repository.prepare(bundle),
ToolExecutionEvidenceConflictError,
);
assert.equal(
client.calls.some(({ text }) => text.startsWith('INSERT INTO')),
false,
);
assert.equal(
client.calls.some(({ text }) => text === 'ROLLBACK'),
true,
);
assert.equal(client.released, true);
});
@@ -0,0 +1,844 @@
const assert = require('node:assert/strict');
const { createHash } = require('node:crypto');
const { test } = require('node:test');
const {
createPluginPackageResourceGenerationFromReferences,
} = require('@qinglong/runtime-core/plugin-package-resource-generation');
const {
createProjectToolDefinitionSnapshot,
projectToolDefinitionRegistry,
} = require('@qinglong/runtime-core/project-tool-definition-snapshot');
const {
createStepRunMutation,
transitionStepRunMutation,
} = require('@qinglong/runtime-core/step-run');
const {
ToolExecutionStartBarrierConflictError,
ToolExecutionStartBarrierUnavailableError,
createToolExecutionStartCommand,
toolExecutionStartBarrierRecord,
} = require('@qinglong/runtime-core/tool-execution-start-barrier');
const {
TOOL_EXECUTION_START_AUDIT_OPERATION,
createToolExecutionEvidenceBundle,
toolExecutionAdmissionEvidence,
} = require('@qinglong/runtime-core/tool-execution-evidence');
const {
TrustedToolHandlerBindingRegistry,
admitTrustedToolExecution,
createTrustedToolHandlerBinding,
createTrustedToolInvocationPlan,
trustedToolContractIdentityDigest,
} = require('@qinglong/runtime-core/trusted-tool-invocation');
const {
prepareToolInvocation,
} = require('@qinglong/runtime-core/tool-registry');
const {
createToolExecutionCompletionCommand,
createToolExecutionResultArtifact,
} = require('@qinglong/runtime-core/tool-execution-completion');
const {
TOOL_EXECUTION_FAILURE_FACTS,
ToolExecutionFailureCompletionConflictError,
createToolExecutionFailureCompletionCommand,
createToolExecutionFailureResult,
} = require('@qinglong/runtime-core/tool-execution-failure-completion');
const {
TRUSTED_TOOL_EXECUTION_RESULT_SCHEMA,
} = require('@qinglong/runtime-core/trusted-tool-execution');
const {
createToolResultKeyCatalogBootstrapCommand,
normalizeToolResultKeyCatalogRecord,
requireActiveToolResultKey,
toolResultKeyCatalogFence,
toolResultKeyMaterialProof,
} = require('@qinglong/runtime-core/tool-result-key-catalog');
const {
PostgresToolExecutionStartBarrierRepository,
} = require('@qinglong/cluster-postgres/tool-execution-start-barrier');
const {
PostgresToolExecutionCompletionRepository,
} = require('@qinglong/cluster-postgres/tool-execution-completion');
const {
PostgresToolExecutionFailureCompletionRepository,
} = require('@qinglong/cluster-postgres/tool-execution-failure-completion');
const DIGEST_A = 'a'.repeat(64);
const DIGEST_B = 'b'.repeat(64);
const DIGEST_C = 'c'.repeat(64);
const REQUESTER = Object.freeze({ type: 'user', id: 'usr-tool-owner' });
const FENCE = Object.freeze({ projectVersion: 3, bindingVersion: 7 });
const NOW_MS = 1_400;
const OUTPUT_DIGEST_DOMAIN = Buffer.from(
'qinglong/trusted-tool-execution-output-digest@v1\0',
'utf8',
);
const RESULT_DIGEST_DOMAIN = Buffer.from(
'qinglong/trusted-tool-execution-result-digest@v1\0',
'utf8',
);
function resultKeyCatalog() {
const command = createToolResultKeyCatalogBootstrapCommand({
keyId: 'tool-result-key-test',
materialProof: toolResultKeyMaterialProof(
'tool-result-key-test',
Buffer.alloc(32, 5),
),
mutationId: 'tool-result-key-bootstrap-test',
});
return normalizeToolResultKeyCatalogRecord({
...command.next,
committedAtMs: 1_200,
});
}
function hash(domain, value) {
return createHash('sha256')
.update(domain)
.update(JSON.stringify(value))
.digest('hex');
}
function authorizer() {
return {
async authorize() {
return { effect: 'allow', reasons: ['role_grant'], fence: FENCE };
},
};
}
function principal() {
return {
subject: REQUESTER,
authenticationId: 'auth-tool-1',
authenticatedAtMs: 800,
expiresAtMs: 10_000,
assurance: 'local_console',
};
}
function projectToolSnapshot() {
const generation = createPluginPackageResourceGenerationFromReferences({
installationId: 'install-demo',
projectId: 'project-001',
packageName: 'demo',
lockDigest: DIGEST_A,
generation: 1,
previousActiveLockDigest: null,
contentDigest: DIGEST_B,
resources: [],
});
return createProjectToolDefinitionSnapshot({
projectId: 'project-001',
contributions: [
{
generation,
revisionDigest: DIGEST_C,
definitions: [
{
name: 'demo.compare',
version: '1.0.0',
description: 'Compare one bounded Run projection',
inputSchema: {
type: 'object',
properties: {
runId: { type: 'string', minLength: 1, maxLength: 64 },
},
required: ['runId'],
additionalProperties: false,
},
outputSchema: {
type: 'object',
properties: {
summary: { type: 'string', maxLength: 1024 },
},
required: ['summary'],
additionalProperties: false,
},
effect: 'read',
risk: 'low',
requiredPermissions: ['run.read'],
timeoutSeconds: 30,
},
],
},
],
});
}
async function command(overrides = {}) {
const snapshot = projectToolSnapshot();
const binding = createTrustedToolHandlerBinding(snapshot, {
tool: { name: 'demo.compare', version: '1.0.0' },
adapter: { id: 'builtin.demo-compare', version: '1.0.0' },
executionClass: 'builtin_in_process',
profiles: ['edge', 'standalone'],
authorities: ['database.read'],
timeoutSeconds: 20,
redactionContract: {
id: 'redaction.demo-compare',
version: '1.0.0',
},
auditContract: { id: 'audit.tool-call', version: '1.0.0' },
});
const bindings = new TrustedToolHandlerBindingRegistry(snapshot, [binding]);
const invocation = await prepareToolInvocation(
projectToolDefinitionRegistry(snapshot),
{
projectId: 'project-001',
principal: principal(),
nowMs: 900,
tool: { name: 'demo.compare', version: '1.0.0' },
input: { runId: 'run-001' },
},
authorizer(),
);
const plan = (
await createTrustedToolInvocationPlan(bindings, invocation, {
actionRef: 'tool-plan:run-001',
inputArtifactId: 'artifact-input-001',
previewArtifactId: 'artifact-preview-001',
artifactKeyId: 'tool-key-test',
artifactKey: Buffer.alloc(32, 7),
artifactNonce: Buffer.alloc(12, 9),
profile: 'edge',
preview: {
title: 'Compare Run',
summary: 'Reads one Run projection',
fields: [{ kind: 'identifier', label: 'Run', value: 'run-001' }],
warnings: [],
},
sealedAtMs: 1_000,
})
).plan;
const creation = createStepRunMutation(
{
id: 'step-run-001',
runId: 'run-001',
stepKey: 'workflow.compare',
kind: 'tool',
definitionRef: 'tool:demo.compare@1.0.0',
definitionDigest: snapshot.definitions[0].definitionDigest,
required: true,
initialStatus: 'ready',
inputRef: 'artifact:step-input-001',
mutationId: 'step-create-001',
createdAtMs: 1_000,
},
{
expectedRunVersion: 0,
expectedRunEventSequence: 0,
eventId: '50000000-0000-4000-8000-000000000001',
dedupeKey: 'step-create:step-run-001',
actor: REQUESTER,
},
);
const evidence = createToolExecutionEvidenceBundle({
traceId: '1'.repeat(32),
spanId: '2'.repeat(16),
projectId: 'project-001',
runId: 'run-001',
stepRunId: creation.stepRun.id,
invocationPlanDigest: plan.planDigest,
bindingDigest: binding.bindingDigest,
adapterDigest: trustedToolContractIdentityDigest(binding.adapter),
redactionContractDigest: trustedToolContractIdentityDigest(
binding.redactionContract,
),
auditContractDigest: trustedToolContractIdentityDigest(
binding.auditContract,
),
audit: {
eventId: '40000000-0000-4000-8000-000000000001',
requestId: 'tool-request-001',
operationId: TOOL_EXECUTION_START_AUDIT_OPERATION,
projectId: 'project-001',
subject: REQUESTER,
authenticationId: 'auth-tool-1',
outcome: 'allowed',
reasons: ['tool_execution_start'],
fence: FENCE,
occurredAtMs: NOW_MS,
},
createdAtMs: NOW_MS,
});
const admission = await admitTrustedToolExecution(bindings, plan, {
principal: principal(),
profile: 'edge',
nowMs: NOW_MS,
authorizer: authorizer(),
evidence: {
stepRun: {
id: creation.stepRun.id,
version: creation.stepRun.version,
digest: creation.stepRun.stepRunDigest,
},
...toolExecutionAdmissionEvidence(evidence),
},
});
const mutation = transitionStepRunMutation(
creation.stepRun,
{
expectedVersion: creation.stepRun.version,
expectedDigest: creation.stepRun.stepRunDigest,
mutationId: 'step-running-002',
to: 'running',
atMs: NOW_MS,
},
{
expectedRunVersion: 1,
expectedRunEventSequence: 1,
eventId: '50000000-0000-4000-8000-000000000002',
dedupeKey: 'step-running:step-run-001',
actor: REQUESTER,
},
);
return createToolExecutionStartCommand({
startId: overrides.startId ?? 'tool-start-001',
admission,
evidence,
stepRunMutation: mutation,
});
}
function barrierRow(barrier, overrides = {}) {
return {
barrierJson: barrier,
storedStartId: barrier.startId,
storedProjectId: barrier.projectId,
storedRunId: barrier.runId,
storedStepRunId: barrier.stepRunId,
storedStepRunVersion: String(barrier.startedStepRunVersion),
storedMutationId: barrier.stepRunMutationId,
storedRunEventId: barrier.runEventId,
storedTraceId: barrier.traceId,
storedSpanId: barrier.spanId,
storedAuditEventId: barrier.auditEventId,
storedCommandDigest: barrier.commandDigest,
storedBarrierDigest: barrier.barrierDigest,
storedStartedAtMs: String(barrier.startedAtMs),
storedMutationDigest: barrier.stepRunMutationDigest,
storedStartedStepRunDigest: barrier.startedStepRunDigest,
storedTraceDigest: barrier.traceDigest,
storedAuditReceiptDigest: barrier.auditReceiptDigest,
storedArtifactProjectId: barrier.projectId,
storedArtifactActionRef: barrier.actionRef,
storedInputArtifactId: barrier.invocationArtifact.artifactId,
storedInputArtifactDigest: barrier.invocationArtifact.artifactDigest,
storedInputDigest: barrier.invocationArtifact.inputDigest,
storedPreviewArtifactId: barrier.previewArtifact.artifactId,
storedPreviewArtifactDigest: barrier.previewArtifact.artifactDigest,
storedArtifactActionDigest: barrier.previewArtifact.actionDigest,
storedPreviewDigest: barrier.previewArtifact.previewDigest,
storedArtifactRedactionContractDigest:
barrier.previewArtifact.redactionContractDigest,
storedArtifactBoundAtMs: String(barrier.startedAtMs),
...overrides,
};
}
function clientWith(handler) {
const calls = [];
let released = false;
return {
calls,
get released() {
return released;
},
async query(text, values = []) {
calls.push({ text, values });
if (
text.startsWith('BEGIN') ||
text.includes("set_config('") ||
text.includes('pg_advisory_xact_lock') ||
text === 'COMMIT' ||
text === 'ROLLBACK'
) {
return { rows: [] };
}
if (text.includes('plugin_package_tool_start_allowed')) {
return { rows: [{ allowed: true }] };
}
return handler(text, values, calls);
},
release() {
released = true;
},
};
}
function repositoryFor(client) {
return new PostgresToolExecutionStartBarrierRepository({
async query() {
throw new Error('pool query is not used by prepare');
},
async connect() {
return client;
},
});
}
function completionRepositoryFor(client) {
return new PostgresToolExecutionCompletionRepository({
async query() {
throw new Error('pool query is not used by commit');
},
async connect() {
return client;
},
});
}
function failureCompletionRepositoryFor(client) {
return new PostgresToolExecutionFailureCompletionRepository({
async query() {
throw new Error('pool query is not used by commit');
},
async connect() {
return client;
},
});
}
function executionResult(barrier, output, completedAtMs) {
const outputDigest = hash(OUTPUT_DIGEST_DOMAIN, output);
const unsigned = Object.freeze({
schema: TRUSTED_TOOL_EXECUTION_RESULT_SCHEMA,
startId: barrier.startId,
barrierDigest: barrier.barrierDigest,
adapterDigest: barrier.adapterDigest,
output,
outputDigest,
completedAtMs,
});
return Object.freeze({
...unsigned,
resultDigest: hash(RESULT_DIGEST_DOMAIN, unsigned),
});
}
test('atomically commits evidence, StepRun start and the barrier', async () => {
const start = await command();
const mutation = start.stepRunMutation;
const client = clientWith(async (sql) => {
if (sql.includes('FROM "ql3"."tool_execution_start_barriers"')) {
return { rows: [] };
}
if (sql.includes('FROM "ql3"."step_runs" AS step')) {
return {
rows: [
{
stepKind: 'tool',
stepStatus: mutation.previousStatus,
stepVersion: String(mutation.expectedStepRunVersion),
stepDigest: mutation.expectedStepRunDigest,
definitionRef: mutation.stepRun.definitionRef,
definitionDigest: mutation.stepRun.definitionDigest,
projectId: 'project-001',
runStatus: 'running',
runVersion: String(mutation.expectedRunVersion),
runEventSequence: String(mutation.expectedRunEventSequence),
},
],
};
}
if (sql.startsWith('INSERT INTO') || sql.startsWith('UPDATE "ql3"')) {
return { rows: [], rowCount: 1 };
}
throw new Error(`unexpected SQL: ${sql}`);
});
const result = await repositoryFor(client).prepare(start);
assert.deepEqual(result, {
status: 'created',
barrier: toolExecutionStartBarrierRecord(start),
});
assert.equal(client.released, true);
assert.deepEqual(
client.calls
.map(({ text }) => text)
.filter(
(sql) =>
sql.startsWith('INSERT INTO') || sql.startsWith('UPDATE "ql3"'),
)
.map((sql) => {
const match = sql.match(/"ql3"\."([^"]+)"/);
return `${sql.startsWith('UPDATE') ? 'update' : 'insert'}:${match[1]}`;
}),
[
'insert:security_audit_events',
'insert:tool_execution_trace_anchors',
'insert:tool_execution_audit_receipts',
'update:step_runs',
'update:runs',
'insert:run_events',
'insert:step_run_mutations',
'insert:tool_execution_start_barriers',
'insert:tool_execution_start_artifact_bindings',
],
);
const mutationInsert = client.calls.find(({ text }) =>
text.includes('INSERT INTO "ql3"."step_run_mutations"'),
);
assert.match(mutationInsert.text, /transaction_timestamp\(\)/);
assert.equal(mutationInsert.values.length, 9);
assert.equal(
client.calls.some(({ text }) => text === 'COMMIT'),
true,
);
assert.equal(
client.calls.some(({ text }) => text === 'ROLLBACK'),
false,
);
});
test('atomically persists encrypted result and the succeeded StepRun fence', async () => {
const start = await command();
const barrier = toolExecutionStartBarrierRecord(start);
const result = executionResult(barrier, { summary: 'Run is healthy' }, 1_500);
const registry = projectToolDefinitionRegistry(projectToolSnapshot());
const artifact = createToolExecutionResultArtifact(
{
artifactId: 'artifact-result-001',
projectId: barrier.projectId,
runId: barrier.runId,
stepRunId: barrier.stepRunId,
tool: { name: 'demo.compare', version: '1.0.0' },
executionResult: result,
keyId: 'tool-result-key-test',
key: Buffer.alloc(32, 5),
},
registry,
() => Buffer.alloc(12, 4),
);
const running = start.stepRunMutation.stepRun;
const mutation = transitionStepRunMutation(
running,
{
expectedVersion: running.version,
expectedDigest: running.stepRunDigest,
mutationId: 'step-succeeded-003',
to: 'succeeded',
atMs: result.completedAtMs,
outputRef: artifact.artifactId,
},
{
expectedRunVersion: 2,
expectedRunEventSequence: 2,
eventId: '50000000-0000-4000-8000-000000000003',
dedupeKey: 'step-succeeded:step-run-001',
actor: REQUESTER,
},
);
const completionCommand = createToolExecutionCompletionCommand({
barrier,
executionResult: result,
resultArtifact: artifact,
resultKeyCatalogFence: toolResultKeyCatalogFence(
resultKeyCatalog(),
requireActiveToolResultKey(resultKeyCatalog()),
),
stepRunMutation: mutation,
});
const client = clientWith(async (sql) => {
if (sql.includes('FROM "ql3"."tool_execution_completions"')) {
return { rows: [] };
}
if (sql.includes('FROM "ql3"."tool_execution_failure_completions"')) {
return { rows: [] };
}
if (sql.includes('FROM "ql3"."tool_result_key_catalog_generations"')) {
return { rows: [{ catalogJson: resultKeyCatalog() }] };
}
if (sql.includes('FROM "ql3"."tool_execution_start_barriers" AS barrier')) {
return {
rows: [
{
barrierJson: barrier,
startedRunVersion: '2',
startedEventSequence: '2',
stepKind: 'tool',
stepStatus: 'running',
stepVersion: String(mutation.expectedStepRunVersion),
stepDigest: mutation.expectedStepRunDigest,
projectId: barrier.projectId,
runStatus: 'running',
runVersion: '2',
runEventSequence: '2',
},
],
};
}
if (sql.startsWith('INSERT INTO') || sql.startsWith('UPDATE "ql3"')) {
return { rows: [], rowCount: 1 };
}
throw new Error(`unexpected SQL: ${sql}`);
});
const committed = await completionRepositoryFor(client).commit(
completionCommand,
);
assert.equal(committed.status, 'created');
assert.equal(committed.completion.startId, barrier.startId);
assert.equal(client.released, true);
assert.deepEqual(
client.calls
.map(({ text }) => text)
.filter(
(sql) =>
sql.startsWith('INSERT INTO') || sql.startsWith('UPDATE "ql3"'),
)
.map((sql) => {
const match = sql.match(/"ql3"\."([^"]+)"/);
return `${sql.startsWith('UPDATE') ? 'update' : 'insert'}:${match[1]}`;
}),
[
'update:step_runs',
'update:runs',
'insert:run_events',
'insert:step_run_mutations',
'insert:tool_execution_completions',
'insert:tool_execution_result_key_bindings',
],
);
});
test('atomically persists fixed Tool failure facts and excludes success', async () => {
const start = await command();
const barrier = toolExecutionStartBarrierRecord(start);
const failure = createToolExecutionFailureResult(barrier, 'timed_out', 1_500);
const running = start.stepRunMutation.stepRun;
const mutation = transitionStepRunMutation(
running,
{
expectedVersion: running.version,
expectedDigest: running.stepRunDigest,
mutationId: 'step-timed-out-003',
to: 'timed_out',
atMs: failure.completedAtMs,
...TOOL_EXECUTION_FAILURE_FACTS.timed_out,
},
{
expectedRunVersion: 2,
expectedRunEventSequence: 2,
eventId: '50000000-0000-4000-8000-000000000003',
dedupeKey: 'step-timed-out:step-run-001',
actor: REQUESTER,
},
);
const completionCommand = createToolExecutionFailureCompletionCommand({
barrier,
failure,
stepRunMutation: mutation,
});
const client = clientWith(async (sql) => {
if (sql.includes('FROM "ql3"."tool_execution_failure_completions"')) {
return { rows: [] };
}
if (sql.includes('FROM "ql3"."tool_execution_completions"')) {
return { rows: [] };
}
if (sql.includes('FROM "ql3"."tool_execution_start_barriers" AS barrier')) {
return {
rows: [
{
barrierJson: barrier,
startedRunVersion: '2',
startedEventSequence: '2',
stepKind: 'tool',
stepStatus: 'running',
stepVersion: String(mutation.expectedStepRunVersion),
stepDigest: mutation.expectedStepRunDigest,
projectId: barrier.projectId,
runStatus: 'running',
runVersion: '2',
runEventSequence: '2',
},
],
};
}
if (sql.startsWith('INSERT INTO') || sql.startsWith('UPDATE "ql3"')) {
return { rows: [], rowCount: 1 };
}
throw new Error(`unexpected SQL: ${sql}`);
});
const committed = await failureCompletionRepositoryFor(client).commit(
completionCommand,
);
assert.equal(committed.status, 'created');
assert.equal(committed.completion.outcome, 'timed_out');
assert.equal(
committed.completion.errorSummary,
'Trusted Tool execution deadline exceeded',
);
assert.deepEqual(
client.calls
.map(({ text }) => text)
.filter(
(sql) =>
sql.startsWith('INSERT INTO') || sql.startsWith('UPDATE "ql3"'),
)
.map((sql) => {
const match = sql.match(/"ql3"\."([^"]+)"/);
return `${sql.startsWith('UPDATE') ? 'update' : 'insert'}:${match[1]}`;
}),
[
'update:step_runs',
'update:runs',
'insert:run_events',
'insert:step_run_mutations',
'insert:tool_execution_failure_completions',
],
);
const conflictClient = clientWith(async (sql) => {
if (sql.includes('FROM "ql3"."tool_execution_failure_completions"')) {
return { rows: [] };
}
if (sql.includes('FROM "ql3"."tool_execution_completions"')) {
return { rows: [{ exists: 1 }] };
}
throw new Error(`unexpected SQL: ${sql}`);
});
await assert.rejects(
failureCompletionRepositoryFor(conflictClient).commit(completionCommand),
ToolExecutionFailureCompletionConflictError,
);
assert.equal(
conflictClient.calls.some(({ text }) => text === 'ROLLBACK'),
true,
);
});
test('exactly replays a committed start and rejects identity drift', async () => {
const start = await command();
const stored = toolExecutionStartBarrierRecord(start);
const exactClient = clientWith(async (sql) => {
if (sql.includes('FROM "ql3"."tool_execution_start_barriers"')) {
return { rows: [barrierRow(stored)] };
}
throw new Error(`unexpected SQL: ${sql}`);
});
assert.deepEqual(await repositoryFor(exactClient).prepare(start), {
status: 'existing',
barrier: stored,
});
assert.equal(
exactClient.calls.some(({ text }) => text.startsWith('INSERT INTO')),
false,
);
const drift = await command({ startId: 'tool-start-other' });
const conflictClient = clientWith(async (sql) => {
if (sql.includes('FROM "ql3"."tool_execution_start_barriers"')) {
return { rows: [barrierRow(stored)] };
}
throw new Error(`unexpected SQL: ${sql}`);
});
await assert.rejects(
repositoryFor(conflictClient).prepare(drift),
ToolExecutionStartBarrierConflictError,
);
assert.equal(
conflictClient.calls.some(({ text }) => text === 'ROLLBACK'),
true,
);
assert.equal(conflictClient.released, true);
});
test('rolls back before evidence writes when the durable fence changed', async () => {
const start = await command();
const mutation = start.stepRunMutation;
const client = clientWith(async (sql) => {
if (sql.includes('FROM "ql3"."tool_execution_start_barriers"')) {
return { rows: [] };
}
if (sql.includes('FROM "ql3"."step_runs" AS step')) {
return {
rows: [
{
stepKind: 'tool',
stepStatus: 'cancelled',
stepVersion: String(mutation.expectedStepRunVersion),
stepDigest: mutation.expectedStepRunDigest,
definitionRef: mutation.stepRun.definitionRef,
definitionDigest: mutation.stepRun.definitionDigest,
projectId: 'project-001',
runStatus: 'running',
runVersion: String(mutation.expectedRunVersion),
runEventSequence: String(mutation.expectedRunEventSequence),
},
],
};
}
throw new Error(`unexpected SQL: ${sql}`);
});
await assert.rejects(
repositoryFor(client).prepare(start),
ToolExecutionStartBarrierConflictError,
);
assert.equal(
client.calls.some(({ text }) => text.startsWith('INSERT INTO')),
false,
);
assert.equal(
client.calls.some(({ text }) => text === 'ROLLBACK'),
true,
);
assert.equal(client.released, true);
});
test('fails closed when a stored barrier projection is corrupted', async () => {
const start = await command();
const barrier = toolExecutionStartBarrierRecord(start);
const repository = new PostgresToolExecutionStartBarrierRepository({
async query(sql) {
if (sql.includes('FROM "ql3"."tool_execution_start_barriers"')) {
return {
rows: [
barrierRow(barrier, {
storedBarrierDigest: '0'.repeat(64),
}),
],
};
}
throw new Error(`unexpected SQL: ${sql}`);
},
async connect() {
throw new Error('unused');
},
});
await assert.rejects(
repository.findByStartId(barrier.startId),
ToolExecutionStartBarrierUnavailableError,
);
});
test('fails closed when a historical barrier has no Artifact binding', async () => {
const start = await command();
const barrier = toolExecutionStartBarrierRecord(start);
const repository = new PostgresToolExecutionStartBarrierRepository({
async query(sql) {
if (sql.includes('FROM "ql3"."tool_execution_start_barriers"')) {
return {
rows: [
barrierRow(barrier, {
storedInputArtifactId: null,
}),
],
};
}
throw new Error(`unexpected SQL: ${sql}`);
},
async connect() {
throw new Error('unused');
},
});
await assert.rejects(
repository.findByStartId(barrier.startId),
ToolExecutionStartBarrierUnavailableError,
);
});
@@ -0,0 +1,329 @@
const assert = require('node:assert/strict');
const { createHash } = require('node:crypto');
const { test } = require('node:test');
const {
ToolInvocationArtifactConflictError,
ToolInvocationArtifactUnavailableError,
createToolInvocationInputArtifact,
createToolInvocationPreviewArtifact,
} = require('@qinglong/runtime-core/tool-invocation-artifact');
const {
PostgresToolInvocationArtifactRepository,
} = require('@qinglong/cluster-postgres/tool-invocation-artifact');
const KEY = Buffer.alloc(32, 7);
const NONCE = Buffer.alloc(12, 9);
const INVOCATION_ACTION_DIGEST = 'a'.repeat(64);
const ACTION_DIGEST = 'b'.repeat(64);
const REDACTION_DIGEST = 'c'.repeat(64);
function digest(value) {
return createHash('sha256').update(JSON.stringify(value)).digest('hex');
}
function artifacts(index = 1, overrides = {}) {
const input = {
runId: `run-${String(index).padStart(3, '0')}`,
token: 'secret-value',
};
const common = {
projectId: 'project-001',
actionRef: `tool-plan:${input.runId}`,
sealedAtMs: 1_000 + index,
...overrides.common,
};
return {
inputArtifact: createToolInvocationInputArtifact(
{
artifactId: `artifact-input-${index}`,
requestedBy: { type: 'user', id: 'usr-owner' },
tool: { name: 'demo.compare', version: '1.0.0' },
input,
inputDigest: digest(input),
invocationActionDigest: INVOCATION_ACTION_DIGEST,
keyId: 'tool-key-test',
key: KEY,
...common,
...overrides.input,
},
() => NONCE,
),
previewArtifact: createToolInvocationPreviewArtifact({
artifactId: `artifact-preview-${index}`,
actionDigest: ACTION_DIGEST,
redactionContractDigest: REDACTION_DIGEST,
preview: {
title: 'Compare Run',
summary: 'Reads one bounded Run projection',
fields: [
{ kind: 'identifier', label: 'Run', value: input.runId },
{ kind: 'redacted', label: 'Credential', value: null },
],
warnings: [],
},
...common,
...overrides.preview,
}),
};
}
function inputRow(artifact, overrides = {}) {
return {
artifactId: artifact.artifactId,
projectId: artifact.projectId,
actionRef: artifact.actionRef,
inputDigest: artifact.inputDigest,
invocationActionDigest: artifact.invocationActionDigest,
artifactDigest: artifact.artifactDigest,
keyId: artifact.keyId,
algorithm: artifact.algorithm,
plaintextBytes: String(artifact.plaintextBytes),
sealedAtMs: String(artifact.sealedAtMs),
artifactJson: artifact,
...overrides,
};
}
function previewRow(artifact, overrides = {}) {
return {
artifactId: artifact.artifactId,
projectId: artifact.projectId,
actionRef: artifact.actionRef,
actionDigest: artifact.actionDigest,
previewDigest: artifact.previewDigest,
redactionContractDigest: artifact.redactionContractDigest,
artifactDigest: artifact.artifactDigest,
byteLength: String(artifact.byteLength),
sealedAtMs: String(artifact.sealedAtMs),
artifactJson: artifact,
...overrides,
};
}
function clientWith(handler) {
const calls = [];
let released = false;
return {
calls,
get released() {
return released;
},
async query(text, values = []) {
calls.push({ text, values });
if (
text.startsWith('BEGIN') ||
text.includes("set_config('") ||
text === 'COMMIT' ||
text === 'ROLLBACK'
) {
return { rows: [] };
}
return handler(text, values, calls);
},
release() {
released = true;
},
};
}
function repositoryFor(client, query = client.query.bind(client)) {
return new PostgresToolInvocationArtifactRepository({
query,
async connect() {
return client;
},
});
}
test('atomically inserts one Artifact pair without storing plaintext JSON', async () => {
const pair = artifacts();
const client = clientWith(async (sql) => {
if (sql.startsWith('SELECT')) return { rows: [] };
if (sql.startsWith('INSERT INTO')) return { rows: [], rowCount: 1 };
throw new Error(`unexpected SQL: ${sql}`);
});
assert.deepEqual(
await repositoryFor(client).put(
pair.inputArtifact,
pair.previewArtifact,
),
{ status: 'inserted' },
);
assert.equal(client.released, true);
assert.equal(
client.calls.filter(({ text }) => text.startsWith('INSERT INTO'))
.length,
2,
);
assert.equal(
client.calls
.filter(({ text }) => text.startsWith('INSERT INTO'))
.some(({ values }) => values.some((value) =>
String(value).includes('secret-value'),
)),
false,
);
assert.equal(
client.calls.some(({ text }) => text === 'COMMIT'),
true,
);
assert.equal(
client.calls.some(({ text }) => text.includes('FOR SHARE')),
false,
);
});
test('exactly replays a complete pair and rejects partial durable state', async () => {
const pair = artifacts();
const exact = clientWith(async (sql) => {
if (sql.includes('tool_invocation_input_artifacts')) {
return { rows: [inputRow(pair.inputArtifact)] };
}
if (sql.includes('tool_invocation_preview_artifacts')) {
return { rows: [previewRow(pair.previewArtifact)] };
}
throw new Error(`unexpected SQL: ${sql}`);
});
assert.deepEqual(
await repositoryFor(exact).put(
pair.inputArtifact,
pair.previewArtifact,
),
{ status: 'existing' },
);
assert.equal(
exact.calls.some(({ text }) => text.startsWith('INSERT INTO')),
false,
);
const partial = clientWith(async (sql) => {
if (sql.includes('tool_invocation_input_artifacts')) {
return { rows: [inputRow(pair.inputArtifact)] };
}
if (sql.includes('tool_invocation_preview_artifacts')) {
return { rows: [] };
}
throw new Error(`unexpected SQL: ${sql}`);
});
await assert.rejects(
repositoryFor(partial).put(pair.inputArtifact, pair.previewArtifact),
ToolInvocationArtifactConflictError,
);
assert.equal(
partial.calls.some(({ text }) => text === 'ROLLBACK'),
true,
);
});
test('rejects detached pairs before opening a transaction', async () => {
const first = artifacts();
const detached = artifacts(2, {
common: {
projectId: 'project-001',
actionRef: 'tool-plan:detached',
sealedAtMs: 1_002,
},
});
let connected = false;
const repository = new PostgresToolInvocationArtifactRepository({
async query() {
throw new Error('unused');
},
async connect() {
connected = true;
throw new Error('unused');
},
});
await assert.rejects(
repository.put(first.inputArtifact, detached.previewArtifact),
ToolInvocationArtifactConflictError,
);
assert.equal(connected, false);
});
test('converges a concurrent unique-key winner through exact replay', async () => {
const pair = artifacts();
const uniqueViolation = clientWith(async (sql) => {
if (sql.startsWith('SELECT')) return { rows: [] };
if (sql.startsWith('INSERT INTO')) {
const error = new Error('duplicate key');
error.code = '23505';
throw error;
}
throw new Error(`unexpected SQL: ${sql}`);
});
const winner = clientWith(async (sql) => {
if (sql.includes('tool_invocation_input_artifacts')) {
return { rows: [inputRow(pair.inputArtifact)] };
}
if (sql.includes('tool_invocation_preview_artifacts')) {
return { rows: [previewRow(pair.previewArtifact)] };
}
throw new Error(`unexpected SQL: ${sql}`);
});
const clients = [uniqueViolation, winner];
let connections = 0;
const repository = new PostgresToolInvocationArtifactRepository({
async query() {
throw new Error('unused');
},
async connect() {
const client = clients[connections];
connections += 1;
return client;
},
});
assert.deepEqual(
await repository.put(pair.inputArtifact, pair.previewArtifact),
{ status: 'existing' },
);
assert.equal(connections, 2);
assert.equal(
uniqueViolation.calls.some(({ text }) => text === 'ROLLBACK'),
true,
);
assert.equal(uniqueViolation.released, true);
assert.equal(winner.released, true);
});
test('fails closed when a stored projection is corrupted', async () => {
const pair = artifacts();
const client = clientWith(async () => {
throw new Error('unused');
});
const repository = repositoryFor(client, async (sql) => {
if (sql.includes('tool_invocation_input_artifacts')) {
return {
rows: [
inputRow(pair.inputArtifact, {
inputDigest: 'd'.repeat(64),
}),
],
};
}
throw new Error(`unexpected SQL: ${sql}`);
});
await assert.rejects(
repository.findInput(pair.inputArtifact.artifactId),
ToolInvocationArtifactUnavailableError,
);
});
test('publishes the explicit runtime adapter subpath', () => {
const root = require('../dist');
const runtime = require('../dist/entrypoints/runtime');
const subpath = require('@qinglong/cluster-postgres/tool-invocation-artifact');
assert.equal(
root.PostgresToolInvocationArtifactRepository,
PostgresToolInvocationArtifactRepository,
);
assert.equal(
runtime.PostgresToolInvocationArtifactRepository,
PostgresToolInvocationArtifactRepository,
);
assert.equal(
subpath.PostgresToolInvocationArtifactRepository,
PostgresToolInvocationArtifactRepository,
);
});
@@ -0,0 +1,238 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
ToolResultKeyCatalogConflictError,
createToolResultKeyCatalogBootstrapCommand,
createToolResultKeyRetirementCommand,
createToolResultKeyRotationCommand,
toolResultKeyMaterialProof,
} = require('@qinglong/runtime-core/tool-result-key-catalog');
const {
createToolResultKeyRetirementReceipt,
} = require('@qinglong/runtime-core/tool-result-rekey');
const {
PostgresToolResultKeyCatalogReader,
} = require('@qinglong/cluster-postgres/runtime');
const {
PostgresToolResultKeyCatalogRepository,
} = require('@qinglong/cluster-postgres/tool-result-key-catalog');
function jsonb(value) {
if (Array.isArray(value)) return value.map(jsonb);
if (!value || typeof value !== 'object') return value;
return Object.fromEntries(
Object.keys(value)
.sort()
.map((key) => [key, jsonb(value[key])]),
);
}
function row(catalog, commandDigest) {
return {
generation: String(catalog.generation),
previousCatalogDigest: catalog.previousCatalogDigest,
activeKeyId: catalog.activeKeyId,
mutationKind: catalog.mutationKind,
mutationId: catalog.mutationId,
catalogDigest: catalog.catalogDigest,
commandDigest,
committedAtMs: String(catalog.committedAtMs),
catalogJson: jsonb(catalog),
};
}
function harness() {
const calls = [];
const history = [];
const receipts = [];
let released = 0;
const client = {
async query(text, values = []) {
calls.push({ text, values });
if (
text.startsWith('BEGIN') ||
text === 'COMMIT' ||
text === 'ROLLBACK' ||
text.includes('pg_advisory_xact_lock')
) {
return { rows: [] };
}
if (text.includes('AND (mutation_id =')) {
const found = history.filter(
({ catalog }) =>
catalog.mutationId === values[1] ||
catalog.generation === values[2] ||
catalog.catalogDigest === values[3],
);
return {
rows: found.map(({ catalog, commandDigest }) =>
row(catalog, commandDigest),
),
};
}
if (text.includes('tool_result_key_retirement_receipts')) {
const receipt = receipts.find(
(candidate) => candidate.receiptDigest === values[0],
);
return { rows: receipt ? [{ receiptJson: receipt }] : [] };
}
if (text.includes('FROM "ql3"."tool_result_key_catalog_generations"')) {
return {
rows: history
.slice()
.reverse()
.slice(0, 2)
.map(({ catalog, commandDigest }) => row(catalog, commandDigest)),
};
}
if (text.includes('clock_timestamp()')) {
return { rows: [{ now: String(1_000 + history.length) }] };
}
if (
text.startsWith(
'INSERT INTO "ql3"."tool_result_key_catalog_generations"',
)
) {
history.push({
catalog: JSON.parse(values[10]),
commandDigest: values[8],
});
return { rows: [], rowCount: 1 };
}
throw new Error(`unexpected SQL: ${text}`);
},
release() {
released += 1;
},
};
const pool = {
async query(text, values) {
return client.query(text, values);
},
async connect() {
return client;
},
};
return {
calls,
history,
receipts,
pool,
repository: new PostgresToolResultKeyCatalogRepository(pool),
reader: new PostgresToolResultKeyCatalogReader(pool),
released: () => released,
};
}
function bootstrapCommand() {
return createToolResultKeyCatalogBootstrapCommand({
keyId: 'tool-result-key-001',
materialProof: toolResultKeyMaterialProof(
'tool-result-key-001',
Buffer.alloc(32, 1),
),
mutationId: 'tool-result-key-bootstrap-001',
});
}
test('serializes PostgreSQL key generations behind the catalog lock', async () => {
const current = harness();
assert.equal(await current.reader.findCurrent(), null);
const bootstrap = bootstrapCommand();
const first = await current.repository.append(bootstrap);
assert.equal(first.status, 'created');
assert.deepEqual(await current.repository.append(bootstrap), {
status: 'existing',
catalog: first.catalog,
});
const second = await current.repository.append(
createToolResultKeyRotationCommand(first.catalog, {
keyId: 'tool-result-key-002',
materialProof: toolResultKeyMaterialProof(
'tool-result-key-002',
Buffer.alloc(32, 2),
),
mutationId: 'tool-result-key-rotate-002',
}),
);
assert.equal(second.catalog.generation, 2);
assert.deepEqual(await current.reader.findCurrent(), second.catalog);
assert.equal(current.history.length, 2);
assert.equal(current.released(), 3);
await assert.rejects(
current.repository.append(
createToolResultKeyRetirementCommand(second.catalog, {
keyId: 'tool-result-key-001',
retirementReceiptDigest: 'f'.repeat(64),
mutationId: 'tool-result-key-retire-forged-001',
}),
),
ToolResultKeyCatalogConflictError,
);
const retiring = second.catalog.keys.find(
(entry) => entry.keyId === 'tool-result-key-001',
);
const receipt = createToolResultKeyRetirementReceipt({
catalogGeneration: second.catalog.generation,
catalogDigest: second.catalog.catalogDigest,
keyId: retiring.keyId,
materialProof: retiring.materialProof,
mutationId: 'tool-result-key-retirement-receipt-001',
bindingCount: 0,
overlayHeadCount: 0,
coverageDigest: 'c'.repeat(64),
createdAtMs: 1_500,
});
current.receipts.push(receipt);
const retired = await current.repository.append(
createToolResultKeyRetirementCommand(second.catalog, {
keyId: 'tool-result-key-001',
retirementReceiptDigest: receipt.receiptDigest,
mutationId: 'tool-result-key-retire-001',
}),
);
assert.equal(
retired.catalog.keys.find(
(entry) => entry.keyId === 'tool-result-key-001',
).state,
'retired',
);
const transactionCalls = current.calls
.map(({ text }) => text)
.filter(
(text) =>
text.startsWith('BEGIN') ||
text.includes('pg_advisory_xact_lock') ||
text.startsWith(
'INSERT INTO "ql3"."tool_result_key_catalog_generations"',
) ||
text === 'COMMIT',
);
assert.equal(
transactionCalls.indexOf('SELECT pg_advisory_xact_lock(190397473, 3)') <
transactionCalls.findIndex((text) =>
text.startsWith(
'INSERT INTO "ql3"."tool_result_key_catalog_generations"',
),
),
true,
);
await assert.rejects(
current.repository.append(
createToolResultKeyRotationCommand(first.catalog, {
keyId: 'tool-result-key-stale',
materialProof: toolResultKeyMaterialProof(
'tool-result-key-stale',
Buffer.alloc(32, 3),
),
mutationId: 'tool-result-key-stale-003',
}),
),
ToolResultKeyCatalogConflictError,
);
});
@@ -0,0 +1,476 @@
const assert = require('node:assert/strict');
const { createHash } = require('node:crypto');
const { test } = require('node:test');
const {
BUILTIN_RUN_READ_TOOL,
BUILTIN_RUN_READ_TOOL_DEFINITION,
} = require('@qinglong/runtime-core/builtin-run-read-tool');
const {
TRUSTED_TOOL_EXECUTION_RESULT_SCHEMA,
} = require('@qinglong/runtime-core/trusted-tool-execution');
const {
TOOL_EXECUTION_RESULT_KEY_BINDING_SCHEMA,
createToolExecutionResultArtifact,
normalizeToolExecutionResultKeyBinding,
} = require('@qinglong/runtime-core/tool-execution-completion');
const {
createToolResultKeyCatalogBootstrapCommand,
createToolResultKeyRotationCommand,
normalizeToolResultKeyCatalogRecord,
requireActiveToolResultKey,
toolResultKeyCatalogFence,
toolResultKeyMaterialProof,
} = require('@qinglong/runtime-core/tool-result-key-catalog');
const {
ToolExecutionResultRekeyConflictError,
createToolExecutionResultRekeyCommand,
createToolResultKeyRetirementReceiptCommand,
} = require('@qinglong/runtime-core/tool-result-rekey');
const {
ToolDefinitionRegistry,
} = require('@qinglong/runtime-core/tool-registry');
const {
PostgresToolResultRekeyReader,
} = require('@qinglong/cluster-postgres/runtime');
const {
PostgresToolResultRekeyRepository,
} = require('@qinglong/cluster-postgres/tool-result-rekey');
const KEY_A = Buffer.alloc(32, 1);
const KEY_B = Buffer.alloc(32, 2);
const OUTPUT_DIGEST_DOMAIN = Buffer.from(
'qinglong/trusted-tool-execution-output-digest@v1\0',
);
const RESULT_DIGEST_DOMAIN = Buffer.from(
'qinglong/trusted-tool-execution-result-digest@v1\0',
);
const BINDING_DIGEST_DOMAIN = Buffer.from(
'qinglong/tool-execution-result-key-binding-digest@v1\0',
);
function hash(domain, value) {
return createHash('sha256')
.update(domain)
.update(JSON.stringify(value))
.digest('hex');
}
function jsonb(value) {
if (Array.isArray(value)) return value.map(jsonb);
if (!value || typeof value !== 'object') return value;
return Object.fromEntries(
Object.keys(value)
.sort()
.map((key) => [key, jsonb(value[key])]),
);
}
function output() {
return {
createdAtMs: 1_000,
eventSequence: 3,
executionOrigin: 'manual',
executionOwner: 'runtime',
found: true,
id: 'run-rekey-postgres-001',
priority: 10,
queuedAtMs: 1_100,
startedAtMs: 1_200,
status: 'succeeded',
taskId: 'task-rekey-postgres-001',
taskRevision: 'task-rekey-postgres-001@1',
version: 2,
};
}
function registry() {
return new ToolDefinitionRegistry([BUILTIN_RUN_READ_TOOL_DEFINITION]);
}
function sourceArtifact() {
const value = output();
const unsigned = {
schema: TRUSTED_TOOL_EXECUTION_RESULT_SCHEMA,
startId: 'tool-start-rekey-postgres-001',
barrierDigest: 'a'.repeat(64),
adapterDigest: 'b'.repeat(64),
output: value,
outputDigest: hash(OUTPUT_DIGEST_DOMAIN, value),
completedAtMs: 1_500,
};
const result = {
...unsigned,
resultDigest: hash(RESULT_DIGEST_DOMAIN, unsigned),
};
return createToolExecutionResultArtifact(
{
artifactId: 'artifact-result-rekey-postgres-001',
projectId: 'project-rekey-postgres-001',
runId: 'run-host-rekey-postgres-001',
stepRunId: 'step-run-rekey-postgres-001',
tool: BUILTIN_RUN_READ_TOOL,
executionResult: result,
keyId: 'result-key-a',
key: KEY_A,
},
registry(),
() => Buffer.alloc(12, 4),
);
}
function sourceBinding(artifact, catalog) {
const unsigned = {
schema: TOOL_EXECUTION_RESULT_KEY_BINDING_SCHEMA,
startId: artifact.startId,
artifactId: artifact.artifactId,
artifactDigest: artifact.artifactDigest,
catalogGeneration: catalog.generation,
catalogDigest: catalog.catalogDigest,
keyId: artifact.keyId,
materialProof: toolResultKeyMaterialProof(artifact.keyId, KEY_A),
};
return normalizeToolExecutionResultKeyBinding({
...unsigned,
bindingDigest: hash(BINDING_DIGEST_DOMAIN, unsigned),
});
}
function catalogs() {
const bootstrap = createToolResultKeyCatalogBootstrapCommand({
keyId: 'result-key-a',
materialProof: toolResultKeyMaterialProof('result-key-a', KEY_A),
mutationId: 'result-key-bootstrap-postgres-a',
});
const first = normalizeToolResultKeyCatalogRecord({
...bootstrap.next,
committedAtMs: 1_000,
});
const rotation = createToolResultKeyRotationCommand(first, {
keyId: 'result-key-b',
materialProof: toolResultKeyMaterialProof('result-key-b', KEY_B),
mutationId: 'result-key-rotate-postgres-b',
});
return {
first,
second: normalizeToolResultKeyCatalogRecord({
...rotation.next,
committedAtMs: 1_001,
}),
};
}
function overlayRow(overlay, commandDigest) {
return {
overlayId: overlay.overlayId,
artifactId: overlay.sourceArtifact.artifactId,
sourceBindingDigest: overlay.sourceBindingDigest,
revision: String(overlay.revision),
previousOverlayDigest: overlay.previousOverlayDigest,
fromKeyId: overlay.fromKeyId,
targetCatalogGeneration: String(overlay.targetCatalogFence.generation),
targetCatalogDigest: overlay.targetCatalogFence.catalogDigest,
targetKeyId: overlay.targetCatalogFence.keyId,
targetMaterialProof: overlay.targetCatalogFence.materialProof,
mutationId: overlay.mutationId,
commandDigest,
overlayDigest: overlay.overlayDigest,
rekeyedAtMs: String(overlay.rekeyedAtMs),
overlayJson: jsonb(overlay),
};
}
function receiptRow(receipt, commandDigest) {
return {
receiptDigest: receipt.receiptDigest,
catalogGeneration: String(receipt.catalogGeneration),
catalogDigest: receipt.catalogDigest,
keyId: receipt.keyId,
materialProof: receipt.materialProof,
mutationId: receipt.mutationId,
commandDigest,
bindingCount: String(receipt.bindingCount),
overlayHeadCount: String(receipt.overlayHeadCount),
uncoveredBindingCount: '0',
uncoveredOverlayHeadCount: '0',
coverageDigest: receipt.coverageDigest,
createdAtMs: String(receipt.createdAtMs),
receiptJson: jsonb(receipt),
};
}
function harness(artifact, binding, catalog) {
const calls = [];
let overlayState = null;
let overlayCommandDigest = null;
let head = null;
let receiptState = null;
let receiptCommandDigest = null;
let released = 0;
const client = {
async query(text, values = []) {
calls.push({ text, values });
if (
text.startsWith('BEGIN') ||
text === 'COMMIT' ||
text === 'ROLLBACK' ||
text.includes("set_config('") ||
text.includes('pg_advisory_xact_lock')
) {
return { rows: [] };
}
if (
text.startsWith(
'INSERT INTO "ql3"."tool_execution_result_rekey_overlays"',
)
) {
overlayState = JSON.parse(values[15]);
overlayCommandDigest = values[12];
return { rows: [], rowCount: 1 };
}
if (text.includes('tool_execution_result_rekey_overlays')) {
if (!overlayState) return { rows: [] };
if (text.includes('JOIN "ql3"."tool_execution_result_rekey_heads"')) {
return {
rows:
head && values[0] === overlayState.sourceArtifact.artifactId
? [overlayRow(overlayState, overlayCommandDigest)]
: [],
};
}
const matches =
values.length === 0 ||
values.includes(overlayState.overlayId) ||
values.includes(overlayState.overlayDigest) ||
values.includes(overlayState.mutationId);
return {
rows: matches
? [overlayRow(overlayState, overlayCommandDigest)]
: [],
};
}
if (
text.includes('tool_execution_result_key_bindings') &&
text.includes('tool_execution_completions')
) {
return {
rows: [
{
bindingArtifactDigest: binding.artifactDigest,
bindingKeyId: binding.keyId,
bindingDigest: binding.bindingDigest,
artifactDigest: artifact.artifactDigest,
outputDigest: artifact.outputDigest,
executionResultDigest: artifact.executionResultDigest,
},
],
};
}
if (
text.includes('tool_execution_result_rekey_heads') &&
text.includes('FOR UPDATE')
) {
return {
rows: head
? [
{
revision: String(head.revision),
overlayDigest: head.overlayDigest,
targetKeyId: head.targetKeyId,
},
]
: [],
};
}
if (text.includes('tool_result_key_catalog_generations')) {
return { rows: [{ catalogJson: catalog }] };
}
if (
text.startsWith(
'INSERT INTO "ql3"."tool_execution_result_rekey_heads"',
)
) {
head = {
revision: values[1],
overlayDigest: values[3],
targetKeyId: values[6],
};
return { rows: [], rowCount: 1 };
}
if (text.includes('tool_result_key_retirement_receipts')) {
if (text.startsWith('INSERT')) {
receiptState = JSON.parse(values[12]);
receiptCommandDigest = values[7];
return { rows: [], rowCount: 1 };
}
const matches =
receiptState &&
(values.includes(receiptState.mutationId) ||
values.includes(receiptState.receiptDigest));
return {
rows: matches
? [receiptRow(receiptState, receiptCommandDigest)]
: [],
};
}
if (
text.includes('LEFT JOIN "ql3"."tool_execution_result_rekey_heads"')
) {
return {
rows: [
{
artifactId: artifact.artifactId,
bindingDigest: binding.bindingDigest,
bindingKeyId: binding.keyId,
headOverlayDigest: head.overlayDigest,
headTargetKeyId: head.targetKeyId,
headTargetCatalogGeneration: String(catalog.generation),
headTargetCatalogDigest: catalog.catalogDigest,
},
],
};
}
if (text.includes('clock_timestamp()')) {
return { rows: [{ now: '1700000000000' }] };
}
throw new Error(`unexpected SQL: ${text}`);
},
release() {
released += 1;
},
};
const pool = {
query: (text, values) => client.query(text, values),
connect: async () => client,
};
return {
calls,
reader: new PostgresToolResultRekeyReader(pool),
repository: new PostgresToolResultRekeyRepository(pool),
released: () => released,
};
}
test('serializes PostgreSQL rekey heads and coverage receipts behind the catalog lock', async () => {
const { first, second } = catalogs();
const artifact = sourceArtifact();
const binding = sourceBinding(artifact, first);
const current = harness(artifact, binding, second);
assert.equal(
await current.reader.findHeadByArtifactId(artifact.artifactId),
null,
);
const overlayCommand = createToolExecutionResultRekeyCommand({
artifact,
binding,
previousOverlay: null,
overlayId: 'result-rekey-overlay-postgres-001',
mutationId: 'result-rekey-mutation-postgres-001',
targetCatalogFence: toolResultKeyCatalogFence(
second,
requireActiveToolResultKey(second),
),
targetKey: KEY_B,
output: output(),
rekeyedAtMs: 1_700,
registry: registry(),
nonceFactory: () => Buffer.alloc(12, 5),
});
const appended = await current.repository.append(overlayCommand);
assert.equal(appended.status, 'created');
assert.deepEqual(await current.repository.append(overlayCommand), {
status: 'existing',
overlay: appended.overlay,
});
assert.deepEqual(
await current.reader.findHeadByArtifactId(artifact.artifactId),
appended.overlay,
);
const receiptCommand = createToolResultKeyRetirementReceiptCommand({
expectedCatalogGeneration: second.generation,
expectedCatalogDigest: second.catalogDigest,
keyId: 'result-key-a',
mutationId: 'result-key-retirement-postgres-a',
});
const receipt = await current.repository.create(receiptCommand);
assert.equal(receipt.status, 'created');
assert.equal(receipt.receipt.bindingCount, 1);
assert.equal(receipt.receipt.overlayHeadCount, 1);
assert.deepEqual(await current.repository.create(receiptCommand), {
status: 'existing',
receipt: receipt.receipt,
});
assert.deepEqual(
await current.repository.findByDigest(receipt.receipt.receiptDigest),
receipt.receipt,
);
assert.equal(current.released(), 4);
const statements = current.calls.map(({ text }) => text);
const advisory = statements.indexOf(
'SELECT pg_advisory_xact_lock(190397473, 3)',
);
const overlayInsert = statements.findIndex((text) =>
text.startsWith(
'INSERT INTO "ql3"."tool_execution_result_rekey_overlays"',
),
);
assert.equal(advisory >= 0 && advisory < overlayInsert, true);
const coverageCall = current.calls.find(({ text }) =>
text.includes(
'LEFT JOIN "ql3"."tool_execution_result_rekey_heads" AS head',
),
);
assert.deepEqual(coverageCall.values, ['result-key-a', '', 64]);
const currentCatalogCalls = current.calls.filter(({ text }) =>
text.includes('tool_result_key_catalog_generations'),
);
assert.equal(
currentCatalogCalls.every(({ text }) => text.includes('LIMIT 1')),
true,
);
assert.equal(
current.calls.some(({ text }) => text.includes('FOR UPDATE OF binding')),
false,
);
});
test('rejects a stale PostgreSQL rekey head before writing an overlay', async () => {
const { first, second } = catalogs();
const artifact = sourceArtifact();
const binding = sourceBinding(artifact, first);
const current = harness(artifact, binding, second);
const command = createToolExecutionResultRekeyCommand({
artifact,
binding,
previousOverlay: null,
overlayId: 'result-rekey-overlay-postgres-stale',
mutationId: 'result-rekey-mutation-postgres-stale',
targetCatalogFence: toolResultKeyCatalogFence(
second,
requireActiveToolResultKey(second),
),
targetKey: KEY_B,
output: output(),
rekeyedAtMs: 1_700,
registry: registry(),
nonceFactory: () => Buffer.alloc(12, 6),
});
const drifted = {
...command,
overlay: {
...command.overlay,
fromKeyId: 'result-key-drift',
},
};
assert.throws(
() => current.repository.append(drifted),
/rekey overlay digest does not match/,
);
await assert.rejects(
current.reader.findHeadByArtifactId('../escape'),
/source Artifact id is invalid/,
);
});
@@ -0,0 +1,149 @@
const assert = require('node:assert/strict');
const { createHash } = require('node:crypto');
const { test } = require('node:test');
const {
createToolResultKeyCatalogBootstrapCommand,
createToolResultKeyRotationCommand,
normalizeToolResultKeyCatalogRecord,
toolResultKeyMaterialProof,
} = require('@qinglong/runtime-core/tool-result-key-catalog');
const {
createToolResultKeyRetirementReceiptCommand,
} = require('@qinglong/runtime-core/tool-result-rekey');
const {
PostgresToolResultRekeyRepository,
} = require('@qinglong/cluster-postgres/tool-result-rekey');
const COVERAGE_COUNT = 129;
const PAGE_SIZE = 64;
const KEY_A = Buffer.alloc(32, 1);
const KEY_B = Buffer.alloc(32, 2);
function digest(value) {
return createHash('sha256').update(value).digest('hex');
}
function rotatedCatalog() {
const bootstrap = createToolResultKeyCatalogBootstrapCommand({
keyId: 'result-key-a',
materialProof: toolResultKeyMaterialProof('result-key-a', KEY_A),
mutationId: 'coverage-pressure-bootstrap-a',
});
const first = normalizeToolResultKeyCatalogRecord({
...bootstrap.next,
committedAtMs: 1_000,
});
const rotation = createToolResultKeyRotationCommand(first, {
keyId: 'result-key-b',
materialProof: toolResultKeyMaterialProof('result-key-b', KEY_B),
mutationId: 'coverage-pressure-rotate-b',
});
return normalizeToolResultKeyCatalogRecord({
...rotation.next,
committedAtMs: 1_001,
});
}
function coverageRows(catalog) {
return Array.from({ length: COVERAGE_COUNT }, (_, index) => {
const suffix = String(index).padStart(3, '0');
return {
artifactId: `artifact-coverage-pressure-${suffix}`,
bindingDigest: digest(`binding-${suffix}`),
bindingKeyId: 'result-key-a',
headOverlayDigest: digest(`overlay-${suffix}`),
headTargetKeyId: 'result-key-b',
headTargetCatalogGeneration: String(catalog.generation),
headTargetCatalogDigest: catalog.catalogDigest,
};
});
}
function harness(catalog) {
const rows = coverageRows(catalog);
const coverageCalls = [];
let released = 0;
const client = {
async query(statement, values = []) {
if (
statement.startsWith('BEGIN') ||
statement === 'COMMIT' ||
statement === 'ROLLBACK' ||
statement.includes("set_config('") ||
statement.includes('pg_advisory_xact_lock')
) {
return { rows: [] };
}
if (statement.includes('tool_result_key_retirement_receipts')) {
if (statement.startsWith('INSERT')) {
return { rows: [], rowCount: 1 };
}
return { rows: [] };
}
if (statement.includes('tool_result_key_catalog_generations')) {
return { rows: [{ catalogJson: catalog }] };
}
if (
statement.includes(
'LEFT JOIN "ql3"."tool_execution_result_rekey_heads" AS head',
)
) {
const cursor = values[1];
const limit = values[2];
coverageCalls.push({ cursor, limit });
const start = rows.findIndex((row) => row.artifactId > cursor);
return {
rows: start < 0 ? [] : rows.slice(start, start + limit),
};
}
if (statement.includes('clock_timestamp()')) {
return { rows: [{ now: '1700000000000' }] };
}
throw new Error(`unexpected SQL: ${statement}`);
},
release() {
released += 1;
},
};
const pool = {
connect: async () => client,
query: (statement, values) => client.query(statement, values),
};
return {
coverageCalls,
released: () => released,
repository: new PostgresToolResultRekeyRepository(pool),
};
}
test('streams 129 covered PostgreSQL bindings through three 64-row keyset pages', async () => {
const catalog = rotatedCatalog();
const current = harness(catalog);
const result = await current.repository.create(
createToolResultKeyRetirementReceiptCommand({
expectedCatalogGeneration: catalog.generation,
expectedCatalogDigest: catalog.catalogDigest,
keyId: 'result-key-a',
mutationId: 'coverage-pressure-retirement-receipt',
}),
);
assert.equal(result.status, 'created');
assert.equal(result.receipt.bindingCount, COVERAGE_COUNT);
assert.equal(result.receipt.overlayHeadCount, COVERAGE_COUNT);
assert.equal(result.receipt.uncoveredBindingCount, 0);
assert.equal(result.receipt.uncoveredOverlayHeadCount, 0);
assert.deepEqual(current.coverageCalls, [
{ cursor: '', limit: PAGE_SIZE },
{
cursor: 'artifact-coverage-pressure-063',
limit: PAGE_SIZE,
},
{
cursor: 'artifact-coverage-pressure-127',
limit: PAGE_SIZE,
},
]);
assert.equal(current.released(), 1);
});
@@ -0,0 +1,399 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
WorkerCredentialDeliveryConflictError,
} = require('@qinglong/runtime-core/worker-credential-delivery');
const {
PostgresWorkerCredentialAdministrationRepository,
} = require('../dist/entrypoints/admin');
const MUTATION_ID = '123e4567-e89b-42d3-a456-426614174801';
function credentialCommand() {
return {
expectedCurrentVersion: 0,
credential: {
credentialId: 'worker_generation_2',
version: 1,
state: 'active',
workerId: 'edge-router-1',
secretDigest: 'a'.repeat(64),
createdAtMs: 1_000,
notBeforeAtMs: 1_000,
expiresAtMs: 2_000,
},
mutation: {
mutationId: MUTATION_ID,
operation: 'issue',
credentialId: 'worker_generation_2',
credentialVersion: 1,
expectedPreviousVersion: 0,
changedBy: { type: 'user', id: 'usr_admin' },
createdAtMs: 1_000,
},
audit: {
eventId: MUTATION_ID,
requestId: 'request-worker-delivery-1',
operationId: 'worker_credential.issue',
projectId: null,
subject: { type: 'user', id: 'usr_admin' },
authenticationId: 'session:admin:1',
outcome: 'allowed',
reasons: ['worker_credential_admin'],
fence: null,
occurredAtMs: 1_000,
},
};
}
function delivery(overrides = {}) {
return {
deliveryId: MUTATION_ID,
version: 1,
state: 'credential_committed',
workerId: 'edge-router-1',
credentialId: 'worker_generation_2',
credentialVersion: 1,
previousCredentialId: 'worker_generation_1',
secretDigest: 'a'.repeat(64),
tokenDigest: 'b'.repeat(64),
deploymentTargetDigest: 'c'.repeat(64),
deploymentGeneration: 'secret-generation-2',
stagedAtMs: 1_000,
credentialCommittedAtMs: 1_000,
publishedAtMs: null,
publicationDigest: null,
observedAtMs: null,
observedSessionId: null,
observedSessionVersion: null,
previousRevokedAtMs: null,
...overrides,
};
}
function stageIntent(overrides = {}) {
const { version, state, credentialCommittedAtMs, publishedAtMs,
publicationDigest, observedAtMs, observedSessionId,
observedSessionVersion, previousRevokedAtMs, ...intent } = delivery();
return { ...intent, ...overrides };
}
function fakePool() {
const state = {
audit: null,
credential: null,
mutation: null,
deliveries: [],
discards: [],
calls: [],
releases: 0,
loseFirstCommitResponse: false,
};
function mutationRow() {
if (!state.mutation) return [];
return [{
mutationId: state.mutation.mutationId,
operation: state.mutation.operation,
credentialId: state.mutation.credentialId,
credentialVersion: state.mutation.credentialVersion,
expectedPreviousVersion: state.mutation.expectedPreviousVersion,
changedByType: state.mutation.changedByType,
changedById: state.mutation.changedById,
createdAtMs: state.mutation.createdAtMs,
state: state.credential.state,
workerId: state.credential.workerId,
secretDigest: state.credential.secretDigest,
notBeforeAtMs: state.credential.notBeforeAtMs,
expiresAtMs: state.credential.expiresAtMs,
auditEventId: state.audit.eventId,
auditRequestId: state.audit.requestId,
auditOperationId: state.audit.operationId,
auditProjectId: state.audit.projectId,
auditSubjectType: state.audit.subjectType,
auditSubjectId: state.audit.subjectId,
auditAuthenticationId: state.audit.authenticationId,
auditOutcome: state.audit.outcome,
auditReasons: state.audit.reasons,
auditProjectVersion: state.audit.projectVersion,
auditBindingVersion: state.audit.bindingVersion,
auditOccurredAtMs: state.audit.occurredAtMs,
}];
}
function deliveryRows() {
return state.deliveries.map((value) => ({ ...value }));
}
async function query(text, params = []) {
state.calls.push(text);
if (text === 'COMMIT') {
if (state.loseFirstCommitResponse) {
state.loseFirstCommitResponse = false;
const error = new Error('commit response lost');
error.code = '40001';
throw error;
}
return { rows: [] };
}
if (
text === 'ROLLBACK' ||
text.startsWith('BEGIN') ||
text.includes('set_config') ||
text.includes('pg_advisory_xact_lock')
) return { rows: [] };
if (text.includes('FROM "ql3"."worker_credential_mutations" AS mutation')) {
return { rows: mutationRow() };
}
if (text.startsWith('WITH observation AS (') &&
text.includes('worker_credential_stage_discards')) {
const latest = new Map();
for (const record of state.discards) latest.set(record.deliveryId, record);
const rows = [...latest.values()]
.filter((record) => record.state === 'discard_authorized')
.sort((a, b) => a.deliveryId.localeCompare(b.deliveryId))
.slice(0, params[1])
.map((record) => ({ ...record, observedAtMs: 1_300 }));
return { rows: rows.length > 0 ? rows : [{ deliveryId: null, observedAtMs: 1_300 }] };
}
if (text.includes('FROM "ql3"."worker_credential_stage_discards"')) {
return { rows: state.discards.map((value) => ({ ...value })) };
}
if (text.includes('AS "authorizedAtMs"') && text.includes('mutationExists')) {
return { rows: [{
authorizedAtMs: 1_100,
mutationExists: state.mutation !== null,
deliveryExists: state.deliveries.length > 0,
}] };
}
if (text.includes('AS "discardedAtMs"')) {
return { rows: [{ discardedAtMs: 1_200 }] };
}
if (text.includes('FROM "ql3"."worker_credential_deliveries"')) {
return { rows: deliveryRows() };
}
if (text.includes('FROM "ql3"."worker_credentials"')) {
return {
rows: state.credential
? [{ version: state.credential.version, workerId: state.credential.workerId }]
: [],
};
}
if (text.includes('INSERT INTO "ql3"."security_audit_events"')) {
state.audit = {
eventId: params[0], requestId: params[1], operationId: params[2],
projectId: params[3], subjectType: params[4], subjectId: params[5],
authenticationId: params[6], outcome: params[7],
reasons: JSON.parse(params[8]), projectVersion: params[9],
bindingVersion: params[10], occurredAtMs: params[11],
};
return { rows: [] };
}
if (text.includes('INSERT INTO "ql3"."worker_credentials"')) {
state.credential = {
credentialId: params[0], version: params[1], state: params[2],
workerId: params[3], secretDigest: params[4], createdAtMs: params[5],
notBeforeAtMs: params[6], expiresAtMs: params[7],
};
return { rows: [] };
}
if (text.includes('INSERT INTO "ql3"."worker_credential_mutations"')) {
state.mutation = {
mutationId: params[0], operation: params[1], credentialId: params[2],
credentialVersion: params[3], expectedPreviousVersion: params[4],
changedByType: params[5], changedById: params[6], createdAtMs: params[8],
};
return { rows: [] };
}
if (text.includes('INSERT INTO "ql3"."worker_credential_deliveries"')) {
state.deliveries.push({
deliveryId: params[0], version: params[1], state: params[2],
workerId: params[3], credentialId: params[4], credentialVersion: params[5],
previousCredentialId: params[6], secretDigest: params[7],
tokenDigest: params[8], deploymentTargetDigest: params[9],
deploymentGeneration: params[10], stagedAtMs: params[11],
credentialCommittedAtMs: params[12], publishedAtMs: params[13],
publicationDigest: params[14], observedAtMs: params[15],
observedSessionId: params[16], observedSessionVersion: params[17],
previousRevokedAtMs: params[18],
});
return { rows: [] };
}
if (text.includes('INSERT INTO "ql3"."worker_credential_stage_discards"')) {
state.discards.push({
deliveryId: params[0], version: params[1], state: params[2],
workerId: params[3], credentialId: params[4], credentialVersion: params[5],
previousCredentialId: params[6], secretDigest: params[7],
tokenDigest: params[8], deploymentTargetDigest: params[9],
deploymentGeneration: params[10], stagedAtMs: params[11],
authorizedAtMs: params[12], discardedAtMs: params[13],
});
return { rows: [] };
}
throw new Error(`unexpected SQL: ${text}`);
}
return {
state,
port: {
query,
async connect() {
return {
query,
release() { state.releases += 1; },
};
},
},
};
}
test('commits credential, mutation and delivery v1 atomically then appends publication v2', async () => {
const database = fakePool();
const repository = new PostgresWorkerCredentialAdministrationRepository(database.port);
const committed = await repository.commitDelivered({
credential: credentialCommand(),
delivery: delivery(),
});
assert.equal(committed.status, 'created');
assert.equal(database.state.deliveries.length, 1);
const mutationInsert = database.state.calls.findIndex((sql) =>
sql.includes('INSERT INTO "ql3"."worker_credential_mutations"'));
const deliveryInsert = database.state.calls.findIndex((sql) =>
sql.includes('INSERT INTO "ql3"."worker_credential_deliveries"'));
const commit = database.state.calls.indexOf('COMMIT');
assert.ok(mutationInsert < deliveryInsert && deliveryInsert < commit);
const published = await repository.markPublished({
deliveryId: MUTATION_ID,
expectedVersion: 1,
publicationDigest: 'd'.repeat(64),
publishedAtMs: 1_100,
});
assert.equal(published.state, 'published');
assert.equal(database.state.deliveries.length, 2);
const resolved = await repository.resolveDelivered(MUTATION_ID);
assert.equal(resolved.delivery.version, 2);
assert.equal(resolved.delivery.publicationDigest, 'd'.repeat(64));
const replay = await repository.commitDelivered({
credential: credentialCommand(),
delivery: delivery(),
});
assert.equal(replay.status, 'existing');
assert.equal(database.state.deliveries.length, 2);
assert.ok(database.state.releases >= 3);
});
test('converges a lost commit response and rejects delivery semantic drift', async () => {
const database = fakePool();
database.state.loseFirstCommitResponse = true;
const repository = new PostgresWorkerCredentialAdministrationRepository(database.port);
const replay = await repository.commitDelivered({
credential: credentialCommand(),
delivery: delivery(),
});
assert.equal(replay.status, 'existing');
assert.equal(database.state.deliveries.length, 1);
await assert.rejects(
repository.commitDelivered({
credential: credentialCommand(),
delivery: delivery({ deploymentGeneration: 'other-generation' }),
}),
WorkerCredentialDeliveryConflictError,
);
await repository.markPublished({
deliveryId: MUTATION_ID,
expectedVersion: 1,
publicationDigest: 'd'.repeat(64),
publishedAtMs: 1_100,
});
const publicationReplay = await repository.markPublished({
deliveryId: MUTATION_ID,
expectedVersion: 1,
publicationDigest: 'd'.repeat(64),
publishedAtMs: 1_200,
});
assert.equal(publicationReplay.version, 2);
assert.equal(database.state.deliveries.length, 2);
});
test('rejects a gapped or rewritten append-only delivery history', async () => {
const database = fakePool();
const repository = new PostgresWorkerCredentialAdministrationRepository(database.port);
database.state.deliveries.push(delivery({
version: 2,
state: 'published',
publishedAtMs: 1_100,
publicationDigest: 'd'.repeat(64),
}));
await assert.rejects(
repository.resolveDelivery(MUTATION_ID),
WorkerCredentialDeliveryConflictError,
);
database.state.deliveries.splice(
0,
1,
delivery(),
delivery({
version: 2,
state: 'published',
deploymentGeneration: 'rewritten-generation',
publishedAtMs: 1_100,
publicationDigest: 'd'.repeat(64),
}),
);
await assert.rejects(
repository.resolveDelivery(MUTATION_ID),
WorkerCredentialDeliveryConflictError,
);
});
test('authorizes one exact orphan discard and permanently fences delivery commit', async () => {
const database = fakePool();
const repository = new PostgresWorkerCredentialAdministrationRepository(database.port);
const authorized = await repository.authorizeStageDiscard(stageIntent());
assert.equal(authorized.state, 'discard_authorized');
assert.equal(authorized.authorizedAtMs, 1_100);
assert.equal(
(await repository.authorizeStageDiscard(stageIntent())).version,
1,
);
assert.equal(database.state.discards.length, 1);
const page = await repository.listStageDiscardRecoveryPage({ limit: 1 });
assert.equal(page.discards[0].deliveryId, MUTATION_ID);
assert.equal(page.truncated, false);
const discarded = await repository.markStageDiscarded({
deliveryId: MUTATION_ID,
expectedVersion: 1,
});
assert.equal(discarded.state, 'discarded');
assert.equal(discarded.discardedAtMs, 1_200);
assert.equal(
(await repository.markStageDiscarded({
deliveryId: MUTATION_ID,
expectedVersion: 1,
})).version,
2,
);
await assert.rejects(
repository.commitDelivered({
credential: credentialCommand(),
delivery: delivery(),
}),
WorkerCredentialDeliveryConflictError,
);
});
test('refuses orphan authorization after credential delivery wins', async () => {
const database = fakePool();
const repository = new PostgresWorkerCredentialAdministrationRepository(database.port);
await repository.commitDelivered({
credential: credentialCommand(),
delivery: delivery(),
});
await assert.rejects(
repository.authorizeStageDiscard(stageIntent()),
WorkerCredentialDeliveryConflictError,
);
assert.equal(database.state.discards.length, 0);
});
@@ -0,0 +1,204 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
PostgresWorkerCredentialAdministrationRepository,
} = require('../dist/entrypoints/admin');
const DELIVERY_ID = '123e4567-e89b-42d3-a456-426614174801';
const REVOKE_ID = '123e4567-e89b-42d3-a456-426614174802';
const SESSION_ID = '019f7094-a853-72f3-82ab-dfa08e6bd1c1';
function delivery(version, overrides = {}) {
return {
deliveryId: DELIVERY_ID,
version,
state: version === 1
? 'credential_committed'
: version === 2
? 'published'
: version === 3
? 'observed'
: 'previous_revoked',
workerId: 'edge-router-1',
credentialId: 'worker_generation_2',
credentialVersion: 1,
previousCredentialId: 'worker_generation_1',
secretDigest: 'a'.repeat(64),
tokenDigest: 'b'.repeat(64),
deploymentTargetDigest: 'c'.repeat(64),
deploymentGeneration: 'secret-generation-2',
stagedAtMs: 1_000,
credentialCommittedAtMs: 1_000,
publishedAtMs: version >= 2 ? 1_100 : null,
publicationDigest: version >= 2 ? 'd'.repeat(64) : null,
observedAtMs: version >= 3 ? 1_200 : null,
observedSessionId: version >= 3 ? SESSION_ID : null,
observedSessionVersion: version >= 3 ? 4 : null,
previousRevokedAtMs: version >= 4 ? 1_300 : null,
...overrides,
};
}
function revokeCommand() {
return {
expectedCurrentVersion: 1,
credential: {
credentialId: 'worker_generation_1',
version: 2,
state: 'revoked',
workerId: 'edge-router-1',
secretDigest: '0'.repeat(64),
createdAtMs: 1_300,
notBeforeAtMs: 1_300,
expiresAtMs: 2_300,
},
mutation: {
mutationId: REVOKE_ID,
operation: 'revoke',
credentialId: 'worker_generation_1',
credentialVersion: 2,
expectedPreviousVersion: 1,
changedBy: { type: 'system', id: 'credential-recovery' },
createdAtMs: 1_300,
},
audit: {
eventId: REVOKE_ID,
requestId: `worker-delivery-revoke:${DELIVERY_ID}`,
operationId: 'worker_credential.revoke',
projectId: null,
subject: { type: 'system', id: 'credential-recovery' },
authenticationId: 'service:credential-recovery',
outcome: 'allowed',
reasons: ['worker_credential_admin'],
fence: null,
occurredAtMs: 1_300,
},
};
}
function database() {
const deliveries = [delivery(1), delivery(2), delivery(3)];
const calls = [];
let revoke = null;
let audit = null;
const query = async (text, values = []) => {
const sql = String(text);
calls.push(sql);
if (
sql.startsWith('BEGIN') || sql === 'COMMIT' || sql === 'ROLLBACK' ||
sql.includes('set_config') || sql.includes('pg_advisory_xact_lock')
) return { rows: [] };
if (sql.startsWith('WITH observation AS')) {
const latest = deliveries.at(-1);
if (latest.state === 'previous_revoked') {
return { rows: [{ observedAtMs: '1400', deliveryId: null }] };
}
return { rows: [{
...latest,
observedAtMs: '1400',
observedAtMsDelivery: latest.observedAtMs,
}] };
}
if (sql.includes('FROM "ql3"."worker_credential_mutations" AS mutation')) {
if (!revoke || values[0] !== REVOKE_ID) return { rows: [] };
return { rows: [{
mutationId: revoke.mutation.mutationId,
operation: revoke.mutation.operation,
credentialId: revoke.mutation.credentialId,
credentialVersion: revoke.mutation.credentialVersion,
expectedPreviousVersion: revoke.mutation.expectedPreviousVersion,
changedByType: revoke.mutation.changedBy.type,
changedById: revoke.mutation.changedBy.id,
createdAtMs: revoke.mutation.createdAtMs,
state: revoke.credential.state,
workerId: revoke.credential.workerId,
secretDigest: revoke.credential.secretDigest,
notBeforeAtMs: revoke.credential.notBeforeAtMs,
expiresAtMs: revoke.credential.expiresAtMs,
auditEventId: audit.eventId,
auditRequestId: audit.requestId,
auditOperationId: audit.operationId,
auditProjectId: null,
auditSubjectType: audit.subjectType,
auditSubjectId: audit.subjectId,
auditAuthenticationId: audit.authenticationId,
auditOutcome: audit.outcome,
auditReasons: audit.reasons,
auditProjectVersion: null,
auditBindingVersion: null,
auditOccurredAtMs: audit.occurredAtMs,
}] };
}
if (sql.includes('FROM "ql3"."worker_credential_deliveries"')) {
return { rows: deliveries.map((value) => ({ ...value })) };
}
if (sql.includes('FROM "ql3"."worker_credentials"')) {
return { rows: [{ version: 1, state: 'active', workerId: 'edge-router-1' }] };
}
if (sql.includes('INSERT INTO "ql3"."security_audit_events"')) {
audit = {
eventId: values[0], requestId: values[1], operationId: values[2],
subjectType: values[4], subjectId: values[5],
authenticationId: values[6], outcome: values[7],
reasons: JSON.parse(values[8]), occurredAtMs: values[11],
};
return { rows: [] };
}
if (sql.includes('INSERT INTO "ql3"."worker_credentials"')) {
return { rows: [] };
}
if (sql.includes('INSERT INTO "ql3"."worker_credential_mutations"')) {
revoke = { credential: revokeCommand().credential, mutation: revokeCommand().mutation };
return { rows: [] };
}
if (sql.includes('INSERT INTO "ql3"."worker_credential_deliveries"')) {
deliveries.push(delivery(4));
return { rows: [] };
}
throw new Error(`unexpected SQL: ${sql}`);
};
return {
calls,
deliveries,
port: {
query,
async connect() { return { query, release() {} }; },
},
};
}
test('pages observed delivery and atomically appends revoke plus v4', async () => {
const db = database();
const repository = new PostgresWorkerCredentialAdministrationRepository(db.port);
const page = await repository.listRecoveryPage({ limit: 1 });
assert.equal(page.deliveries[0].state, 'observed');
assert.equal(page.observedAtMs, 1_400);
const result = await repository.revokePreviousDelivered({
credential: revokeCommand(),
delivery: delivery(4),
});
assert.equal(result.status, 'created');
assert.equal(db.deliveries.at(-1).state, 'previous_revoked');
const auditInsert = db.calls.findIndex((sql) =>
sql.includes('INSERT INTO "ql3"."security_audit_events"'));
const credentialInsert = db.calls.findIndex((sql) =>
sql.includes('INSERT INTO "ql3"."worker_credentials"'));
const mutationInsert = db.calls.findIndex((sql) =>
sql.includes('INSERT INTO "ql3"."worker_credential_mutations"'));
const deliveryInsert = db.calls.findIndex((sql) =>
sql.includes('INSERT INTO "ql3"."worker_credential_deliveries"'));
const commit = db.calls.lastIndexOf('COMMIT');
assert.ok(
auditInsert < credentialInsert &&
credentialInsert < mutationInsert &&
mutationInsert < deliveryInsert &&
deliveryInsert < commit,
);
const replay = await repository.revokePreviousDelivered({
credential: revokeCommand(),
delivery: delivery(4),
});
assert.equal(replay.status, 'existing');
assert.equal(db.deliveries.length, 4);
assert.deepEqual((await repository.listRecoveryPage()).deliveries, []);
});
@@ -0,0 +1,140 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
WorkerCredentialManagementPlanConflictError,
WorkerCredentialManagementPlanUnavailableError,
createWorkerCredentialManagementPlan,
} = require('@qinglong/runtime-core/worker-credential-management-plan');
const {
PostgresWorkerCredentialManagementPlanRepository,
} = require('@qinglong/cluster-postgres/worker-credential-management-plan');
function plan(actionRef = 'worker-credential:worker-a:generation-2') {
return createWorkerCredentialManagementPlan({
actionRef,
authorityProjectId: 'cluster-instance-authority',
action: 'rotate',
target: {
deliveryId: '123e4567-e89b-42d3-a456-426614174702',
workerId: 'worker-a',
credentialId: 'credential-b',
previousCredentialId: 'credential-a',
credentialNotBeforeAtMs: 11_000,
credentialExpiresAtMs: 21_000,
deploymentTargetDigest: '1'.repeat(64),
deploymentGeneration: 'generation-2',
},
requestedBy: { type: 'user', id: 'operator-a' },
plannedAtMs: 10_000,
expiresAtMs: 20_000,
});
}
function fixture() {
const stored = new Map();
const queries = [];
const pool = {
async query(text, values) {
queries.push({ text, values });
if (text.includes('INSERT INTO')) {
const actionRef = values[0];
if (stored.has(actionRef)) return { rows: [], rowCount: 0 };
stored.set(actionRef, JSON.parse(values[17]));
return { rows: [{ actionRef }], rowCount: 1 };
}
if (text.includes('SELECT plan_json')) {
const value = stored.get(values[0]);
return { rows: value ? [{ planJson: value }] : [], rowCount: value ? 1 : 0 };
}
throw new Error('unexpected query');
},
};
return {
repository: new PostgresWorkerCredentialManagementPlanRepository(pool),
stored,
queries,
};
}
test('creates and exactly replays one immutable Worker credential plan', async () => {
const state = fixture();
const expected = plan();
const created = await state.repository.create(expected);
const replay = await state.repository.create(expected);
assert.equal(created.status, 'created');
assert.equal(replay.status, 'existing');
assert.deepEqual(created.plan, expected);
assert.deepEqual(replay.plan, expected);
assert.equal(state.stored.size, 1);
const insert = state.queries.find(({ text }) => text.includes('INSERT INTO'));
assert.equal(insert.values.includes('credential-a'), true);
assert.equal(insert.values.some((value) => /ql3w|token/i.test(String(value))), false);
});
test('replays the stored plan when only server-authored time fields differ', async () => {
const state = fixture();
const first = plan();
const replayedRequest = createWorkerCredentialManagementPlan({
actionRef: first.actionRef,
authorityProjectId: first.authorityProjectId,
action: first.action,
target: first.target,
requestedBy: first.requestedBy,
plannedAtMs: first.plannedAtMs + 500,
expiresAtMs: first.expiresAtMs + 500,
});
assert.notEqual(replayedRequest.planDigest, first.planDigest);
await state.repository.create(first);
const replay = await state.repository.create(replayedRequest);
assert.equal(replay.status, 'existing');
assert.deepEqual(replay.plan, first);
assert.equal(state.stored.size, 1);
});
test('rejects an actionRef replay bound to different plan content', async () => {
const state = fixture();
const first = plan();
await state.repository.create(first);
const changed = createWorkerCredentialManagementPlan({
actionRef: first.actionRef,
authorityProjectId: first.authorityProjectId,
action: first.action,
target: {
...first.target,
deploymentGeneration: 'generation-3',
},
requestedBy: first.requestedBy,
plannedAtMs: first.plannedAtMs,
expiresAtMs: first.expiresAtMs,
});
await assert.rejects(
state.repository.create(changed),
WorkerCredentialManagementPlanConflictError,
);
});
test('maps malformed durable JSON and storage errors to unavailable', async () => {
const malformed = {
async query() {
return { rows: [{ planJson: { schema: 'wrong' } }], rowCount: 1 };
},
};
await assert.rejects(
new PostgresWorkerCredentialManagementPlanRepository(malformed)
.findByActionRef('worker-credential:worker-a:generation-2'),
WorkerCredentialManagementPlanUnavailableError,
);
const failed = {
async query() { throw new Error('sensitive database failure'); },
};
await assert.rejects(
new PostgresWorkerCredentialManagementPlanRepository(failed)
.findByActionRef('worker-credential:worker-a:generation-2'),
(error) => {
assert.ok(error instanceof WorkerCredentialManagementPlanUnavailableError);
assert.doesNotMatch(error.message, /sensitive|database failure/i);
return true;
},
);
});
@@ -0,0 +1,85 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
PostgresWorkerCredentialManagementQuotaRepository,
} = require('@qinglong/cluster-postgres/worker-credential-manager');
const command = Object.freeze({
projectId: 'cluster-authority',
subject: Object.freeze({ type: 'user', id: 'operator-a' }),
operation: 'worker-credential.plan',
idempotencyKey: 'worker-credential:worker-a:generation-2',
});
test('uses one database-clock UPSERT and exact replay receipt', async () => {
const calls = [];
const repository = new PostgresWorkerCredentialManagementQuotaRepository(
{
async query(text, values) {
calls.push({ text, values });
return {
rows: [{
admitted: true,
consumedCount: 1,
resetAtMs: 60_000,
observedAtMs: 1_000,
}],
};
},
},
{ windowMs: 60_000, limits: { 'worker-credential.plan': 1 } },
);
assert.deepEqual(await repository.consume(command), {
admitted: true,
retryAfterMs: null,
});
assert.equal(calls.length, 1);
assert.match(calls[0].text, /clock_timestamp\(\)/);
assert.match(calls[0].text, /receipt_ids \? \$5::text/);
assert.deepEqual(calls[0].values.slice(0, 5), [
command.projectId,
'user',
'operator-a',
command.operation,
command.idempotencyKey,
]);
});
test('returns one bounded durable rejection without application-clock decisions', async () => {
let calls = 0;
const repository = new PostgresWorkerCredentialManagementQuotaRepository({
async query(text) {
calls += 1;
if (text.includes('INSERT INTO')) return { rows: [] };
return {
rows: [{
admitted: false,
consumedCount: 30,
resetAtMs: 60_000,
observedAtMs: 59_000,
}],
};
},
});
assert.deepEqual(await repository.consume(command), {
admitted: false,
retryAfterMs: 1_000,
});
assert.equal(calls, 2);
});
test('rejects widened operations and configuration before PostgreSQL', async () => {
let queries = 0;
const pool = { async query() { queries += 1; return { rows: [] }; } };
const repository = new PostgresWorkerCredentialManagementQuotaRepository(pool);
await assert.rejects(
repository.consume({ ...command, operation: 'worker-credential.execute' }),
TypeError,
);
assert.equal(queries, 0);
assert.throws(
() => new PostgresWorkerCredentialManagementQuotaRepository(pool, { windowMs: 999 }),
TypeError,
);
});
@@ -0,0 +1,65 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
WorkerCredentialUnavailableError,
} = require('@qinglong/runtime-core/worker-credential');
const {
PostgresWorkerCredentialRepository,
} = require('../dist/entrypoints/workerIngress');
function row(overrides = {}) {
return {
credentialId: 'worker_primary',
version: '2',
state: 'active',
workerId: 'edge-1',
secretDigest: 'a'.repeat(64),
createdAtMs: '100',
notBeforeAtMs: '100',
expiresAtMs: '1000',
...overrides,
};
}
test('resolves only the latest normalized Worker credential', async () => {
const calls = [];
const repository = new PostgresWorkerCredentialRepository({
async query(text, values) {
calls.push({ text, values });
return { rows: [row(), row({ version: '1' })] };
},
});
assert.deepEqual(await repository.resolve('worker_primary'), {
credentialId: 'worker_primary',
version: 2,
state: 'active',
workerId: 'edge-1',
secretDigest: 'a'.repeat(64),
createdAtMs: 100,
notBeforeAtMs: 100,
expiresAtMs: 1000,
});
assert.deepEqual(calls[0].values, ['worker_primary']);
assert.match(calls[0].text, /ORDER BY version DESC/);
assert.match(calls[0].text, /LIMIT 2/);
});
test('returns null for absence and fails closed on corrupt or unordered rows', async () => {
const empty = new PostgresWorkerCredentialRepository({
async query() { return { rows: [] }; },
});
assert.equal(await empty.resolve('worker_primary'), null);
for (const rows of [
[row({ secretDigest: 'corrupt' })],
[row({ version: '1' }), row({ version: '2' })],
]) {
const repository = new PostgresWorkerCredentialRepository({
async query() { return { rows }; },
});
await assert.rejects(
repository.resolve('worker_primary'),
WorkerCredentialUnavailableError,
);
}
await assert.rejects(empty.resolve('../escape'), TypeError);
});
@@ -0,0 +1,174 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
PostgresWorkerSessionRepository,
} = require('@qinglong/cluster-postgres/worker-ingress');
const {
WorkerCredentialDeliveryUnavailableError,
} = require('@qinglong/runtime-core/worker-credential-delivery');
const SESSION_ID = '019f7094-a853-72f3-82ab-dfa08e6bd1c1';
const DELIVERY_ID = '123e4567-e89b-42d3-a456-426614174601';
const CAPABILITIES_HASH =
'44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a';
function session(version, overrides = {}) {
return {
workerId: 'edge-router-1',
sessionId: SESSION_ID,
generation: 2,
status: 'online',
version,
capabilitiesJson: '{}',
capabilitiesHash: CAPABILITIES_HASH,
maxConcurrentRuns: 2,
availableSlots: 1,
registeredAtMs: '900',
lastHeartbeatAtMs: '1000',
leaseExpiresAtMs: '5000',
updatedAtMs: '1000',
...overrides,
};
}
function delivery(version, overrides = {}) {
const published = version >= 2;
const observed = version >= 3;
return {
deliveryId: DELIVERY_ID,
version,
state: version === 1
? 'credential_committed'
: version === 2
? 'published'
: 'observed',
workerId: 'edge-router-1',
credentialId: 'worker_generation_2',
credentialVersion: 1,
previousCredentialId: 'worker_generation_1',
secretDigest: 'a'.repeat(64),
tokenDigest: 'b'.repeat(64),
deploymentTargetDigest: 'c'.repeat(64),
deploymentGeneration: 'secret-generation-2',
stagedAtMs: '800',
credentialCommittedAtMs: '850',
publishedAtMs: published ? '900' : null,
publicationDigest: published ? 'd'.repeat(64) : null,
observedAtMs: observed ? '1100' : null,
observedSessionId: observed ? SESSION_ID : null,
observedSessionVersion: observed ? 4 : null,
previousRevokedAtMs: null,
...overrides,
};
}
function database(deliveryRows) {
const events = [];
let released = 0;
const client = {
async query(text, values) {
const sql = String(text);
if (sql === 'BEGIN' || sql === 'COMMIT' || sql === 'ROLLBACK') {
events.push(sql);
return { rows: [] };
}
if (sql.startsWith('SET LOCAL')) return { rows: [] };
if (sql.includes('FROM "ql3"."worker_sessions"') && sql.includes('FOR UPDATE')) {
events.push('lock-session');
return { rows: [session(3)] };
}
if (sql.includes('statement_timestamp()')) {
events.push('database-time');
return { rows: [{ observedAtMs: '1200' }] };
}
if (sql.includes('UPDATE "ql3"."worker_sessions"')) {
events.push('update-session');
return { rows: [session(4, {
lastHeartbeatAtMs: '1200',
leaseExpiresAtMs: '31200',
updatedAtMs: '1200',
})] };
}
if (sql.includes('FROM "ql3"."worker_credential_deliveries"')) {
assert.equal(sql.includes('FOR UPDATE'), false);
events.push(`read-delivery:${values.join(':')}`);
return { rows: deliveryRows };
}
if (sql.includes('INSERT INTO "ql3"."worker_credential_deliveries"')) {
events.push(`insert-observation:${values[1]}:${values[2]}`);
assert.equal(values[15], 1200);
assert.equal(values[16], SESSION_ID);
assert.equal(values[17], 4);
return { rows: [] };
}
throw new Error(`unexpected query: ${sql}`);
},
release() { released += 1; },
};
return {
pool: {
async connect() { return client; },
async query() { throw new Error('not used'); },
},
events,
released: () => released,
};
}
function heartbeat(repository) {
return repository.heartbeatAuthenticated(
{
workerId: 'edge-router-1',
sessionId: SESSION_ID,
generation: 2,
expectedVersion: 3,
availableSlots: 1,
leaseDurationMs: 30_000,
},
{
workerId: 'edge-router-1',
credentialId: 'worker_generation_2',
credentialVersion: 1,
},
);
}
test('appends authenticated observation in the Session heartbeat transaction', async () => {
const db = database([delivery(1), delivery(2)]);
const worker = await heartbeat(new PostgresWorkerSessionRepository(db.pool));
assert.equal(worker.version, 4);
assert.deepEqual(db.events, [
'BEGIN',
'lock-session',
'database-time',
'update-session',
'read-delivery:edge-router-1:worker_generation_2:1',
'insert-observation:3:observed',
'COMMIT',
]);
assert.equal(db.released(), 1);
});
test('rolls Session mutation back until the exact delivery is published', async () => {
const db = database([delivery(1)]);
await assert.rejects(
heartbeat(new PostgresWorkerSessionRepository(db.pool)),
WorkerCredentialDeliveryUnavailableError,
);
assert.equal(db.events.includes('COMMIT'), false);
assert.equal(db.events.at(-1), 'ROLLBACK');
assert.equal(db.released(), 1);
});
test('replays an existing observation and ignores credentials outside delivery', async () => {
for (const rows of [[delivery(1), delivery(2), delivery(3)], []]) {
const db = database(rows);
const worker = await heartbeat(new PostgresWorkerSessionRepository(db.pool));
assert.equal(worker.version, 4);
assert.equal(
db.events.some((event) => event.startsWith('insert-observation:')),
false,
);
assert.equal(db.events.at(-1), 'COMMIT');
}
});
@@ -0,0 +1,90 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
PostgresWorkerSessionRepository,
} = require('@qinglong/cluster-postgres/worker-ingress');
const SESSION_ID = '019f7094-a853-72f3-82ab-dfa08e6bd1c1';
const CAPABILITIES_HASH =
'44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a';
test('pins the shared transition status parameter to PostgreSQL varchar', async () => {
const events = [];
const session = {
workerId: 'edge-router-1',
sessionId: SESSION_ID,
generation: 2,
status: 'online',
version: 3,
capabilitiesJson: '{}',
capabilitiesHash: CAPABILITIES_HASH,
maxConcurrentRuns: 2,
availableSlots: 1,
registeredAtMs: '900',
lastHeartbeatAtMs: '1000',
leaseExpiresAtMs: '5000',
updatedAtMs: '1000',
};
const client = {
async query(text) {
const sql = String(text);
if (sql === 'BEGIN' || sql === 'COMMIT' || sql === 'ROLLBACK') {
events.push(sql);
return { rows: [] };
}
if (sql.startsWith('SET LOCAL')) return { rows: [] };
if (
sql.includes('FROM "ql3"."worker_sessions"') &&
sql.includes('FOR UPDATE')
) {
return { rows: [session] };
}
if (sql.includes('statement_timestamp()')) {
return { rows: [{ observedAtMs: '1200' }] };
}
if (sql.includes('UPDATE "ql3"."worker_sessions"')) {
assert.match(sql, /status = \$5::varchar/);
assert.match(sql, /WHEN \$5::varchar = 'offline'/);
return {
rows: [{
...session,
status: 'draining',
version: 4,
availableSlots: 0,
updatedAtMs: '1200',
}],
};
}
if (sql.includes('FROM "ql3"."worker_credential_deliveries"')) {
return { rows: [] };
}
throw new Error(`unexpected query: ${sql}`);
},
release() {},
};
const repository = new PostgresWorkerSessionRepository({
async connect() {
return client;
},
async query() {
throw new Error('not used');
},
});
const transitioned = await repository.transitionAuthenticated(
{
workerId: 'edge-router-1',
sessionId: SESSION_ID,
generation: 2,
expectedVersion: 3,
status: 'draining',
},
{
workerId: 'edge-router-1',
credentialId: 'worker_generation_2',
credentialVersion: 1,
},
);
assert.equal(transitioned.status, 'draining');
assert.equal(transitioned.version, 4);
assert.deepEqual(events, ['BEGIN', 'COMMIT']);
});
@@ -0,0 +1,163 @@
const assert = require('node:assert/strict');
const { createHash } = require('node:crypto');
const { test } = require('node:test');
const {
PostgresRunDispatchLeaseRepository,
} = require('../dist/remote-execution/runDispatchLeaseRepository');
const SESSION_ID = '018f5c64-9b9d-7f1a-8c2d-1234567890ac';
const LEASE_TOKEN = 'workflow_task_lease_capability_0000000000000001';
const LEASE_DIGEST = createHash('sha256')
.update(LEASE_TOKEN)
.digest('hex');
test('leases a Workflow Task Attempt without changing the aggregate Run status', async () => {
const calls = [];
const client = {
async query(sql, params = []) {
const normalized = sql.replace(/\s+/g, ' ').trim();
calls.push({ sql: normalized, params });
if (
normalized === 'BEGIN' ||
normalized.startsWith('SET LOCAL') ||
normalized === 'COMMIT' ||
normalized === 'ROLLBACK' ||
normalized.startsWith('SELECT pg_advisory_xact_lock')
) {
return { rows: [], rowCount: 0 };
}
if (normalized.includes('FROM "ql3"."worker_sessions"')) {
return {
rows: [{
workerId: 'edge-1',
sessionId: SESSION_ID,
generation: 2,
status: 'online',
maxConcurrentRuns: 4,
availableSlots: 2,
leaseExpiresAtMs: 20_000,
}],
rowCount: 1,
};
}
if (
normalized.includes('INNER JOIN "ql3"."run_attempts"') &&
normalized.includes('workflow_task')
) {
return {
rows: [{
runId: 'workflow-run-1',
runStatus: 'running',
executionOwner: 'runtime',
cancelRequestedAtMs: null,
runVersion: 7,
eventSequence: 7,
attemptId: 'workflow-attempt-1',
attemptStatus: 'claimed',
attemptRunId: 'workflow-run-1',
attemptStepRunId: 'workflow-step-1',
workflowAttemptId: 'workflow-attempt-1',
workflowStepRunId: 'workflow-step-1',
admittedWorkflowStepVersion: 2,
admittedWorkflowStepDigest: 'a'.repeat(64),
}],
rowCount: 1,
};
}
if (
normalized.includes('FROM "ql3"."step_runs"') &&
normalized.includes('FOR UPDATE')
) {
return {
rows: [{
workflowStepStatus: 'ready',
workflowStepVersion: 2,
workflowStepDigest: 'a'.repeat(64),
}],
rowCount: 1,
};
}
if (
normalized.startsWith('SELECT') &&
normalized.includes('FROM "ql3"."run_dispatch_leases"') &&
normalized.includes('WHERE attempt_id = $1 FOR UPDATE')
) {
return { rows: [], rowCount: 0 };
}
if (normalized.includes('statement_timestamp()')) {
return { rows: [{ nowMs: 10_000 }], rowCount: 1 };
}
if (normalized.includes('count(*)::integer AS "activeCount"')) {
return { rows: [{ activeCount: 0 }], rowCount: 1 };
}
if (
normalized.startsWith(
'INSERT INTO "ql3"."run_dispatch_leases"',
)
) {
return {
rows: [{
attemptId: 'workflow-attempt-1',
runId: 'workflow-run-1',
status: 'leased',
version: 0,
leaseGeneration: 1,
workerId: 'edge-1',
workerSessionId: SESSION_ID,
workerGeneration: 2,
leaseTokenDigest: LEASE_DIGEST,
acquiredAtMs: 10_000,
renewedAtMs: 10_000,
expiresAtMs: 40_000,
releasedAtMs: null,
releaseReason: null,
completedAtMs: null,
updatedAtMs: 10_000,
}],
rowCount: 1,
};
}
if (
normalized.startsWith('UPDATE ') ||
normalized.startsWith('INSERT INTO ')
) {
return { rows: [], rowCount: 1 };
}
throw new Error(`Unexpected SQL: ${normalized}`);
},
release() {
calls.push({ sql: 'RELEASE', params: [] });
},
};
const repository = new PostgresRunDispatchLeaseRepository({
async connect() {
return client;
},
});
const result = await repository.claim({
runId: 'workflow-run-1',
attemptId: 'workflow-attempt-1',
workerId: 'edge-1',
workerSessionId: SESSION_ID,
workerGeneration: 2,
leaseToken: LEASE_TOKEN,
leaseDurationMs: 30_000,
eventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a1',
offerId: 'workflow-offer-1',
});
assert.equal(result.status, 'claimed');
const runUpdate = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."runs"'));
assert.equal(runUpdate.params[1], 'running');
const event = calls.find(
({ sql, params }) =>
sql.startsWith('INSERT INTO "ql3"."run_events"') &&
params[3] === 'workflow.task_dispatch_leased',
);
assert.equal(event.params[7], 'workflow-step-1');
assert.match(event.params[8], /"execution_scope":"workflow_task"/);
assert.equal(calls.at(-2).sql, 'COMMIT');
assert.equal(calls.at(-1).sql, 'RELEASE');
});