mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 18:08:20 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
ApiCredentialUnavailableError,
|
||||
InvalidApiCredentialValueError,
|
||||
} = require('@qinglong/runtime-core/api-credential');
|
||||
const {
|
||||
PostgresApiCredentialRepository,
|
||||
} = require('@qinglong/cluster-postgres/runtime');
|
||||
|
||||
function row(overrides = {}) {
|
||||
return {
|
||||
credentialId: 'app_primary',
|
||||
version: '2',
|
||||
state: 'active',
|
||||
subjectType: 'api_app',
|
||||
subjectId: 'app_primary',
|
||||
subjectStatus: 'active',
|
||||
pepperKeyId: 'legacy-v1',
|
||||
secretDigest: 'a'.repeat(64),
|
||||
createdAtMs: '100',
|
||||
notBeforeAtMs: '100',
|
||||
expiresAtMs: '1000',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('resolves only the latest normalized credential and stable subject', async () => {
|
||||
const calls = [];
|
||||
const repository = new PostgresApiCredentialRepository({
|
||||
async query(sql, values) {
|
||||
calls.push({ sql, values });
|
||||
return { rows: [row()] };
|
||||
},
|
||||
});
|
||||
assert.deepEqual(await repository.resolve('app_primary'), {
|
||||
credentialId: 'app_primary',
|
||||
version: 2,
|
||||
pepperKeyId: 'legacy-v1',
|
||||
state: 'active',
|
||||
subject: { type: 'api_app', id: 'app_primary' },
|
||||
subjectStatus: 'active',
|
||||
secretDigest: 'a'.repeat(64),
|
||||
createdAtMs: 100,
|
||||
notBeforeAtMs: 100,
|
||||
expiresAtMs: 1000,
|
||||
});
|
||||
assert.deepEqual(calls[0].values, ['app_primary']);
|
||||
assert.match(calls[0].sql, /ORDER BY credential\.version DESC/);
|
||||
assert.match(calls[0].sql, /identity_subjects/);
|
||||
});
|
||||
|
||||
test('returns null for unknown credentials and rejects invalid ids before SQL', async () => {
|
||||
let calls = 0;
|
||||
const repository = new PostgresApiCredentialRepository({
|
||||
async query() {
|
||||
calls += 1;
|
||||
return { rows: [] };
|
||||
},
|
||||
});
|
||||
assert.equal(await repository.resolve('unknown'), null);
|
||||
await assert.rejects(
|
||||
repository.resolve('../escape'),
|
||||
InvalidApiCredentialValueError,
|
||||
);
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
|
||||
test('fails closed on corrupt rows and database errors', async () => {
|
||||
for (const query of [
|
||||
async () => ({ rows: [row({ secretDigest: 'corrupt' })] }),
|
||||
async () => {
|
||||
throw new Error('driver detail');
|
||||
},
|
||||
]) {
|
||||
const repository = new PostgresApiCredentialRepository({ query });
|
||||
await assert.rejects(
|
||||
repository.resolve('app_primary'),
|
||||
ApiCredentialUnavailableError,
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,168 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
approvalRequestDigest,
|
||||
createApprovalRequest,
|
||||
} = require('@qinglong/runtime-core/approved-action');
|
||||
const {
|
||||
createToolInvocationPreviewArtifact,
|
||||
} = require('@qinglong/runtime-core/tool-invocation-artifact');
|
||||
const {
|
||||
PostgresApprovalRequestSource,
|
||||
} = require('@qinglong/cluster-postgres/approval-discovery');
|
||||
|
||||
function request(id, atMs) {
|
||||
return createApprovalRequest({
|
||||
id,
|
||||
projectId: 'default',
|
||||
action: {
|
||||
permission: 'run.start',
|
||||
actionType: 'tool.invoke',
|
||||
actionRef: `tool:${id}`,
|
||||
actionDigest: 'a'.repeat(64),
|
||||
previewDigest: 'b'.repeat(64),
|
||||
},
|
||||
risk: 'medium',
|
||||
decisionMode: 'separation_of_duty',
|
||||
requestedBy: { type: 'agent', id: 'agent-planner' },
|
||||
requestedAtMs: atMs,
|
||||
expiresAtMs: atMs + 60_000,
|
||||
requestFence: { projectVersion: 1, bindingVersion: 1 },
|
||||
});
|
||||
}
|
||||
|
||||
function row(value) {
|
||||
return {
|
||||
requestJson: value,
|
||||
requestDigest: approvalRequestDigest(value),
|
||||
updatedAtMs: value.requestedAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
test('uses the Project descending keyset and returns limit plus one', async () => {
|
||||
const calls = [];
|
||||
const values = [request('approval-3', 3_000), request('approval-2', 2_000)];
|
||||
const source = new PostgresApprovalRequestSource({
|
||||
async query(sql, parameters) {
|
||||
calls.push({ sql, parameters });
|
||||
return { rows: values.map(row) };
|
||||
},
|
||||
});
|
||||
const page = await source.listApprovalRequests({
|
||||
projectId: 'default',
|
||||
limit: 1,
|
||||
after: { updatedAtMs: 4_000, requestId: 'approval-4' },
|
||||
});
|
||||
assert.deepEqual(calls[0].parameters, [
|
||||
'default',
|
||||
4_000,
|
||||
'approval-4',
|
||||
2,
|
||||
]);
|
||||
assert.match(calls[0].sql, /ORDER BY updated_at_ms DESC, request_id DESC/);
|
||||
assert.deepEqual(page.requests.map(({ id }) => id), ['approval-3']);
|
||||
assert.equal(page.truncated, true);
|
||||
assert.deepEqual(page.next, {
|
||||
updatedAtMs: 3_000,
|
||||
requestId: 'approval-3',
|
||||
});
|
||||
});
|
||||
|
||||
test('fails closed on malformed rows and database errors', async () => {
|
||||
const value = request('approval-1', 1_000);
|
||||
for (const pool of [
|
||||
{ async query() { return { rows: [{ ...row(value), updatedAtMs: 2_000 }] }; } },
|
||||
{ async query() { throw new Error('private database detail'); } },
|
||||
]) {
|
||||
await assert.rejects(
|
||||
new PostgresApprovalRequestSource(pool).listApprovalRequests({
|
||||
projectId: 'default',
|
||||
limit: 1,
|
||||
}),
|
||||
{ code: 'APPROVAL_UNAVAILABLE' },
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('reads one Approval detail through the exact Project and Artifact binding', async () => {
|
||||
const previewArtifact = createToolInvocationPreviewArtifact({
|
||||
artifactId: 'preview-1',
|
||||
projectId: 'default',
|
||||
actionRef: 'tool:approval-1',
|
||||
actionDigest: 'c'.repeat(64),
|
||||
redactionContractDigest: 'd'.repeat(64),
|
||||
sealedAtMs: 1_000,
|
||||
preview: {
|
||||
title: 'Run task',
|
||||
summary: 'Runs one selected task.',
|
||||
fields: [{ kind: 'redacted', label: 'Token', value: null }],
|
||||
warnings: ['external_effect'],
|
||||
},
|
||||
});
|
||||
const approval = createApprovalRequest({
|
||||
id: 'approval-1',
|
||||
projectId: 'default',
|
||||
action: {
|
||||
permission: 'run.start',
|
||||
actionType: 'tool.invoke',
|
||||
actionRef: previewArtifact.actionRef,
|
||||
actionDigest: previewArtifact.actionDigest,
|
||||
previewDigest: previewArtifact.previewDigest,
|
||||
},
|
||||
risk: 'medium',
|
||||
decisionMode: 'human_confirmation',
|
||||
requestedBy: { type: 'agent', id: 'agent-planner' },
|
||||
requestedAtMs: 1_000,
|
||||
expiresAtMs: 61_000,
|
||||
requestFence: { projectVersion: 1, bindingVersion: 1 },
|
||||
});
|
||||
const detailRow = {
|
||||
...row(approval),
|
||||
previewArtifactId: previewArtifact.artifactId,
|
||||
previewProjectId: previewArtifact.projectId,
|
||||
previewActionRef: previewArtifact.actionRef,
|
||||
previewActionDigest: previewArtifact.actionDigest,
|
||||
storedPreviewDigest: previewArtifact.previewDigest,
|
||||
redactionContractDigest: previewArtifact.redactionContractDigest,
|
||||
previewArtifactDigest: previewArtifact.artifactDigest,
|
||||
previewByteLength: previewArtifact.byteLength,
|
||||
previewSealedAtMs: previewArtifact.sealedAtMs,
|
||||
previewArtifactJson: previewArtifact,
|
||||
};
|
||||
const calls = [];
|
||||
const source = new PostgresApprovalRequestSource({
|
||||
async query(sql, parameters) {
|
||||
calls.push({ sql, parameters });
|
||||
return { rows: [detailRow] };
|
||||
},
|
||||
});
|
||||
const detail = await source.getApprovalRequestDetail({
|
||||
projectId: 'default',
|
||||
requestId: 'approval-1',
|
||||
});
|
||||
assert.deepEqual(calls[0].parameters, ['default', 'approval-1']);
|
||||
assert.match(calls[0].sql, /LEFT JOIN "ql3"\."tool_invocation_preview_artifacts"/);
|
||||
assert.equal(detail.request.id, 'approval-1');
|
||||
assert.equal(detail.preview.title, 'Run task');
|
||||
await assert.rejects(
|
||||
new PostgresApprovalRequestSource({
|
||||
async query() {
|
||||
return { rows: [{ ...detailRow, previewByteLength: 1 }] };
|
||||
},
|
||||
}).getApprovalRequestDetail({ projectId: 'default', requestId: 'approval-1' }),
|
||||
{ code: 'APPROVAL_UNAVAILABLE' },
|
||||
);
|
||||
});
|
||||
|
||||
test('exports only through the read authority subpath', () => {
|
||||
const root = require('@qinglong/cluster-postgres');
|
||||
const mutation = require('@qinglong/cluster-postgres/approved-action');
|
||||
const discovery = require('@qinglong/cluster-postgres/approval-discovery');
|
||||
assert.equal(root.PostgresApprovalRequestSource, undefined);
|
||||
assert.equal(mutation.PostgresApprovalRequestSource, undefined);
|
||||
assert.equal(
|
||||
discovery.PostgresApprovalRequestSource,
|
||||
PostgresApprovalRequestSource,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
POSTGRES_CA_MAX_CERTIFICATES,
|
||||
POSTGRES_CA_MAX_FILE_BYTES,
|
||||
PostgresCertificateAuthorityFileError,
|
||||
inspectPostgresCertificateAuthorityFile,
|
||||
loadPostgresCertificateAuthorityFile,
|
||||
} = require('@qinglong/cluster-postgres/runtime');
|
||||
|
||||
const FIXTURES = path.resolve(
|
||||
__dirname,
|
||||
'../../ql3-cluster-control/test/fixtures/mtls',
|
||||
);
|
||||
const CA = fs.readFileSync(path.join(FIXTURES, 'ca-cert.pem'), 'utf8');
|
||||
|
||||
function temporaryDirectory(t) {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-pg-ca-'));
|
||||
t.after(() => fs.rmSync(directory, { recursive: true, force: true }));
|
||||
return directory;
|
||||
}
|
||||
|
||||
function expectCode(callback, code) {
|
||||
assert.throws(
|
||||
callback,
|
||||
(error) =>
|
||||
error instanceof PostgresCertificateAuthorityFileError &&
|
||||
error.code === code,
|
||||
);
|
||||
}
|
||||
|
||||
test('loads a bounded CA bundle through a projected-Secret symlink', (t) => {
|
||||
const directory = temporaryDirectory(t);
|
||||
const dataDirectory = path.join(directory, '..data');
|
||||
fs.mkdirSync(dataDirectory);
|
||||
fs.writeFileSync(path.join(dataDirectory, 'ca.crt'), CA, { mode: 0o444 });
|
||||
fs.symlinkSync(path.join('..data', 'ca.crt'), path.join(directory, 'ca.crt'));
|
||||
|
||||
const bundle = loadPostgresCertificateAuthorityFile(
|
||||
path.join(directory, 'ca.crt'),
|
||||
);
|
||||
assert.match(bundle, /^-----BEGIN CERTIFICATE-----/);
|
||||
assert.match(bundle, /-----END CERTIFICATE-----\n$/);
|
||||
|
||||
const inspection = inspectPostgresCertificateAuthorityFile(
|
||||
path.join(directory, 'ca.crt'),
|
||||
);
|
||||
assert.equal(inspection.bundle, bundle);
|
||||
assert.equal(inspection.fingerprints256.length, 1);
|
||||
assert.match(
|
||||
inspection.fingerprints256[0],
|
||||
/^(?:[0-9A-F]{2}:){31}[0-9A-F]{2}$/,
|
||||
);
|
||||
assert.equal(Object.isFrozen(inspection), true);
|
||||
assert.equal(Object.isFrozen(inspection.fingerprints256), true);
|
||||
});
|
||||
|
||||
test('rejects ambiguous paths, file types, permissions and sizes', (t) => {
|
||||
const directory = temporaryDirectory(t);
|
||||
expectCode(
|
||||
() => loadPostgresCertificateAuthorityFile('relative-ca.pem'),
|
||||
'QL3_POSTGRES_CA_INVALID_PATH',
|
||||
);
|
||||
expectCode(
|
||||
() =>
|
||||
loadPostgresCertificateAuthorityFile(path.join(directory, 'missing.pem')),
|
||||
'QL3_POSTGRES_CA_UNAVAILABLE',
|
||||
);
|
||||
expectCode(
|
||||
() => loadPostgresCertificateAuthorityFile(directory),
|
||||
'QL3_POSTGRES_CA_NOT_REGULAR',
|
||||
);
|
||||
|
||||
const writable = path.join(directory, 'writable.pem');
|
||||
fs.writeFileSync(writable, CA, { mode: 0o666 });
|
||||
fs.chmodSync(writable, 0o666);
|
||||
expectCode(
|
||||
() => loadPostgresCertificateAuthorityFile(writable),
|
||||
'QL3_POSTGRES_CA_INSECURE_PERMISSIONS',
|
||||
);
|
||||
|
||||
const oversized = path.join(directory, 'oversized.pem');
|
||||
fs.writeFileSync(oversized, Buffer.alloc(POSTGRES_CA_MAX_FILE_BYTES + 1), {
|
||||
mode: 0o444,
|
||||
});
|
||||
expectCode(
|
||||
() => loadPostgresCertificateAuthorityFile(oversized),
|
||||
'QL3_POSTGRES_CA_INVALID_SIZE',
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects malformed, non-CA, duplicate and oversized bundles', (t) => {
|
||||
const directory = temporaryDirectory(t);
|
||||
const malformed = path.join(directory, 'malformed.pem');
|
||||
fs.writeFileSync(malformed, `${CA}\nunreviewed`, { mode: 0o444 });
|
||||
expectCode(
|
||||
() => loadPostgresCertificateAuthorityFile(malformed),
|
||||
'QL3_POSTGRES_CA_INVALID_PEM',
|
||||
);
|
||||
|
||||
const nonCa = path.join(directory, 'non-ca.pem');
|
||||
fs.copyFileSync(path.join(FIXTURES, 'server-cert.pem'), nonCa);
|
||||
fs.chmodSync(nonCa, 0o444);
|
||||
expectCode(
|
||||
() => loadPostgresCertificateAuthorityFile(nonCa),
|
||||
'QL3_POSTGRES_CA_NOT_CA',
|
||||
);
|
||||
|
||||
const duplicate = path.join(directory, 'duplicate.pem');
|
||||
fs.writeFileSync(duplicate, `${CA}\n${CA}`, { mode: 0o444 });
|
||||
expectCode(
|
||||
() => loadPostgresCertificateAuthorityFile(duplicate),
|
||||
'QL3_POSTGRES_CA_DUPLICATE_CERTIFICATE',
|
||||
);
|
||||
|
||||
const tooMany = path.join(directory, 'too-many.pem');
|
||||
fs.writeFileSync(
|
||||
tooMany,
|
||||
Array.from({ length: POSTGRES_CA_MAX_CERTIFICATES + 1 }, () => CA).join(
|
||||
'\n',
|
||||
),
|
||||
{ mode: 0o444 },
|
||||
);
|
||||
expectCode(
|
||||
() => loadPostgresCertificateAuthorityFile(tooMany),
|
||||
'QL3_POSTGRES_CA_TOO_MANY_CERTIFICATES',
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,87 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PostgresClusterDispatchSource,
|
||||
} = require('../dist/remote-execution/clusterDispatchRepository');
|
||||
|
||||
const SESSION = '018f0000-0000-7000-8000-000000000001';
|
||||
|
||||
function candidateRow(overrides = {}) {
|
||||
return {
|
||||
observedAtMs: '1000',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
projectId: 'project-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: `qltd:v1:1:${'a'.repeat(64)}`,
|
||||
priority: 4,
|
||||
queuedAtMs: '100',
|
||||
attemptCreatedAtMs: '101',
|
||||
attemptNumber: 1,
|
||||
executorType: 'remote_worker',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function leaseRow(overrides = {}) {
|
||||
return candidateRow({
|
||||
leaseStatus: 'leased',
|
||||
leaseVersion: 0,
|
||||
leaseGeneration: 1,
|
||||
workerId: 'edge-1',
|
||||
workerSessionId: SESSION,
|
||||
workerGeneration: 2,
|
||||
leaseTokenDigest: createHash('sha256').update('x'.repeat(32)).digest('hex'),
|
||||
acquiredAtMs: '900',
|
||||
renewedAtMs: '900',
|
||||
expiresAtMs: '1900',
|
||||
releasedAtMs: null,
|
||||
releaseReason: null,
|
||||
completedAtMs: null,
|
||||
leaseUpdatedAtMs: '900',
|
||||
workerCurrent: true,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
test('lists a bounded PostgreSQL-clock candidate page with stable cursor parameters', async () => {
|
||||
let observed;
|
||||
const source = new PostgresClusterDispatchSource({
|
||||
async query(sql, params) {
|
||||
observed = { sql, params };
|
||||
return { rows: [candidateRow(), candidateRow({ attemptId: 'attempt-2' })] };
|
||||
},
|
||||
});
|
||||
const page = await source.listClusterDispatchCandidates({ limit: 1 });
|
||||
assert.equal(page.observedAtMs, 1000);
|
||||
assert.equal(page.candidates.length, 1);
|
||||
assert.equal(page.truncated, true);
|
||||
assert.equal(page.next.attemptId, 'attempt-1');
|
||||
assert.deepEqual(observed.params, [null, null, null, null, 2]);
|
||||
assert.match(observed.sql, /clock_timestamp\(\)/);
|
||||
assert.match(observed.sql, /lease\.expires_at_ms <= observation\.observed_at_ms/);
|
||||
assert.match(
|
||||
observed.sql,
|
||||
/plugin_package_workflow_task_attempt_admissions/,
|
||||
);
|
||||
assert.match(observed.sql, /workflow_step\.status = 'ready'/);
|
||||
assert.match(
|
||||
observed.sql,
|
||||
/newer\.step_run_id = attempt\.step_run_id/,
|
||||
);
|
||||
});
|
||||
|
||||
test('returns one exact offer recovery with durable lease and Worker fence evidence', async () => {
|
||||
const source = new PostgresClusterDispatchSource({
|
||||
async query(sql, params) {
|
||||
assert.match(sql, /worker\.lease_expires_at_ms > observation\.observed_at_ms/);
|
||||
assert.deepEqual(params, ['offer-1']);
|
||||
return { rows: [leaseRow()] };
|
||||
},
|
||||
});
|
||||
const recovery = await source.findClusterDispatchRecovery('offer-1');
|
||||
assert.equal(recovery.workerCurrent, true);
|
||||
assert.equal(recovery.lease.workerSessionId, SESSION);
|
||||
assert.equal(recovery.candidate.executorType, 'remote_worker');
|
||||
});
|
||||
@@ -0,0 +1,272 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const { ClusterControlRecoveryStoreError } = require('@qinglong/runtime-core');
|
||||
const { PostgresClusterControlRecoveryClaimRepository } = require('../dist');
|
||||
|
||||
function sourceRows(observedAtMs = '1000') {
|
||||
return [
|
||||
{
|
||||
observedAtMs,
|
||||
kind: 'attempt',
|
||||
id: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
status: 'running',
|
||||
createdAtMs: '900',
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function claimRows() {
|
||||
return [
|
||||
{
|
||||
targetKind: 'attempt',
|
||||
targetId: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
targetStatus: 'running',
|
||||
targetCreatedAtMs: '900',
|
||||
observedAtMs: '1000',
|
||||
claimOwner: 'replica-a',
|
||||
claimToken: '00000000-0000-4000-8000-000000000001',
|
||||
claimVersion: 1,
|
||||
claimExpiresAtMs: '31000',
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
test('discovers and claims one bounded page in a short transaction', async () => {
|
||||
const calls = [];
|
||||
let released = false;
|
||||
const client = {
|
||||
async query(text, values = []) {
|
||||
calls.push({ text, values });
|
||||
if (text.includes('run_candidates AS')) return { rows: sourceRows() };
|
||||
if (text.startsWith('INSERT INTO "ql3"."run_recovery_controls"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('FOR UPDATE OF control SKIP LOCKED')) {
|
||||
return { rows: claimRows(), rowCount: 1 };
|
||||
}
|
||||
return { rows: [] };
|
||||
},
|
||||
release() {
|
||||
released = true;
|
||||
},
|
||||
};
|
||||
const repository = new PostgresClusterControlRecoveryClaimRepository(
|
||||
{
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
async query() {
|
||||
throw new Error('pool query not expected');
|
||||
},
|
||||
},
|
||||
() => '00000000-0000-4000-8000-000000000001',
|
||||
);
|
||||
|
||||
const page = await repository.claim({
|
||||
ownerId: 'replica-a',
|
||||
limit: 4,
|
||||
leaseMs: 30_000,
|
||||
});
|
||||
|
||||
assert.equal(page.discovered, 1);
|
||||
assert.equal(page.hasMore, false);
|
||||
assert.deepEqual(page.claims[0], {
|
||||
candidate: {
|
||||
kind: 'attempt',
|
||||
id: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
status: 'running',
|
||||
createdAtMs: 900,
|
||||
},
|
||||
observedAtMs: 1000,
|
||||
ownerId: 'replica-a',
|
||||
token: '00000000-0000-4000-8000-000000000001',
|
||||
version: 1,
|
||||
expiresAtMs: 31000,
|
||||
});
|
||||
assert.deepEqual(
|
||||
calls.map(({ text }) => text.split('\n', 1)[0]),
|
||||
[
|
||||
'BEGIN ISOLATION LEVEL READ COMMITTED',
|
||||
"SET LOCAL statement_timeout = '5000ms'",
|
||||
"SET LOCAL lock_timeout = '1000ms'",
|
||||
'WITH observation AS (',
|
||||
'INSERT INTO "ql3"."run_recovery_controls" (',
|
||||
'WITH discovered AS (',
|
||||
'COMMIT',
|
||||
],
|
||||
);
|
||||
assert.deepEqual(calls[5].values.slice(1), [
|
||||
1000,
|
||||
4,
|
||||
'replica-a',
|
||||
'00000000-0000-4000-8000-000000000001',
|
||||
30000,
|
||||
]);
|
||||
assert.equal(released, true);
|
||||
});
|
||||
|
||||
test('uses an injected runtime-only discovery source without widening claim authority', async () => {
|
||||
let sourceQueryable;
|
||||
let sourceLimit;
|
||||
const client = {
|
||||
async query(text) {
|
||||
if (text.startsWith('INSERT INTO "ql3"."run_recovery_controls"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('FOR UPDATE OF control SKIP LOCKED')) {
|
||||
return { rows: claimRows(), rowCount: 1 };
|
||||
}
|
||||
return { rows: [] };
|
||||
},
|
||||
release() {},
|
||||
};
|
||||
const repository = new PostgresClusterControlRecoveryClaimRepository(
|
||||
{
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
async query() {
|
||||
throw new Error('not expected');
|
||||
},
|
||||
},
|
||||
() => '00000000-0000-4000-8000-000000000001',
|
||||
(queryable) => {
|
||||
sourceQueryable = queryable;
|
||||
return {
|
||||
async listOutstanding(limit) {
|
||||
sourceLimit = limit;
|
||||
return {
|
||||
observedAtMs: 1000,
|
||||
candidates: [{
|
||||
kind: 'attempt',
|
||||
id: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
status: 'running',
|
||||
createdAtMs: 900,
|
||||
}],
|
||||
hasMore: false,
|
||||
};
|
||||
},
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
const page = await repository.claim({
|
||||
ownerId: 'replica-a',
|
||||
limit: 4,
|
||||
leaseMs: 30_000,
|
||||
});
|
||||
assert.equal(sourceQueryable, client);
|
||||
assert.equal(sourceLimit, 4);
|
||||
assert.equal(page.claims.length, 1);
|
||||
});
|
||||
|
||||
test('rolls back and wraps claim-store failures without leaking the client', async () => {
|
||||
const calls = [];
|
||||
let released = false;
|
||||
const repository = new PostgresClusterControlRecoveryClaimRepository(
|
||||
{
|
||||
async connect() {
|
||||
return {
|
||||
async query(text) {
|
||||
calls.push(text);
|
||||
if (text.includes('run_candidates AS')) throw new Error('offline');
|
||||
return { rows: [] };
|
||||
},
|
||||
release() {
|
||||
released = true;
|
||||
},
|
||||
};
|
||||
},
|
||||
async query() {
|
||||
throw new Error('not expected');
|
||||
},
|
||||
},
|
||||
() => '00000000-0000-4000-8000-000000000002',
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
repository.claim({ ownerId: 'replica-b', limit: 1, leaseMs: 1000 }),
|
||||
ClusterControlRecoveryStoreError,
|
||||
);
|
||||
assert.equal(calls.at(-1), 'ROLLBACK');
|
||||
assert.equal(released, true);
|
||||
});
|
||||
|
||||
test('settles only under the full owner-token-version-expiry fence', async () => {
|
||||
const calls = [];
|
||||
const results = [
|
||||
{ rows: [{ targetId: 'attempt-1' }], rowCount: 1 },
|
||||
{ rows: [], rowCount: 0 },
|
||||
];
|
||||
const repository = new PostgresClusterControlRecoveryClaimRepository({
|
||||
async connect() {
|
||||
throw new Error('not expected');
|
||||
},
|
||||
async query(text, values) {
|
||||
calls.push({ text, values });
|
||||
return results.shift();
|
||||
},
|
||||
});
|
||||
const claim = Object.freeze({
|
||||
candidate: Object.freeze({
|
||||
kind: 'attempt',
|
||||
id: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
status: 'running',
|
||||
createdAtMs: 900,
|
||||
}),
|
||||
observedAtMs: 1000,
|
||||
ownerId: 'replica-a',
|
||||
token: '00000000-0000-4000-8000-000000000001',
|
||||
version: 7,
|
||||
expiresAtMs: 31000,
|
||||
});
|
||||
|
||||
assert.equal(
|
||||
await repository.settle(claim, { status: 'retry', delayMs: 2500 }),
|
||||
'settled',
|
||||
);
|
||||
assert.equal(
|
||||
await repository.settle(claim, { status: 'resolved' }),
|
||||
'fenced',
|
||||
);
|
||||
assert.match(
|
||||
calls[0].text,
|
||||
/claim_expires_at_ms > observation\.observed_at_ms/,
|
||||
);
|
||||
assert.deepEqual(calls[0].values, [
|
||||
'attempt',
|
||||
'attempt-1',
|
||||
'replica-a',
|
||||
'00000000-0000-4000-8000-000000000001',
|
||||
7,
|
||||
'retry',
|
||||
2500,
|
||||
]);
|
||||
});
|
||||
|
||||
test('rejects unsafe options before acquiring a database connection', async () => {
|
||||
let connects = 0;
|
||||
const repository = new PostgresClusterControlRecoveryClaimRepository({
|
||||
async connect() {
|
||||
connects += 1;
|
||||
throw new Error('not expected');
|
||||
},
|
||||
async query() {
|
||||
throw new Error('not expected');
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
repository.claim({ ownerId: 'bad owner', limit: 1, leaseMs: 1000 }),
|
||||
/ownerId is invalid/,
|
||||
);
|
||||
await assert.rejects(
|
||||
repository.claim({ ownerId: 'ok', limit: 129, leaseMs: 1000 }),
|
||||
/claim limit/,
|
||||
);
|
||||
assert.equal(connects, 0);
|
||||
});
|
||||
@@ -0,0 +1,438 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PostgresClusterControlRecoveryResolutionRepository,
|
||||
} = require('../dist');
|
||||
const {
|
||||
pluginPackageWorkflowTaskAttemptAdmissionReceiptDigest,
|
||||
} = require('@qinglong/runtime-core/plugin-package-workflow-task-attempt-admission');
|
||||
const {
|
||||
createStepRunRecord,
|
||||
} = require('@qinglong/runtime-core/step-run');
|
||||
|
||||
const RUN_ID = '019f70b0-0000-7000-8000-000000000101';
|
||||
const ATTEMPT_ID = '019f70b0-0000-7000-8000-000000000102';
|
||||
const STEP_RUN_ID = '019f70b0-0000-7000-8000-000000000103';
|
||||
|
||||
function workflowStepRun() {
|
||||
return createStepRunRecord({
|
||||
id: STEP_RUN_ID,
|
||||
runId: RUN_ID,
|
||||
stepKey: 'collect',
|
||||
kind: 'task',
|
||||
definitionRef: 'pkg:demo:alpha',
|
||||
definitionDigest: 'a'.repeat(64),
|
||||
required: true,
|
||||
initialStatus: 'ready',
|
||||
mutationId: '019f70b0-0000-7000-8000-000000000104',
|
||||
createdAtMs: 300,
|
||||
});
|
||||
}
|
||||
|
||||
function workflowAdmission(stepRun) {
|
||||
const unsigned = {
|
||||
schema:
|
||||
'qinglong/plugin-package-workflow-task-attempt-admission@v1',
|
||||
attemptId: ATTEMPT_ID,
|
||||
planDigest: 'c'.repeat(64),
|
||||
runId: RUN_ID,
|
||||
stepRunId: STEP_RUN_ID,
|
||||
stepRunVersion: stepRun.version,
|
||||
stepRunDigest: stepRun.stepRunDigest,
|
||||
resourceTaskId: 'alpha',
|
||||
taskReconciliationReceiptDigest: 'd'.repeat(64),
|
||||
taskId: 'pkg:demo:alpha',
|
||||
taskRevision: `qltd:v1:1:${'a'.repeat(64)}`,
|
||||
taskDefinitionDigest: 'a'.repeat(64),
|
||||
executorType: 'remote_worker',
|
||||
executionDigest: 'e'.repeat(64),
|
||||
attemptNumber: 1,
|
||||
eventId: '019f70b0-0000-7000-8000-000000000105',
|
||||
runVersion: 3,
|
||||
runEventSequence: 3,
|
||||
admittedAtMs: 400,
|
||||
};
|
||||
return {
|
||||
...unsigned,
|
||||
receiptDigest:
|
||||
pluginPackageWorkflowTaskAttemptAdmissionReceiptDigest(unsigned),
|
||||
};
|
||||
}
|
||||
|
||||
function claim() {
|
||||
return {
|
||||
candidate: {
|
||||
kind: 'attempt',
|
||||
id: ATTEMPT_ID,
|
||||
runId: RUN_ID,
|
||||
status: 'claimed',
|
||||
createdAtMs: 200,
|
||||
},
|
||||
observedAtMs: 1000,
|
||||
ownerId: 'replica-a',
|
||||
token: '00000000-0000-4000-8000-000000000001',
|
||||
version: 1,
|
||||
expiresAtMs: 31000,
|
||||
};
|
||||
}
|
||||
|
||||
function runRow(overrides = {}) {
|
||||
return {
|
||||
id: RUN_ID,
|
||||
projectId: 'default',
|
||||
taskId: 'task-1',
|
||||
taskRevision: 'v1',
|
||||
taskName: null,
|
||||
taskSnapshotRef: null,
|
||||
legacyCronId: null,
|
||||
parentRunId: null,
|
||||
retryOfRunId: null,
|
||||
triggerId: null,
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
triggeredBy: null,
|
||||
requestId: null,
|
||||
scheduledForMs: null,
|
||||
status: 'dispatching',
|
||||
version: 1,
|
||||
eventSequence: 0,
|
||||
priority: 0,
|
||||
idempotencyKey: null,
|
||||
inputRef: null,
|
||||
outputRef: null,
|
||||
createdAtMs: '100',
|
||||
queuedAtMs: null,
|
||||
startedAtMs: null,
|
||||
finishedAtMs: null,
|
||||
cancelRequestedAtMs: null,
|
||||
cancelReason: null,
|
||||
errorCode: null,
|
||||
errorSummary: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function attemptRow(overrides = {}) {
|
||||
return {
|
||||
id: ATTEMPT_ID,
|
||||
runId: RUN_ID,
|
||||
stepRunId: null,
|
||||
attempt: 1,
|
||||
status: 'claimed',
|
||||
executorType: 'worker',
|
||||
workerId: 'worker-1',
|
||||
executorHandle: null,
|
||||
pid: null,
|
||||
logArtifactId: null,
|
||||
leaseToken: 'lease-token',
|
||||
leaseExpiresAtMs: '900',
|
||||
deadlineAtMs: null,
|
||||
callbackTokenHash: null,
|
||||
callbackSequence: 0,
|
||||
createdAtMs: '200',
|
||||
startedAtMs: null,
|
||||
finishedAtMs: null,
|
||||
exitCode: null,
|
||||
errorCode: null,
|
||||
errorSummary: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function harness(options = {}) {
|
||||
const queries = [];
|
||||
let released = 0;
|
||||
let runUpdates = 0;
|
||||
const client = {
|
||||
async query(text, values = []) {
|
||||
queries.push({ text, values });
|
||||
if (text.includes('run_recovery_controls')) {
|
||||
return options.fenced
|
||||
? { rows: [], rowCount: 0 }
|
||||
: { rows: [{ observedAtMs: '1000' }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."runs"')) {
|
||||
return {
|
||||
rows: [
|
||||
runRow(
|
||||
options.workflowTask
|
||||
? {
|
||||
taskId: 'workflow-alpha',
|
||||
taskRevision: 'b'.repeat(64),
|
||||
triggerType: 'plugin_package_workflow',
|
||||
executionOrigin: 'system',
|
||||
requestId: 'workflow-plan-1',
|
||||
idempotencyKey:
|
||||
'plugin-package-workflow:workflow-plan-1',
|
||||
status: 'running',
|
||||
version: 5,
|
||||
eventSequence: 5,
|
||||
startedAtMs: '200',
|
||||
}
|
||||
: {},
|
||||
),
|
||||
],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."run_attempts"')) {
|
||||
return {
|
||||
rows: [
|
||||
attemptRow(
|
||||
{
|
||||
...(options.workflowTask
|
||||
? {
|
||||
stepRunId: STEP_RUN_ID,
|
||||
executorType: 'remote_worker',
|
||||
createdAtMs: '400',
|
||||
}
|
||||
: {}),
|
||||
...(options.leased
|
||||
? {
|
||||
workerSessionId:
|
||||
'019f70b0-0000-7000-8000-000000000201',
|
||||
workerGeneration: 2,
|
||||
leaseTokenDigest: 'f'.repeat(64),
|
||||
leaseGeneration: 3,
|
||||
leaseVersion: 7,
|
||||
offerId: 'offer-1',
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
),
|
||||
],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'plugin_package_workflow_task_attempt_admissions',
|
||||
)
|
||||
) {
|
||||
return options.workflowTask
|
||||
? {
|
||||
rows: [{
|
||||
admissionJson: options.workflowTask.admission,
|
||||
stepRunJson: options.workflowTask.stepRun,
|
||||
}],
|
||||
rowCount: 1,
|
||||
}
|
||||
: { rows: [], rowCount: 0 };
|
||||
}
|
||||
if (text.startsWith('UPDATE "ql3"."runs"')) {
|
||||
runUpdates += 1;
|
||||
return { rows: [{ id: RUN_ID }], rowCount: 1 };
|
||||
}
|
||||
if (text.startsWith('UPDATE "ql3"."run_attempts"')) {
|
||||
return options.attemptConflict
|
||||
? { rows: [], rowCount: 0 }
|
||||
: { rows: [{ id: ATTEMPT_ID }], rowCount: 1 };
|
||||
}
|
||||
if (text.startsWith('UPDATE "ql3"."run_dispatch_leases"')) {
|
||||
return options.leaseConflict
|
||||
? { rows: [], rowCount: 0 }
|
||||
: { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text.startsWith('UPDATE "ql3"."step_runs"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text.startsWith('INSERT INTO "ql3"."run_events"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
text.startsWith(
|
||||
'INSERT INTO "ql3"."step_run_mutations"',
|
||||
)
|
||||
) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
return { rows: [], rowCount: 0 };
|
||||
},
|
||||
release() {
|
||||
released += 1;
|
||||
},
|
||||
};
|
||||
return {
|
||||
queries,
|
||||
released: () => released,
|
||||
runUpdates: () => runUpdates,
|
||||
pool: {
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
query: (...args) => client.query(...args),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('loads only under a live claim fence and returns a normalized snapshot', async () => {
|
||||
const database = harness();
|
||||
const repository = new PostgresClusterControlRecoveryResolutionRepository(
|
||||
database.pool,
|
||||
);
|
||||
const loaded = await repository.load(claim());
|
||||
|
||||
assert.equal(loaded.observedAtMs, 1000);
|
||||
assert.equal(loaded.run.status, 'dispatching');
|
||||
assert.equal(loaded.attempt.status, 'claimed');
|
||||
assert.equal(loaded.attempt.leaseExpiresAtMs, 900);
|
||||
assert.equal(Object.isFrozen(loaded), true);
|
||||
assert.equal(database.queries[4].text.includes('FOR UPDATE'), false);
|
||||
assert.equal(database.queries.at(-1).text, 'COMMIT');
|
||||
assert.equal(database.released(), 1);
|
||||
});
|
||||
|
||||
test('locks the claim and commits both lost transitions and events atomically', async () => {
|
||||
const database = harness();
|
||||
let event = 0;
|
||||
const repository = new PostgresClusterControlRecoveryResolutionRepository(
|
||||
database.pool,
|
||||
() => `00000000-0000-4000-8000-${String(++event).padStart(12, '0')}`,
|
||||
);
|
||||
const loaded = await repository.load(claim());
|
||||
assert.equal(
|
||||
await repository.applyLost(claim(), loaded, {
|
||||
kind: 'mark_attempt_and_run_lost',
|
||||
reason: 'unstarted_claim_expired',
|
||||
}),
|
||||
'applied',
|
||||
);
|
||||
|
||||
const applyQueries = database.queries.slice(8);
|
||||
assert.equal(
|
||||
applyQueries.some(({ text }) => text.includes('FOR UPDATE OF control')),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
applyQueries.filter(({ text }) => text.startsWith('UPDATE "ql3"."runs"'))
|
||||
.length,
|
||||
2,
|
||||
);
|
||||
assert.equal(
|
||||
applyQueries.filter(({ text }) =>
|
||||
text.startsWith('UPDATE "ql3"."run_attempts"'),
|
||||
).length,
|
||||
1,
|
||||
);
|
||||
const events = applyQueries.filter(({ text }) =>
|
||||
text.startsWith('INSERT INTO "ql3"."run_events"'),
|
||||
);
|
||||
assert.equal(events.length, 2);
|
||||
assert.equal(events[0].values.includes('reconciler'), true);
|
||||
assert.equal(events[0].values.includes('replica-a'), true);
|
||||
assert.equal(applyQueries.at(-1).text, 'COMMIT');
|
||||
});
|
||||
|
||||
test('releases an expired dispatch lease under the same Attempt authority transaction', async () => {
|
||||
const database = harness({ leased: true });
|
||||
const repository = new PostgresClusterControlRecoveryResolutionRepository(
|
||||
database.pool,
|
||||
);
|
||||
const loaded = await repository.load(claim());
|
||||
|
||||
assert.equal(
|
||||
await repository.applyLost(claim(), loaded, {
|
||||
kind: 'mark_attempt_and_run_lost',
|
||||
reason: 'unstarted_claim_expired',
|
||||
}),
|
||||
'applied',
|
||||
);
|
||||
const leaseUpdate = database.queries.find(({ text }) =>
|
||||
text.startsWith('UPDATE "ql3"."run_dispatch_leases"'),
|
||||
);
|
||||
assert.ok(leaseUpdate);
|
||||
assert.deepEqual(leaseUpdate.values, [
|
||||
ATTEMPT_ID,
|
||||
8,
|
||||
1000,
|
||||
7,
|
||||
3,
|
||||
]);
|
||||
const attemptUpdate = database.queries.find(({ text }) =>
|
||||
text.startsWith('UPDATE "ql3"."run_attempts"'),
|
||||
);
|
||||
assert.equal(attemptUpdate.values.includes(8), true);
|
||||
assert.equal(database.queries.at(-1).text, 'COMMIT');
|
||||
});
|
||||
|
||||
test('atomically loses and requeues one admission-bound Workflow Task epoch', async () => {
|
||||
const stepRun = workflowStepRun();
|
||||
const database = harness({
|
||||
workflowTask: {
|
||||
admission: workflowAdmission(stepRun),
|
||||
stepRun,
|
||||
},
|
||||
});
|
||||
const repository =
|
||||
new PostgresClusterControlRecoveryResolutionRepository(database.pool);
|
||||
const loaded = await repository.load(claim());
|
||||
assert.equal(loaded.workflowTask.admission.attemptId, ATTEMPT_ID);
|
||||
assert.equal(loaded.workflowTask.stepRun.status, 'ready');
|
||||
|
||||
assert.equal(
|
||||
await repository.applyLost(claim(), loaded, {
|
||||
kind: 'recover_workflow_task',
|
||||
reason: 'unstarted_claim_expired',
|
||||
}),
|
||||
'applied',
|
||||
);
|
||||
|
||||
const runUpdate = database.queries.find(
|
||||
({ text, values }) =>
|
||||
text.startsWith('UPDATE "ql3"."runs"') &&
|
||||
values[0] === RUN_ID &&
|
||||
values.length === 5,
|
||||
);
|
||||
assert.ok(runUpdate);
|
||||
assert.deepEqual(runUpdate.values.slice(1), [7, 7, 5, 5]);
|
||||
const attemptUpdate = database.queries.find(({ text }) =>
|
||||
text.startsWith('UPDATE "ql3"."run_attempts"'));
|
||||
assert.ok(attemptUpdate);
|
||||
assert.equal(attemptUpdate.values.includes('lost'), true);
|
||||
const stepUpdate = database.queries.find(({ text }) =>
|
||||
text.startsWith('UPDATE "ql3"."step_runs"'));
|
||||
assert.ok(stepUpdate);
|
||||
assert.equal(stepUpdate.values[0], 'ready');
|
||||
assert.equal(stepUpdate.values[1], stepRun.version + 1);
|
||||
assert.notEqual(stepUpdate.values[12], stepRun.stepRunDigest);
|
||||
const events = database.queries.filter(({ text }) =>
|
||||
text.startsWith('INSERT INTO "ql3"."run_events"'));
|
||||
assert.equal(events.length, 2);
|
||||
assert.equal(
|
||||
database.queries.filter(({ text }) =>
|
||||
text.startsWith('INSERT INTO "ql3"."step_run_mutations"'),
|
||||
).length,
|
||||
1,
|
||||
);
|
||||
assert.equal(database.queries.at(-1).text, 'COMMIT');
|
||||
});
|
||||
|
||||
test('rolls back partial CAS work and reports a stale aggregate', async () => {
|
||||
const database = harness({ attemptConflict: true });
|
||||
const repository = new PostgresClusterControlRecoveryResolutionRepository(
|
||||
database.pool,
|
||||
);
|
||||
const loaded = await repository.load(claim());
|
||||
assert.equal(
|
||||
await repository.applyLost(claim(), loaded, {
|
||||
kind: 'mark_attempt_and_run_lost',
|
||||
reason: 'unstarted_claim_expired',
|
||||
}),
|
||||
'stale',
|
||||
);
|
||||
assert.equal(database.runUpdates(), 1);
|
||||
assert.equal(database.queries.at(-1).text, 'ROLLBACK');
|
||||
});
|
||||
|
||||
test('does not read or mutate Run state after the recovery fence is lost', async () => {
|
||||
const database = harness({ fenced: true });
|
||||
const repository = new PostgresClusterControlRecoveryResolutionRepository(
|
||||
database.pool,
|
||||
);
|
||||
assert.equal(await repository.load(claim()), 'fenced');
|
||||
assert.equal(
|
||||
database.queries.some(({ text }) => text.includes('FROM "ql3"."runs"')),
|
||||
false,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,166 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const { PostgresClusterControlRecoverySource } = require('../dist/entrypoints/runtime');
|
||||
|
||||
function sourceWith(rows, observations = []) {
|
||||
return new PostgresClusterControlRecoverySource({
|
||||
async query(text, values) {
|
||||
observations.push({ text, values });
|
||||
return { rows };
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
test('reads Run and Attempt recovery candidates through one bounded query', async () => {
|
||||
const observations = [];
|
||||
const source = sourceWith(
|
||||
[
|
||||
{
|
||||
observedAtMs: '100',
|
||||
kind: 'run',
|
||||
id: 'run-1',
|
||||
runId: 'run-1',
|
||||
status: 'running',
|
||||
createdAtMs: '10',
|
||||
},
|
||||
{
|
||||
observedAtMs: '100',
|
||||
kind: 'attempt',
|
||||
id: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
status: 'starting',
|
||||
createdAtMs: 11,
|
||||
},
|
||||
{
|
||||
observedAtMs: '100',
|
||||
kind: 'run',
|
||||
id: 'run-2',
|
||||
runId: 'run-2',
|
||||
status: 'created',
|
||||
createdAtMs: 12,
|
||||
},
|
||||
],
|
||||
observations,
|
||||
);
|
||||
|
||||
assert.deepEqual(await source.listOutstanding(2), {
|
||||
observedAtMs: 100,
|
||||
candidates: [
|
||||
{
|
||||
kind: 'run',
|
||||
id: 'run-1',
|
||||
runId: 'run-1',
|
||||
status: 'running',
|
||||
createdAtMs: 10,
|
||||
},
|
||||
{
|
||||
kind: 'attempt',
|
||||
id: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
status: 'starting',
|
||||
createdAtMs: 11,
|
||||
},
|
||||
],
|
||||
hasMore: true,
|
||||
});
|
||||
assert.deepEqual(observations[0].values, [3]);
|
||||
assert.match(observations[0].text, /WITH observation AS/);
|
||||
assert.match(observations[0].text, /statement_timestamp\(\)/);
|
||||
assert.match(observations[0].text, /LIMIT \$1/);
|
||||
assert.match(observations[0].text, /execution_owner = 'runtime'/);
|
||||
assert.match(
|
||||
observations[0].text,
|
||||
/trigger_type <> 'plugin_package_workflow'/,
|
||||
);
|
||||
assert.match(observations[0].text, /attempt_candidates/);
|
||||
assert.match(
|
||||
observations[0].text,
|
||||
/plugin_package_workflow_task_attempt_admissions/,
|
||||
);
|
||||
assert.match(
|
||||
observations[0].text,
|
||||
/attempt_run\.trigger_type = 'plugin_package_workflow'/,
|
||||
);
|
||||
assert.match(
|
||||
observations[0].text,
|
||||
/workflow_task\.attempt_id = attempt\.id/,
|
||||
);
|
||||
assert.match(
|
||||
observations[0].text,
|
||||
/lease_expires_at_ms > observation\.observed_at_ms/,
|
||||
);
|
||||
assert.match(
|
||||
observations[0].text,
|
||||
/lease_expires_at_ms <= observation\.observed_at_ms/,
|
||||
);
|
||||
assert.match(observations[0].text, /INNER JOIN "ql3"\."runs" AS attempt_run/);
|
||||
assert.match(observations[0].text, /attempt_run\.status = 'queued'/);
|
||||
assert.match(
|
||||
observations[0].text,
|
||||
/attempt\.executor_type = 'remote_worker'/,
|
||||
);
|
||||
assert.match(observations[0].text, /attempt\.callback_sequence = 0/);
|
||||
assert.match(
|
||||
observations[0].text,
|
||||
/newer_attempt\.attempt > attempt\.attempt/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects unbounded page sizes before touching PostgreSQL', async () => {
|
||||
let queries = 0;
|
||||
const source = sourceWith([], {
|
||||
push() {
|
||||
queries += 1;
|
||||
},
|
||||
});
|
||||
await assert.rejects(source.listOutstanding(0), /between 1 and 128/);
|
||||
await assert.rejects(source.listOutstanding(129), /between 1 and 128/);
|
||||
assert.equal(queries, 0);
|
||||
});
|
||||
|
||||
test('fails closed on malformed or terminal PostgreSQL rows', async () => {
|
||||
await assert.rejects(
|
||||
sourceWith([
|
||||
{
|
||||
observedAtMs: '100',
|
||||
kind: 'run',
|
||||
id: 'run-1',
|
||||
runId: 'run-1',
|
||||
status: 'succeeded',
|
||||
createdAtMs: 1,
|
||||
},
|
||||
]).listOutstanding(1),
|
||||
/kind or status is invalid/,
|
||||
);
|
||||
await assert.rejects(
|
||||
sourceWith([
|
||||
{
|
||||
observedAtMs: '100',
|
||||
kind: 'attempt',
|
||||
id: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
status: 'running',
|
||||
createdAtMs: '9007199254740992',
|
||||
},
|
||||
]).listOutstanding(1),
|
||||
/createdAtMs is invalid/,
|
||||
);
|
||||
});
|
||||
|
||||
test('represents an empty page without losing the database observation', async () => {
|
||||
const source = sourceWith([
|
||||
{
|
||||
observedAtMs: '100',
|
||||
kind: null,
|
||||
id: null,
|
||||
runId: null,
|
||||
status: null,
|
||||
createdAtMs: null,
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(await source.listOutstanding(1), {
|
||||
observedAtMs: 100,
|
||||
candidates: [],
|
||||
hasMore: false,
|
||||
});
|
||||
});
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
ClusterRunCancellationConvergenceUnavailableError,
|
||||
} = require('@qinglong/runtime-core/cluster-run-cancellation-convergence');
|
||||
const {
|
||||
PostgresClusterRunCancellationConvergenceRepository,
|
||||
} = require('../dist/entrypoints/runtime');
|
||||
|
||||
function row(overrides = {}) {
|
||||
return {
|
||||
observedAtMs: '1750000000100',
|
||||
runId: 'run-1',
|
||||
runStatus: 'queued',
|
||||
runVersion: 3,
|
||||
eventSequence: 4,
|
||||
runCreatedAtMs: '1750000000000',
|
||||
runQueuedAtMs: '1750000000010',
|
||||
runStartedAtMs: null,
|
||||
cancelRequestedAtMs: '1750000000050',
|
||||
cancelReason: 'user',
|
||||
attemptId: 'attempt-1',
|
||||
attemptStatus: 'claimed',
|
||||
stepRunId: null,
|
||||
attemptNumber: 1,
|
||||
attemptCreatedAtMs: '1750000000010',
|
||||
attemptStartedAtMs: null,
|
||||
attemptFinishedAtMs: null,
|
||||
leaseStatus: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(options = {}) {
|
||||
const calls = [];
|
||||
let runUpdates = 0;
|
||||
const client = {
|
||||
async query(sql, params = []) {
|
||||
const normalized = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ sql: normalized, params });
|
||||
if (
|
||||
normalized === 'BEGIN' || normalized === 'COMMIT' ||
|
||||
normalized === 'ROLLBACK' || normalized.startsWith('SELECT set_config')
|
||||
) return { rows: [], rowCount: 0 };
|
||||
if (normalized.startsWith('WITH observation AS MATERIALIZED')) {
|
||||
return { rows: options.rows ?? [row()], rowCount: options.rows?.length ?? 1 };
|
||||
}
|
||||
if (normalized.startsWith('UPDATE "ql3"."run_attempts"')) {
|
||||
return { rows: [], rowCount: options.attemptRowCount ?? 1 };
|
||||
}
|
||||
if (normalized.startsWith('UPDATE "ql3"."runs"')) {
|
||||
runUpdates += 1;
|
||||
return {
|
||||
rows: [],
|
||||
rowCount: options.failRunUpdate === runUpdates ? 0 : 1,
|
||||
};
|
||||
}
|
||||
if (normalized.startsWith('INSERT INTO "ql3"."run_events"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (normalized.startsWith('SELECT id AS "runId" FROM "ql3"."runs"')) {
|
||||
return {
|
||||
rows: options.workflowRows ?? [],
|
||||
rowCount: options.workflowRows?.length ?? 0,
|
||||
};
|
||||
}
|
||||
if (normalized.startsWith('SELECT EXISTS')) {
|
||||
return { rows: [{ hasMore: options.hasMore ?? false }], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`Unexpected SQL: ${normalized}`);
|
||||
},
|
||||
release() { calls.push({ sql: 'RELEASE', params: [] }); },
|
||||
};
|
||||
return {
|
||||
repository: new PostgresClusterRunCancellationConvergenceRepository({
|
||||
async connect() { return client; },
|
||||
}),
|
||||
calls,
|
||||
};
|
||||
}
|
||||
|
||||
test('atomically settles queued claimed and lost terminal-attempt Runs', async () => {
|
||||
const { repository, calls } = fixture({
|
||||
rows: [
|
||||
row(),
|
||||
row({
|
||||
runId: 'run-2',
|
||||
runStatus: 'lost',
|
||||
runVersion: 7,
|
||||
eventSequence: 8,
|
||||
cancelReason: 'policy',
|
||||
attemptId: 'attempt-2',
|
||||
attemptStatus: 'lost',
|
||||
attemptFinishedAtMs: '1750000000020',
|
||||
}),
|
||||
],
|
||||
});
|
||||
assert.deepEqual(await repository.convergePage({
|
||||
limit: 2,
|
||||
}), {
|
||||
scanned: 2,
|
||||
settledRuns: 2,
|
||||
settledAttempts: 1,
|
||||
blocked: 0,
|
||||
hasMore: false,
|
||||
});
|
||||
const attemptUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."run_attempts"'));
|
||||
assert.equal(attemptUpdate.params[1], 'cancelled');
|
||||
const runUpdates = calls.filter(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"'));
|
||||
assert.equal(runUpdates.length, 2);
|
||||
assert.equal(runUpdates[0].params[5], 5);
|
||||
assert.equal(runUpdates[0].params[6], 6);
|
||||
assert.equal(runUpdates[1].params[5], 8);
|
||||
assert.equal(runUpdates[1].params[6], 9);
|
||||
const events = calls.filter(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"'));
|
||||
assert.equal(events.length, 3);
|
||||
assert.match(events[0].params[0], /^qca-[0-9a-f]{32}$/);
|
||||
assert.match(events[1].params[0], /^qcr-[0-9a-f]{32}$/);
|
||||
assert.match(events[2].params[0], /^qcr-[0-9a-f]{32}$/);
|
||||
assert.equal(new Set(events.map(({ params }) => params[0])).size, 3);
|
||||
const candidate = calls.find(({ sql }) =>
|
||||
sql.startsWith('WITH observation AS MATERIALIZED'));
|
||||
assert.match(
|
||||
candidate.sql,
|
||||
/run\.trigger_type <> 'plugin_package_workflow'/,
|
||||
);
|
||||
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), true);
|
||||
});
|
||||
|
||||
test('maps timeout intent to timed_out and preserves database-scale time', async () => {
|
||||
const { repository, calls } = fixture({
|
||||
rows: [row({
|
||||
attemptId: null,
|
||||
attemptStatus: null,
|
||||
attemptNumber: null,
|
||||
attemptCreatedAtMs: null,
|
||||
cancelReason: 'timeout',
|
||||
})],
|
||||
});
|
||||
assert.equal((await repository.convergePage({
|
||||
limit: 1,
|
||||
})).settledRuns, 1);
|
||||
const update = calls.find(({ sql }) => sql.startsWith('UPDATE "ql3"."runs"'));
|
||||
assert.equal(update.params[1], 'timed_out');
|
||||
assert.equal(update.params[2], 1_750_000_000_100);
|
||||
assert.equal(update.params[3], 'EXECUTION_TIMED_OUT');
|
||||
});
|
||||
|
||||
test('does not forge a terminal state for an execution that crossed start', async () => {
|
||||
const { repository, calls } = fixture({
|
||||
rows: [row({
|
||||
runStatus: 'waiting_approval',
|
||||
attemptStatus: 'running',
|
||||
leaseStatus: 'leased',
|
||||
})],
|
||||
hasMore: true,
|
||||
});
|
||||
assert.deepEqual(await repository.convergePage({
|
||||
limit: 1,
|
||||
}), {
|
||||
scanned: 1,
|
||||
settledRuns: 0,
|
||||
settledAttempts: 0,
|
||||
blocked: 1,
|
||||
hasMore: true,
|
||||
});
|
||||
assert.equal(calls.some(({ sql }) => sql.startsWith('UPDATE')), false);
|
||||
});
|
||||
|
||||
test('rolls back the whole page when a Run fence changes', async () => {
|
||||
const { repository, calls } = fixture({ failRunUpdate: 1 });
|
||||
await assert.rejects(
|
||||
repository.convergePage({ limit: 1 }),
|
||||
ClusterRunCancellationConvergenceUnavailableError,
|
||||
);
|
||||
assert.equal(calls.some(({ sql }) => sql === 'ROLLBACK'), true);
|
||||
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), false);
|
||||
});
|
||||
@@ -0,0 +1,264 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
ClusterRunCancellationFenceRejectedError,
|
||||
ClusterRunCancellationNotFoundError,
|
||||
} = require('@qinglong/runtime-core/cluster-run-cancellation');
|
||||
const {
|
||||
PostgresClusterRunCancellationRepository,
|
||||
} = require('../dist/entrypoints/runtime');
|
||||
|
||||
function command(overrides = {}) {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
mutationId: 'mutation-1',
|
||||
eventId: '018f0000-0000-7000-8000-000000000001',
|
||||
subject: { type: 'user', id: 'user-1' },
|
||||
policyFence: { projectVersion: 2, bindingVersion: 3 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function run(overrides = {}) {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
runStatus: 'running',
|
||||
runVersion: 4,
|
||||
eventSequence: 6,
|
||||
cancelRequestedAtMs: null,
|
||||
cancelReason: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(options = {}) {
|
||||
const calls = [];
|
||||
const client = {
|
||||
async query(sql, params = []) {
|
||||
const normalized = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ sql: normalized, params });
|
||||
if (
|
||||
normalized.startsWith('BEGIN') || normalized === 'COMMIT' ||
|
||||
normalized === 'ROLLBACK' || normalized.startsWith('SELECT set_config')
|
||||
) return { rows: [], rowCount: 0 };
|
||||
if (normalized.includes('FROM "ql3"."projects"')) {
|
||||
return {
|
||||
rows: options.projectRows ?? [{
|
||||
projectStatus: 'active',
|
||||
projectVersion: 2,
|
||||
}],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."project_role_bindings"')) {
|
||||
return {
|
||||
rows: options.bindingRows ?? [{
|
||||
bindingVersion: 3,
|
||||
bindingState: 'active',
|
||||
bindingRole: 'operator',
|
||||
}],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (
|
||||
normalized.includes('FROM "ql3"."runs"') &&
|
||||
normalized.startsWith('SELECT')
|
||||
) {
|
||||
return {
|
||||
rows: options.runRows ?? [run()],
|
||||
rowCount: options.runRows?.length ?? 1,
|
||||
};
|
||||
}
|
||||
if (
|
||||
normalized.includes(
|
||||
'FROM "ql3"."plugin_package_workflow_admissions"',
|
||||
)
|
||||
) {
|
||||
const rows = options.workflowAdmissionRows ?? [
|
||||
{
|
||||
projectId: 'project-1',
|
||||
packageName: 'example',
|
||||
workflowId: 'daily',
|
||||
},
|
||||
];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: options.nowMs ?? 1_000 }], rowCount: 1 };
|
||||
}
|
||||
if (normalized.startsWith('UPDATE "ql3"."runs"')) {
|
||||
return {
|
||||
rows: options.updatedRows ?? [run({
|
||||
runVersion: 5,
|
||||
eventSequence: 7,
|
||||
cancelRequestedAtMs: options.nowMs ?? 1_000,
|
||||
cancelReason: 'user',
|
||||
})],
|
||||
rowCount: options.updatedRows?.length ?? 1,
|
||||
};
|
||||
}
|
||||
if (normalized.startsWith('INSERT INTO "ql3"."run_events"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`Unexpected SQL: ${normalized}`);
|
||||
},
|
||||
release() { calls.push({ sql: 'RELEASE', params: [] }); },
|
||||
};
|
||||
return {
|
||||
repository: new PostgresClusterRunCancellationRepository({
|
||||
async connect() { return client; },
|
||||
}),
|
||||
calls,
|
||||
};
|
||||
}
|
||||
|
||||
test('revalidates policy authority and commits one database-timed intent', async () => {
|
||||
const { repository, calls } = fixture();
|
||||
assert.deepEqual(await repository.requestUserCancellation(command()), {
|
||||
status: 'accepted',
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
runStatus: 'running',
|
||||
runVersion: 5,
|
||||
eventSequence: 7,
|
||||
cancelRequestedAtMs: 1_000,
|
||||
cancelReason: 'user',
|
||||
});
|
||||
const projectIndex = calls.findIndex(({ sql }) =>
|
||||
sql.includes('FROM "ql3"."projects"'));
|
||||
const bindingIndex = calls.findIndex(({ sql }) =>
|
||||
sql.includes('FROM "ql3"."project_role_bindings"'));
|
||||
const runIndex = calls.findIndex(({ sql }) =>
|
||||
sql.includes('FROM "ql3"."runs"'));
|
||||
assert.ok(projectIndex < bindingIndex && bindingIndex < runIndex);
|
||||
const update = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"'));
|
||||
assert.deepEqual(update.params, ['run-1', 1_000, 5, 7, 4]);
|
||||
const event = calls.find(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"'));
|
||||
assert.equal(event.params[0], command().eventId);
|
||||
assert.equal(event.params[3], 'user-cancel:mutation-1');
|
||||
assert.equal(event.params[4], 'user');
|
||||
assert.equal(JSON.parse(event.params[6]).reason, 'user');
|
||||
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), true);
|
||||
});
|
||||
|
||||
test('returns existing intent and terminal state without adding an event', async () => {
|
||||
const existing = fixture({
|
||||
runRows: [run({ cancelRequestedAtMs: 900, cancelReason: 'timeout' })],
|
||||
});
|
||||
assert.equal(
|
||||
(await existing.repository.requestUserCancellation(command())).status,
|
||||
'already_requested',
|
||||
);
|
||||
assert.equal(existing.calls.some(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"')), false);
|
||||
|
||||
const terminal = fixture({
|
||||
runRows: [run({ runStatus: 'succeeded', runVersion: 5 })],
|
||||
});
|
||||
assert.equal(
|
||||
(await terminal.repository.requestUserCancellation(command())).status,
|
||||
'already_terminal',
|
||||
);
|
||||
assert.equal(terminal.calls.some(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"')), false);
|
||||
});
|
||||
|
||||
test('accepts cancellation for a lost Run that still owns retry authority', async () => {
|
||||
const { repository } = fixture({
|
||||
runRows: [run({ runStatus: 'lost' })],
|
||||
updatedRows: [run({
|
||||
runStatus: 'lost',
|
||||
runVersion: 5,
|
||||
eventSequence: 7,
|
||||
cancelRequestedAtMs: 1_000,
|
||||
cancelReason: 'user',
|
||||
})],
|
||||
});
|
||||
assert.equal(
|
||||
(await repository.requestUserCancellation(command())).status,
|
||||
'accepted',
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a revoked policy fence before locking the Run', async () => {
|
||||
const { repository, calls } = fixture({
|
||||
bindingRows: [{
|
||||
bindingVersion: 4,
|
||||
bindingState: 'revoked',
|
||||
bindingRole: null,
|
||||
}],
|
||||
});
|
||||
await assert.rejects(
|
||||
repository.requestUserCancellation(command()),
|
||||
(error) =>
|
||||
error instanceof ClusterRunCancellationFenceRejectedError &&
|
||||
error.reason === 'authorization_changed',
|
||||
);
|
||||
assert.equal(calls.some(({ sql }) => sql.includes('FROM "ql3"."runs"')), false);
|
||||
assert.equal(calls.some(({ sql }) => sql === 'ROLLBACK'), true);
|
||||
});
|
||||
|
||||
test('masks cross-Project and missing Runs', async () => {
|
||||
for (const runRows of [[], [run({ projectId: 'project-other' })]]) {
|
||||
const { repository } = fixture({ runRows });
|
||||
await assert.rejects(
|
||||
repository.requestUserCancellation(command()),
|
||||
ClusterRunCancellationNotFoundError,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('binds Workflow cancellation to the immutable admission target', async () => {
|
||||
const targeted = command({
|
||||
workflowTarget: { packageName: 'example', workflowId: 'daily' },
|
||||
});
|
||||
const accepted = fixture();
|
||||
assert.equal(
|
||||
(await accepted.repository.requestUserCancellation(targeted)).status,
|
||||
'accepted',
|
||||
);
|
||||
const admission = accepted.calls.find(({ sql }) =>
|
||||
sql.includes('FROM "ql3"."plugin_package_workflow_admissions"'),
|
||||
);
|
||||
assert.deepEqual(admission.params, ['run-1']);
|
||||
assert.equal(
|
||||
admission.sql.includes('FOR SHARE'),
|
||||
false,
|
||||
'immutable admission lookup must not require UPDATE authority',
|
||||
);
|
||||
|
||||
for (const workflowAdmissionRows of [
|
||||
[],
|
||||
[
|
||||
{
|
||||
projectId: 'project-1',
|
||||
packageName: 'other',
|
||||
workflowId: 'daily',
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
projectId: 'project-1',
|
||||
packageName: 'example',
|
||||
workflowId: 'other',
|
||||
},
|
||||
],
|
||||
]) {
|
||||
const rejected = fixture({ workflowAdmissionRows });
|
||||
await assert.rejects(
|
||||
rejected.repository.requestUserCancellation(targeted),
|
||||
ClusterRunCancellationNotFoundError,
|
||||
);
|
||||
assert.equal(
|
||||
rejected.calls.some(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"'),
|
||||
),
|
||||
false,
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,244 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
PostgresClusterRunLostRetryRepository,
|
||||
} = require('../dist/entrypoints/runtime');
|
||||
|
||||
function runRow(status) {
|
||||
return {
|
||||
id: 'run-1',
|
||||
projectId: 'project-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: 'revision-1',
|
||||
taskName: null,
|
||||
taskSnapshotRef: null,
|
||||
legacyCronId: null,
|
||||
parentRunId: null,
|
||||
retryOfRunId: null,
|
||||
triggerId: null,
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
triggeredBy: null,
|
||||
requestId: null,
|
||||
scheduledForMs: null,
|
||||
status,
|
||||
version: 4,
|
||||
eventSequence: 4,
|
||||
priority: 0,
|
||||
idempotencyKey: null,
|
||||
inputRef: null,
|
||||
outputRef: null,
|
||||
createdAtMs: '100',
|
||||
queuedAtMs: '110',
|
||||
startedAtMs: '150',
|
||||
finishedAtMs: null,
|
||||
cancelRequestedAtMs: null,
|
||||
cancelReason: null,
|
||||
errorCode: 'CLUSTER_RECOVERY_EXECUTION_NOT_RUNNING',
|
||||
errorSummary: 'lost',
|
||||
};
|
||||
}
|
||||
|
||||
function attemptRow() {
|
||||
return {
|
||||
id: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
stepRunId: null,
|
||||
attempt: 1,
|
||||
status: 'lost',
|
||||
executorType: 'remote_worker',
|
||||
workerId: 'worker-1',
|
||||
workerSessionId: null,
|
||||
workerGeneration: null,
|
||||
executorHandle: null,
|
||||
pid: null,
|
||||
logArtifactId: null,
|
||||
leaseToken: null,
|
||||
leaseTokenDigest: null,
|
||||
leaseGeneration: null,
|
||||
leaseVersion: null,
|
||||
leaseExpiresAtMs: null,
|
||||
offerId: null,
|
||||
deadlineAtMs: null,
|
||||
callbackTokenHash: null,
|
||||
callbackSequence: 0,
|
||||
createdAtMs: '120',
|
||||
startedAtMs: '150',
|
||||
finishedAtMs: '200',
|
||||
exitCode: null,
|
||||
errorCode: 'CLUSTER_RECOVERY_EXECUTION_NOT_RUNNING',
|
||||
errorSummary: 'lost',
|
||||
};
|
||||
}
|
||||
|
||||
function policyRow(nextAttemptAtMs = null) {
|
||||
return {
|
||||
runId: 'run-1',
|
||||
maxAttempts: 3,
|
||||
retryOnLost: true,
|
||||
safety: 'idempotent',
|
||||
backoffBaseMs: '1000',
|
||||
backoffMaxMs: '8000',
|
||||
nextAttemptAtMs,
|
||||
version: 0,
|
||||
createdAtMs: '100',
|
||||
updatedAtMs: '100',
|
||||
};
|
||||
}
|
||||
|
||||
function harness(status = 'lost') {
|
||||
const calls = [];
|
||||
let id = 0;
|
||||
const client = {
|
||||
async query(text, values = []) {
|
||||
calls.push({ text, values });
|
||||
if (text.includes('pg_advisory_xact_lock')) {
|
||||
return { rows: [{ locked: true }], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
text.includes('FROM "ql3"."runs" AS run') &&
|
||||
text.includes('FOR UPDATE')
|
||||
) {
|
||||
return { rows: [{ runId: 'run-1' }], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
text.startsWith('SELECT') &&
|
||||
text.includes('FROM "ql3"."runs" WHERE')
|
||||
) {
|
||||
return { rows: [runRow(status)], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
text.startsWith('SELECT') &&
|
||||
text.includes('FROM "ql3"."run_attempts"')
|
||||
) {
|
||||
return { rows: [attemptRow()], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
text.startsWith('SELECT') &&
|
||||
text.includes('FROM "ql3"."run_retry_policies" WHERE') &&
|
||||
!text.includes('FOR UPDATE')
|
||||
) {
|
||||
return {
|
||||
rows: [policyRow(status === 'retry_wait' ? '250' : null)],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes('FROM "ql3"."run_retry_policies"') &&
|
||||
text.includes('FOR UPDATE')
|
||||
) {
|
||||
return { rows: [{ runId: 'run-1' }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('AS "observedAtMs"')) {
|
||||
return { rows: [{ observedAtMs: '300' }], rowCount: 1 };
|
||||
}
|
||||
if (text.startsWith('UPDATE "ql3"."runs"')) {
|
||||
return { rows: [{ id: 'run-1' }], rowCount: 1 };
|
||||
}
|
||||
if (text.startsWith('UPDATE "ql3"."run_retry_policies"')) {
|
||||
return { rows: [{ run_id: 'run-1' }], rowCount: 1 };
|
||||
}
|
||||
if (text.startsWith('INSERT INTO "ql3"."run_attempts"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text.startsWith('INSERT INTO "ql3"."run_events"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
return { rows: [], rowCount: 0 };
|
||||
},
|
||||
release() {},
|
||||
};
|
||||
return {
|
||||
calls,
|
||||
repository: new PostgresClusterRunLostRetryRepository(
|
||||
{
|
||||
async query(text, values) {
|
||||
calls.push({ text, values });
|
||||
assert.match(text, /JOIN LATERAL/);
|
||||
return {
|
||||
rows: [{ runId: 'run-1', attemptId: 'attempt-1' }],
|
||||
rowCount: 1,
|
||||
};
|
||||
},
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
},
|
||||
() => `00000000-0000-4000-8000-${String(++id).padStart(12, '0')}`,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
test('schedules one safe lost Run in a bounded atomic page', async () => {
|
||||
const database = harness('lost');
|
||||
assert.deepEqual(await database.repository.reconcilePage({ limit: 2 }), {
|
||||
scanned: 1,
|
||||
scheduled: 1,
|
||||
requeued: 0,
|
||||
failed: 0,
|
||||
raced: 0,
|
||||
hasMore: false,
|
||||
});
|
||||
assert.equal(
|
||||
database.calls.filter(({ text }) =>
|
||||
text.startsWith('UPDATE "ql3"."runs"'),
|
||||
).length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
database.calls.filter(({ text }) =>
|
||||
text.startsWith('UPDATE "ql3"."run_retry_policies"'),
|
||||
).length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
database.calls.filter(({ text }) =>
|
||||
text.startsWith('INSERT INTO "ql3"."run_events"'),
|
||||
).length,
|
||||
1,
|
||||
);
|
||||
assert.equal(database.calls.at(-1).text, 'COMMIT');
|
||||
});
|
||||
|
||||
test('creates exactly one fresh Attempt when retry_wait is due', async () => {
|
||||
const database = harness('retry_wait');
|
||||
assert.deepEqual(await database.repository.reconcilePage({ limit: 1 }), {
|
||||
scanned: 1,
|
||||
scheduled: 0,
|
||||
requeued: 1,
|
||||
failed: 0,
|
||||
raced: 0,
|
||||
hasMore: false,
|
||||
});
|
||||
assert.equal(
|
||||
database.calls.filter(({ text }) =>
|
||||
text.startsWith('UPDATE "ql3"."runs"'),
|
||||
).length,
|
||||
2,
|
||||
);
|
||||
assert.equal(
|
||||
database.calls.filter(({ text }) =>
|
||||
text.startsWith('INSERT INTO "ql3"."run_attempts"'),
|
||||
).length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
database.calls.filter(({ text }) =>
|
||||
text.startsWith('INSERT INTO "ql3"."run_events"'),
|
||||
).length,
|
||||
2,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects widened pages before querying PostgreSQL', async () => {
|
||||
const database = harness('lost');
|
||||
await assert.rejects(
|
||||
database.repository.reconcilePage({ limit: 65 }),
|
||||
/page size/,
|
||||
);
|
||||
assert.equal(database.calls.length, 0);
|
||||
});
|
||||
@@ -0,0 +1,115 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
PostgresClusterRuntimeRecoverySource,
|
||||
} = require('../dist/entrypoints/runtime');
|
||||
|
||||
function source(rows, calls = []) {
|
||||
return new PostgresClusterRuntimeRecoverySource({
|
||||
async query(text, values) {
|
||||
calls.push({ text, values });
|
||||
return { rows };
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
test('returns only one bounded page of active expired Attempts', async () => {
|
||||
const calls = [];
|
||||
const recovery = source(
|
||||
[
|
||||
{
|
||||
observedAtMs: '1000',
|
||||
attemptId: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
status: 'starting',
|
||||
createdAtMs: '100',
|
||||
},
|
||||
{
|
||||
observedAtMs: '1000',
|
||||
attemptId: 'attempt-2',
|
||||
runId: 'run-2',
|
||||
status: 'running',
|
||||
createdAtMs: '200',
|
||||
},
|
||||
{
|
||||
observedAtMs: '1000',
|
||||
attemptId: 'attempt-3',
|
||||
runId: 'run-3',
|
||||
status: 'claimed',
|
||||
createdAtMs: '300',
|
||||
},
|
||||
],
|
||||
calls,
|
||||
);
|
||||
|
||||
assert.deepEqual(await recovery.listOutstanding(2), {
|
||||
observedAtMs: 1000,
|
||||
candidates: [
|
||||
{
|
||||
kind: 'attempt',
|
||||
id: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
status: 'starting',
|
||||
createdAtMs: 100,
|
||||
},
|
||||
{
|
||||
kind: 'attempt',
|
||||
id: 'attempt-2',
|
||||
runId: 'run-2',
|
||||
status: 'running',
|
||||
createdAtMs: 200,
|
||||
},
|
||||
],
|
||||
hasMore: true,
|
||||
});
|
||||
assert.deepEqual(calls[0].values, [3]);
|
||||
assert.match(calls[0].text, /lease_expires_at_ms <= observation/);
|
||||
assert.match(calls[0].text, /run\.status IN \('dispatching', 'running', 'lost'\)/);
|
||||
assert.match(calls[0].text, /attempt\.worker_id IS NULL/);
|
||||
assert.doesNotMatch(calls[0].text, /status = 'created'/);
|
||||
});
|
||||
|
||||
test('retains a database observation for an empty runtime page', async () => {
|
||||
const recovery = source([
|
||||
{
|
||||
observedAtMs: '1000',
|
||||
attemptId: null,
|
||||
runId: null,
|
||||
status: null,
|
||||
createdAtMs: null,
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(await recovery.listOutstanding(1), {
|
||||
observedAtMs: 1000,
|
||||
candidates: [],
|
||||
hasMore: false,
|
||||
});
|
||||
});
|
||||
|
||||
test('rejects malformed rows and unbounded requests', async () => {
|
||||
let queries = 0;
|
||||
const recovery = new PostgresClusterRuntimeRecoverySource({
|
||||
async query() {
|
||||
queries += 1;
|
||||
return { rows: [] };
|
||||
},
|
||||
});
|
||||
await assert.rejects(recovery.listOutstanding(0), /between 1 and 128/);
|
||||
assert.equal(queries, 0);
|
||||
|
||||
await assert.rejects(
|
||||
source([
|
||||
{
|
||||
observedAtMs: '1000',
|
||||
attemptId: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
status: 'succeeded',
|
||||
createdAtMs: '100',
|
||||
},
|
||||
]).listOutstanding(1),
|
||||
/Attempt status is invalid/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,443 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
resolveClusterScheduleDecision,
|
||||
} = require('@qinglong/runtime-core/cluster-scheduler');
|
||||
const {
|
||||
createTaskDefinitionRecord,
|
||||
} = require('@qinglong/runtime-core/task-definition');
|
||||
const {
|
||||
createBuiltInTaskSpecSemanticRegistry,
|
||||
} = require('@qinglong/runtime-core/task-spec-semantic');
|
||||
const {
|
||||
compileClusterCommandTaskDefinition,
|
||||
} = require('@qinglong/runtime-core/cluster-execution-revision');
|
||||
const {
|
||||
createBuiltInTriggerSpecSemanticRegistry,
|
||||
createTriggerRecord,
|
||||
} = require('@qinglong/runtime-core/trigger');
|
||||
const {
|
||||
PostgresClusterScheduleRepository,
|
||||
PostgresClusterScheduleUnavailableError,
|
||||
} = require('../dist/scheduling/clusterScheduleRepository');
|
||||
|
||||
function nextMinute(schedule, afterMs) {
|
||||
if (schedule.expression !== '* * * * *' || schedule.timezone !== 'UTC') {
|
||||
throw new Error('unsupported test schedule');
|
||||
}
|
||||
return Math.floor(afterMs / 60_000 + 1) * 60_000;
|
||||
}
|
||||
|
||||
const CLAIM_TOKEN = '019f7700-0000-7000-8000-000000000001';
|
||||
const TASK_COMMAND = Object.freeze({
|
||||
projectId: 'default',
|
||||
taskId: 'task-00001',
|
||||
expectedRevision: null,
|
||||
mutationId: '019f7700-0000-7000-8000-000000000010',
|
||||
name: 'Scheduled Task',
|
||||
kind: 'command',
|
||||
spec: Object.freeze({
|
||||
schema: 'qinglong/command@v1',
|
||||
config: Object.freeze({
|
||||
command: Object.freeze({
|
||||
kind: 'argv',
|
||||
file: '/bin/echo',
|
||||
args: Object.freeze(['scheduled']),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
labels: Object.freeze({}),
|
||||
enabled: true,
|
||||
occurredAtMs: 101,
|
||||
});
|
||||
const taskSemantics = createBuiltInTaskSpecSemanticRegistry();
|
||||
const TASK = createTaskDefinitionRecord(
|
||||
Object.freeze({
|
||||
...TASK_COMMAND,
|
||||
spec: taskSemantics.normalize({
|
||||
projectId: TASK_COMMAND.projectId,
|
||||
taskId: TASK_COMMAND.taskId,
|
||||
kind: TASK_COMMAND.kind,
|
||||
spec: TASK_COMMAND.spec,
|
||||
}),
|
||||
}),
|
||||
TASK_COMMAND.occurredAtMs,
|
||||
);
|
||||
const EXECUTION = compileClusterCommandTaskDefinition(TASK, taskSemantics);
|
||||
const TRIGGER_COMMAND = Object.freeze({
|
||||
projectId: TASK.projectId,
|
||||
triggerId: 'trigger-00001',
|
||||
expectedRevision: null,
|
||||
mutationId: '019f7700-0000-7000-8000-000000000011',
|
||||
taskId: TASK.taskId,
|
||||
taskRevision: TASK.revision,
|
||||
taskContentDigest: TASK.contentDigest,
|
||||
spec: Object.freeze({
|
||||
schema: 'qinglong/cron@v1',
|
||||
config: Object.freeze({
|
||||
expression: '* * * * *',
|
||||
timezone: 'UTC',
|
||||
misfirePolicy: 'skip',
|
||||
}),
|
||||
}),
|
||||
enabled: true,
|
||||
occurredAtMs: 201,
|
||||
});
|
||||
const triggerSemantics = createBuiltInTriggerSpecSemanticRegistry();
|
||||
const TRIGGER = createTriggerRecord(
|
||||
Object.freeze({
|
||||
...TRIGGER_COMMAND,
|
||||
spec: triggerSemantics.normalize({
|
||||
projectId: TRIGGER_COMMAND.projectId,
|
||||
triggerId: TRIGGER_COMMAND.triggerId,
|
||||
taskId: TRIGGER_COMMAND.taskId,
|
||||
taskRevision: TRIGGER_COMMAND.taskRevision,
|
||||
spec: TRIGGER_COMMAND.spec,
|
||||
}),
|
||||
}),
|
||||
TRIGGER_COMMAND.occurredAtMs,
|
||||
);
|
||||
|
||||
function claim(overrides = {}) {
|
||||
return {
|
||||
projectId: TRIGGER.projectId,
|
||||
triggerId: TRIGGER.triggerId,
|
||||
triggerRevision: TRIGGER.revision,
|
||||
triggerContentDigest: TRIGGER.contentDigest,
|
||||
triggerUpdatedAtMs: TRIGGER.updatedAtMs,
|
||||
taskId: TRIGGER.taskId,
|
||||
taskRevision: TRIGGER.taskRevision,
|
||||
taskContentDigest: TRIGGER.taskContentDigest,
|
||||
expression: TRIGGER.spec.config.expression,
|
||||
timezone: TRIGGER.spec.config.timezone,
|
||||
misfirePolicy: TRIGGER.spec.config.misfirePolicy,
|
||||
stateVersion: 1,
|
||||
nextFireAtMs: 60_000,
|
||||
claimOwner: 'scheduler-a',
|
||||
claimToken: CLAIM_TOKEN,
|
||||
claimVersion: 1,
|
||||
claimAcquiredAtMs: 61_000,
|
||||
claimExpiresAtMs: 91_000,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function claimRow(overrides = {}) {
|
||||
const value = claim(overrides);
|
||||
return {
|
||||
projectId: value.projectId,
|
||||
triggerId: value.triggerId,
|
||||
triggerRevision: value.triggerRevision,
|
||||
triggerContentDigest: value.triggerContentDigest,
|
||||
triggerUpdatedAtMs: String(value.triggerUpdatedAtMs),
|
||||
taskId: value.taskId,
|
||||
taskRevision: value.taskRevision,
|
||||
taskContentDigest: value.taskContentDigest,
|
||||
specJson: TRIGGER.spec,
|
||||
taskName: TASK.name,
|
||||
stateVersion: value.stateVersion,
|
||||
nextFireAtMs:
|
||||
value.nextFireAtMs === null ? null : String(value.nextFireAtMs),
|
||||
claimOwner: value.claimOwner,
|
||||
claimToken: value.claimToken,
|
||||
claimVersion: value.claimVersion,
|
||||
claimAcquiredAtMs: String(value.claimAcquiredAtMs),
|
||||
claimExpiresAtMs: String(value.claimExpiresAtMs),
|
||||
commitObservedAtMs: String(overrides.commitObservedAtMs ?? 62_000),
|
||||
};
|
||||
}
|
||||
|
||||
function executionRow(overrides = {}) {
|
||||
return {
|
||||
projectId: EXECUTION.projectId,
|
||||
taskId: EXECUTION.taskId,
|
||||
sourceRevision: EXECUTION.sourceRevision,
|
||||
taskRevision: EXECUTION.taskRevision,
|
||||
sourceContentDigest: EXECUTION.sourceContentDigest,
|
||||
executorType: EXECUTION.executorType,
|
||||
planSchema: EXECUTION.planSchema,
|
||||
planJson: {
|
||||
command: EXECUTION.command,
|
||||
environment: EXECUTION.environment,
|
||||
...(EXECUTION.workingDirectory === undefined
|
||||
? {}
|
||||
: { workingDirectory: EXECUTION.workingDirectory }),
|
||||
...(EXECUTION.timeoutMs === undefined
|
||||
? {}
|
||||
: { timeoutMs: EXECUTION.timeoutMs }),
|
||||
...(EXECUTION.placement === undefined
|
||||
? {}
|
||||
: { placement: EXECUTION.placement }),
|
||||
},
|
||||
contentDigest: EXECUTION.contentDigest,
|
||||
createdAtMs: String(EXECUTION.createdAtMs),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(options = {}) {
|
||||
const queries = [];
|
||||
let commitFailuresRemaining = options.failCommitOnce ? 1 : 0;
|
||||
const client = {
|
||||
async query(text, values) {
|
||||
queries.push({ text, values });
|
||||
if (text === 'COMMIT' && commitFailuresRemaining > 0) {
|
||||
commitFailuresRemaining -= 1;
|
||||
const error = new Error('injected committed response loss');
|
||||
error.code = '40001';
|
||||
throw error;
|
||||
}
|
||||
if (text.includes('FOR UPDATE OF schedule')) {
|
||||
return {
|
||||
rows: options.missingClaim
|
||||
? []
|
||||
: [
|
||||
claimRow({
|
||||
...options.claimOverrides,
|
||||
commitObservedAtMs: options.commitObservedAtMs,
|
||||
}),
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."task_execution_revisions"')) {
|
||||
return {
|
||||
rows: options.missingExecution
|
||||
? []
|
||||
: [executionRow(options.executionOverrides)],
|
||||
};
|
||||
}
|
||||
if (
|
||||
options.failAt &&
|
||||
text.includes(`INSERT INTO "ql3"."${options.failAt}"`)
|
||||
) {
|
||||
throw new Error('injected write failure');
|
||||
}
|
||||
if (text.includes('UPDATE "ql3"."trigger_schedules"')) {
|
||||
return { rows: [], rowCount: options.advanceRaced ? 0 : 1 };
|
||||
}
|
||||
return { rows: [], rowCount: 1 };
|
||||
},
|
||||
release() {
|
||||
queries.push({ text: 'RELEASE' });
|
||||
},
|
||||
};
|
||||
return {
|
||||
queries,
|
||||
pool: {
|
||||
async query(text, values) {
|
||||
queries.push({ text, values });
|
||||
return {
|
||||
rows: options.noDue ? [] : [claimRow(options.claimOverrides)],
|
||||
};
|
||||
},
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function admissionCommand(claimed = claim()) {
|
||||
return {
|
||||
claim: claimed,
|
||||
decision: resolveClusterScheduleDecision(claimed, 5_000, nextMinute),
|
||||
runId: '019f7700-0000-7000-8000-000000000020',
|
||||
attemptId: '019f7700-0000-7000-8000-000000000021',
|
||||
createdEventId: '019f7700-0000-7000-8000-000000000022',
|
||||
queuedEventId: '019f7700-0000-7000-8000-000000000023',
|
||||
};
|
||||
}
|
||||
|
||||
test('claims one due schedule with ordered SKIP LOCKED lease takeover', async () => {
|
||||
const db = fixture();
|
||||
const repository = new PostgresClusterScheduleRepository(db.pool);
|
||||
assert.deepEqual(
|
||||
await repository.claimNextClusterSchedule({
|
||||
ownerId: 'scheduler-a',
|
||||
claimToken: CLAIM_TOKEN,
|
||||
leaseMs: 30_000,
|
||||
}),
|
||||
claim(),
|
||||
);
|
||||
assert.equal(db.queries.length, 1);
|
||||
assert.match(db.queries[0].text, /FOR UPDATE OF schedule SKIP LOCKED/);
|
||||
assert.match(db.queries[0].text, /clock_timestamp\(\)/);
|
||||
assert.match(
|
||||
db.queries[0].text,
|
||||
/schedule\.claim_expires_at_ms <= observation\.observed_at_ms/,
|
||||
);
|
||||
assert.match(db.queries[0].text, /NULLS FIRST/);
|
||||
assert.match(
|
||||
db.queries[0].text,
|
||||
/task_head\.current_revision = task\.revision/,
|
||||
);
|
||||
assert.deepEqual(db.queries[0].values, ['scheduler-a', CLAIM_TOKEN, 30_000]);
|
||||
|
||||
const empty = fixture({ noDue: true });
|
||||
assert.equal(
|
||||
await new PostgresClusterScheduleRepository(
|
||||
empty.pool,
|
||||
).claimNextClusterSchedule({
|
||||
ownerId: 'scheduler-a',
|
||||
claimToken: CLAIM_TOKEN,
|
||||
leaseMs: 30_000,
|
||||
}),
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
test('admits Run, Attempt and two events before advancing the exact claim', async () => {
|
||||
const db = fixture();
|
||||
const result = await new PostgresClusterScheduleRepository(
|
||||
db.pool,
|
||||
).commitClusterScheduleDecision(admissionCommand());
|
||||
assert.deepEqual(result, {
|
||||
status: 'admitted',
|
||||
disposition: 'admit',
|
||||
runId: '019f7700-0000-7000-8000-000000000020',
|
||||
attemptId: '019f7700-0000-7000-8000-000000000021',
|
||||
});
|
||||
const sql = db.queries.map(({ text }) => text);
|
||||
assert.match(
|
||||
sql.find((text) => text.includes('FOR UPDATE OF schedule')),
|
||||
/task_head\.current_revision = task\.revision/,
|
||||
);
|
||||
const run = sql.findIndex((text) =>
|
||||
text.includes('INSERT INTO "ql3"."runs"'),
|
||||
);
|
||||
const attempt = sql.findIndex((text) =>
|
||||
text.includes('INSERT INTO "ql3"."run_attempts"'),
|
||||
);
|
||||
const events = sql.filter((text) =>
|
||||
text.includes('INSERT INTO "ql3"."run_events"'),
|
||||
);
|
||||
const advance = sql.findIndex((text) =>
|
||||
text.includes('UPDATE "ql3"."trigger_schedules"'),
|
||||
);
|
||||
assert.ok(
|
||||
run > 0 && attempt > run && events.length === 2 && advance > attempt,
|
||||
);
|
||||
assert.match(sql[attempt], /'remote_worker'/);
|
||||
assert.match(sql[advance], /claim_token = \$10::uuid/);
|
||||
assert.equal(
|
||||
db.queries
|
||||
.find(({ text }) => text.includes('INSERT INTO "ql3"."runs"'))
|
||||
.values.at(-1),
|
||||
62_000,
|
||||
);
|
||||
assert.equal(sql.includes('COMMIT'), true);
|
||||
assert.equal(sql.at(-1), 'RELEASE');
|
||||
});
|
||||
|
||||
test('does not retry after COMMIT was sent and its response is lost', async () => {
|
||||
const db = fixture({ failCommitOnce: true });
|
||||
await assert.rejects(
|
||||
new PostgresClusterScheduleRepository(
|
||||
db.pool,
|
||||
).commitClusterScheduleDecision(admissionCommand()),
|
||||
PostgresClusterScheduleUnavailableError,
|
||||
);
|
||||
const sql = db.queries.map(({ text }) => text);
|
||||
assert.equal(sql.filter((text) => text === 'COMMIT').length, 1);
|
||||
assert.equal(sql.includes('ROLLBACK'), false);
|
||||
});
|
||||
|
||||
test('returns raced without writes when the durable claim fence changed', async () => {
|
||||
const db = fixture({ claimOverrides: { stateVersion: 2 } });
|
||||
assert.deepEqual(
|
||||
await new PostgresClusterScheduleRepository(
|
||||
db.pool,
|
||||
).commitClusterScheduleDecision(admissionCommand()),
|
||||
{ status: 'raced' },
|
||||
);
|
||||
const sql = db.queries.map(({ text }) => text);
|
||||
assert.equal(
|
||||
sql.some((text) => text.includes('INSERT INTO')),
|
||||
false,
|
||||
);
|
||||
assert.equal(sql.includes('ROLLBACK'), true);
|
||||
});
|
||||
|
||||
test('uses the database commit clock for expiry and rejects clock regression', async () => {
|
||||
const expired = fixture({ commitObservedAtMs: 91_000 });
|
||||
assert.deepEqual(
|
||||
await new PostgresClusterScheduleRepository(
|
||||
expired.pool,
|
||||
).commitClusterScheduleDecision(admissionCommand()),
|
||||
{ status: 'raced' },
|
||||
);
|
||||
assert.equal(
|
||||
expired.queries.some(({ text }) => text.includes('INSERT INTO')),
|
||||
false,
|
||||
);
|
||||
|
||||
const backwards = fixture({ commitObservedAtMs: 60_999 });
|
||||
await assert.rejects(
|
||||
new PostgresClusterScheduleRepository(
|
||||
backwards.pool,
|
||||
).commitClusterScheduleDecision(admissionCommand()),
|
||||
/clock moved backwards/,
|
||||
);
|
||||
assert.equal(
|
||||
backwards.queries.some(({ text }) => text === 'ROLLBACK'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('advances a skipped occurrence without creating a Run', async () => {
|
||||
const claimed = claim({
|
||||
claimAcquiredAtMs: 900_000,
|
||||
claimExpiresAtMs: 930_000,
|
||||
});
|
||||
const db = fixture({
|
||||
claimOverrides: {
|
||||
claimAcquiredAtMs: 900_000,
|
||||
claimExpiresAtMs: 930_000,
|
||||
},
|
||||
commitObservedAtMs: 900_001,
|
||||
});
|
||||
const decision = resolveClusterScheduleDecision(claimed, 0, nextMinute);
|
||||
assert.equal(decision.disposition, 'skip');
|
||||
assert.deepEqual(
|
||||
await new PostgresClusterScheduleRepository(
|
||||
db.pool,
|
||||
).commitClusterScheduleDecision({ claim: claimed, decision }),
|
||||
{ status: 'advanced', disposition: 'skip' },
|
||||
);
|
||||
assert.equal(
|
||||
db.queries.some(({ text }) => text.includes('INSERT INTO "ql3"."runs"')),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('rolls back partial admission and rejects corrupt execution revisions', async () => {
|
||||
const failed = fixture({ failAt: 'run_attempts' });
|
||||
await assert.rejects(
|
||||
new PostgresClusterScheduleRepository(
|
||||
failed.pool,
|
||||
).commitClusterScheduleDecision(admissionCommand()),
|
||||
PostgresClusterScheduleUnavailableError,
|
||||
);
|
||||
assert.equal(
|
||||
failed.queries.some(({ text }) => text === 'ROLLBACK'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
failed.queries.some(({ text }) => text === 'COMMIT'),
|
||||
false,
|
||||
);
|
||||
|
||||
const corrupt = fixture({
|
||||
executionOverrides: { contentDigest: 'f'.repeat(64) },
|
||||
});
|
||||
await assert.rejects(
|
||||
new PostgresClusterScheduleRepository(
|
||||
corrupt.pool,
|
||||
).commitClusterScheduleDecision(admissionCommand()),
|
||||
PostgresClusterScheduleUnavailableError,
|
||||
);
|
||||
assert.equal(
|
||||
corrupt.queries.some(({ text }) => text === 'ROLLBACK'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,101 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PostgresConnectionEnvironmentError,
|
||||
loadPostgresConnectionEnvironment,
|
||||
} = require('../dist/entrypoints/runtime.js');
|
||||
|
||||
const KEYS = Object.freeze({
|
||||
connectionString: 'POSTGRES_URL',
|
||||
host: 'POSTGRES_HOST',
|
||||
port: 'POSTGRES_PORT',
|
||||
database: 'POSTGRES_DATABASE',
|
||||
user: 'POSTGRES_USER',
|
||||
password: 'POSTGRES_PASSWORD',
|
||||
});
|
||||
|
||||
test('loads a legacy URL without exposing TLS query overrides', () => {
|
||||
assert.deepEqual(
|
||||
loadPostgresConnectionEnvironment(
|
||||
{
|
||||
POSTGRES_URL:
|
||||
'postgresql://ql3_runtime:secret@postgres-rw.internal:5432/qinglong',
|
||||
},
|
||||
KEYS,
|
||||
),
|
||||
{
|
||||
connectionString:
|
||||
'postgresql://ql3_runtime:secret@postgres-rw.internal:5432/qinglong',
|
||||
},
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
loadPostgresConnectionEnvironment(
|
||||
{
|
||||
POSTGRES_URL:
|
||||
'postgresql://ql3_runtime:secret@postgres-rw.internal/qinglong?sslmode=disable',
|
||||
},
|
||||
KEYS,
|
||||
),
|
||||
/TLS query parameters are forbidden/,
|
||||
);
|
||||
});
|
||||
|
||||
test('loads an exact discrete operator credential with the default port', () => {
|
||||
assert.deepEqual(
|
||||
loadPostgresConnectionEnvironment(
|
||||
{
|
||||
POSTGRES_HOST: 'ql3-postgres-rw.qinglong3-system.svc',
|
||||
POSTGRES_DATABASE: 'qinglong',
|
||||
POSTGRES_USER: 'ql3_runtime',
|
||||
POSTGRES_PASSWORD: 'secret',
|
||||
},
|
||||
KEYS,
|
||||
),
|
||||
{
|
||||
host: 'ql3-postgres-rw.qinglong3-system.svc',
|
||||
port: 5432,
|
||||
database: 'qinglong',
|
||||
user: 'ql3_runtime',
|
||||
password: 'secret',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects mixed, partial, unbounded and unsafe discrete credentials', () => {
|
||||
for (const environment of [
|
||||
{
|
||||
POSTGRES_URL: 'postgresql://ql3_runtime:secret@database/qinglong',
|
||||
POSTGRES_HOST: 'database',
|
||||
},
|
||||
{
|
||||
POSTGRES_HOST: 'database',
|
||||
POSTGRES_DATABASE: 'qinglong',
|
||||
POSTGRES_USER: 'ql3_runtime',
|
||||
},
|
||||
{
|
||||
POSTGRES_HOST: 'database',
|
||||
POSTGRES_PORT: '0',
|
||||
POSTGRES_DATABASE: 'qinglong',
|
||||
POSTGRES_USER: 'ql3_runtime',
|
||||
POSTGRES_PASSWORD: 'secret',
|
||||
},
|
||||
{
|
||||
POSTGRES_HOST: 'database',
|
||||
POSTGRES_DATABASE: 'qinglong',
|
||||
POSTGRES_USER: 'role-with-hyphen',
|
||||
POSTGRES_PASSWORD: 'secret',
|
||||
},
|
||||
{
|
||||
POSTGRES_HOST: 'database',
|
||||
POSTGRES_DATABASE: 'qinglong',
|
||||
POSTGRES_USER: 'ql3_runtime',
|
||||
POSTGRES_PASSWORD: 'line\nbreak',
|
||||
},
|
||||
]) {
|
||||
assert.throws(
|
||||
() => loadPostgresConnectionEnvironment(environment, KEYS),
|
||||
PostgresConnectionEnvironmentError,
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,880 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
TaskDefinitionConflictError,
|
||||
TaskDefinitionUnavailableError,
|
||||
createTaskDefinitionRecord,
|
||||
} = require('@qinglong/runtime-core/task-definition');
|
||||
const {
|
||||
createBuiltInTaskSpecSemanticRegistry,
|
||||
} = require('@qinglong/runtime-core/task-spec-semantic');
|
||||
const {
|
||||
TriggerConflictError,
|
||||
TriggerUnavailableError,
|
||||
createBuiltInTriggerSpecSemanticRegistry,
|
||||
createTriggerRecord,
|
||||
} = require('@qinglong/runtime-core/trigger');
|
||||
const {
|
||||
compileClusterCommandTaskDefinition,
|
||||
} = require('@qinglong/runtime-core/cluster-execution-revision');
|
||||
const {
|
||||
TaskDefinitionAdministrationAuthorizationFenceConflictError,
|
||||
TaskDefinitionAdministrationMutationConflictError,
|
||||
TaskDefinitionAdministrationReadConflictError,
|
||||
} = require('@qinglong/runtime-core/task-definition-administration');
|
||||
const {
|
||||
TriggerAdministrationAuthorizationFenceConflictError,
|
||||
TriggerAdministrationReadConflictError,
|
||||
} = require('@qinglong/runtime-core/trigger-administration');
|
||||
const {
|
||||
PostgresTaskDefinitionRepository,
|
||||
PostgresTaskDefinitionSource,
|
||||
PostgresTaskExecutionRevisionSource,
|
||||
} = require('../dist/automation/taskDefinitionRepository');
|
||||
const {
|
||||
PostgresTriggerRepository,
|
||||
PostgresTriggerSource,
|
||||
} = require('../dist/scheduling/triggerRepository');
|
||||
const {
|
||||
PostgresTaskDefinitionAdministrationRepository,
|
||||
PostgresTriggerAdministrationRepository,
|
||||
} = require('../dist/automation/automationAdministrationRepository');
|
||||
|
||||
const TASK_COMMAND = Object.freeze({
|
||||
projectId: 'default',
|
||||
taskId: 'task-00001',
|
||||
expectedRevision: null,
|
||||
mutationId: '123e4567-e89b-42d3-a456-426614174001',
|
||||
name: 'Task 1',
|
||||
kind: 'command',
|
||||
spec: Object.freeze({
|
||||
schema: 'qinglong/command@v1',
|
||||
config: Object.freeze({
|
||||
command: Object.freeze({
|
||||
kind: 'argv',
|
||||
file: '/bin/echo',
|
||||
args: Object.freeze(['1']),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
labels: Object.freeze({ source: 'postgres-test' }),
|
||||
enabled: true,
|
||||
occurredAtMs: 101,
|
||||
});
|
||||
|
||||
const NORMALIZED_TASK_COMMAND = Object.freeze({
|
||||
...TASK_COMMAND,
|
||||
spec: createBuiltInTaskSpecSemanticRegistry().normalize({
|
||||
projectId: TASK_COMMAND.projectId,
|
||||
taskId: TASK_COMMAND.taskId,
|
||||
kind: TASK_COMMAND.kind,
|
||||
spec: TASK_COMMAND.spec,
|
||||
}),
|
||||
});
|
||||
const TASK = createTaskDefinitionRecord(
|
||||
NORMALIZED_TASK_COMMAND,
|
||||
TASK_COMMAND.occurredAtMs,
|
||||
);
|
||||
const DRIFT_TASK = createTaskDefinitionRecord(
|
||||
Object.freeze({ ...NORMALIZED_TASK_COMMAND, name: 'drift' }),
|
||||
TASK_COMMAND.occurredAtMs,
|
||||
);
|
||||
const TASK_EXECUTION = compileClusterCommandTaskDefinition(
|
||||
TASK,
|
||||
createBuiltInTaskSpecSemanticRegistry(),
|
||||
);
|
||||
|
||||
const TRIGGER_COMMAND = Object.freeze({
|
||||
projectId: 'default',
|
||||
triggerId: 'trigger-00001',
|
||||
expectedRevision: null,
|
||||
mutationId: '123e4567-e89b-42d3-a456-426614174002',
|
||||
taskId: TASK.taskId,
|
||||
taskRevision: TASK.revision,
|
||||
taskContentDigest: TASK.contentDigest,
|
||||
spec: Object.freeze({
|
||||
schema: 'qinglong/cron@v1',
|
||||
config: Object.freeze({
|
||||
expression: '*/5 * * * *',
|
||||
timezone: 'Etc/UTC',
|
||||
misfirePolicy: 'skip',
|
||||
}),
|
||||
}),
|
||||
enabled: true,
|
||||
occurredAtMs: 201,
|
||||
});
|
||||
const NORMALIZED_TRIGGER_COMMAND = Object.freeze({
|
||||
...TRIGGER_COMMAND,
|
||||
spec: createBuiltInTriggerSpecSemanticRegistry().normalize({
|
||||
projectId: TRIGGER_COMMAND.projectId,
|
||||
triggerId: TRIGGER_COMMAND.triggerId,
|
||||
taskId: TRIGGER_COMMAND.taskId,
|
||||
taskRevision: TRIGGER_COMMAND.taskRevision,
|
||||
spec: TRIGGER_COMMAND.spec,
|
||||
}),
|
||||
});
|
||||
const TRIGGER = createTriggerRecord(
|
||||
NORMALIZED_TRIGGER_COMMAND,
|
||||
TRIGGER_COMMAND.occurredAtMs,
|
||||
);
|
||||
const DRIFT_TRIGGER = createTriggerRecord(
|
||||
Object.freeze({ ...NORMALIZED_TRIGGER_COMMAND, enabled: false }),
|
||||
TRIGGER_COMMAND.occurredAtMs,
|
||||
);
|
||||
const ACTOR = Object.freeze({ type: 'user', id: 'usr_cluster_admin' });
|
||||
const FENCE = Object.freeze({ projectVersion: 1, bindingVersion: 1 });
|
||||
const TASK_AUDIT = Object.freeze({
|
||||
eventId: TASK_COMMAND.mutationId,
|
||||
requestId: 'request-task-create-00001',
|
||||
operationId: 'task.create',
|
||||
projectId: TASK_COMMAND.projectId,
|
||||
subject: ACTOR,
|
||||
authenticationId: 'oidc:session-task-00001',
|
||||
outcome: 'allowed',
|
||||
reasons: Object.freeze(['role_grant']),
|
||||
fence: FENCE,
|
||||
occurredAtMs: 102,
|
||||
});
|
||||
const TRIGGER_AUDIT = Object.freeze({
|
||||
...TASK_AUDIT,
|
||||
eventId: TRIGGER_COMMAND.mutationId,
|
||||
requestId: 'request-trigger-create-00001',
|
||||
operationId: 'trigger.create',
|
||||
occurredAtMs: 202,
|
||||
});
|
||||
const TASK_READ_AUDIT = Object.freeze({
|
||||
...TASK_AUDIT,
|
||||
eventId: '123e4567-e89b-42d3-a456-426614174010',
|
||||
requestId: 'request-task-read-00001',
|
||||
operationId: 'task.read',
|
||||
occurredAtMs: 302,
|
||||
});
|
||||
const TRIGGER_READ_AUDIT = Object.freeze({
|
||||
...TASK_AUDIT,
|
||||
eventId: '123e4567-e89b-42d3-a456-426614174011',
|
||||
requestId: 'request-trigger-read-00001',
|
||||
operationId: 'trigger.read',
|
||||
occurredAtMs: 303,
|
||||
});
|
||||
|
||||
function administrationAuditRow(audit) {
|
||||
return {
|
||||
auditEventId: audit.eventId,
|
||||
auditRequestId: audit.requestId,
|
||||
auditOperationId: audit.operationId,
|
||||
auditProjectId: audit.projectId,
|
||||
auditSubjectType: audit.subject.type,
|
||||
auditSubjectId: audit.subject.id,
|
||||
auditAuthenticationId: audit.authenticationId,
|
||||
auditOutcome: audit.outcome,
|
||||
auditReasons: [...audit.reasons],
|
||||
auditProjectVersion: audit.fence.projectVersion,
|
||||
auditBindingVersion: audit.fence.bindingVersion,
|
||||
auditOccurredAtMs: String(audit.occurredAtMs),
|
||||
};
|
||||
}
|
||||
|
||||
function taskRow(overrides = {}) {
|
||||
return {
|
||||
projectId: TASK.projectId,
|
||||
taskId: TASK.taskId,
|
||||
revision: TASK.revision,
|
||||
mutationId: TASK.mutationId,
|
||||
name: TASK.name,
|
||||
description: null,
|
||||
kind: TASK.kind,
|
||||
specJson: TASK.spec,
|
||||
labelsJson: TASK.labels,
|
||||
enabled: TASK.enabled,
|
||||
contentDigest: TASK.contentDigest,
|
||||
createdAtMs: String(TASK.createdAtMs),
|
||||
updatedAtMs: String(TASK.updatedAtMs),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function triggerRow(overrides = {}) {
|
||||
return {
|
||||
projectId: TRIGGER.projectId,
|
||||
triggerId: TRIGGER.triggerId,
|
||||
revision: TRIGGER.revision,
|
||||
mutationId: TRIGGER.mutationId,
|
||||
taskId: TRIGGER.taskId,
|
||||
taskRevision: TRIGGER.taskRevision,
|
||||
taskContentDigest: TRIGGER.taskContentDigest,
|
||||
specJson: TRIGGER.spec,
|
||||
enabled: TRIGGER.enabled,
|
||||
contentDigest: TRIGGER.contentDigest,
|
||||
createdAtMs: String(TRIGGER.createdAtMs),
|
||||
updatedAtMs: String(TRIGGER.updatedAtMs),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function taskExecutionRow(overrides = {}) {
|
||||
return {
|
||||
projectId: TASK_EXECUTION.projectId,
|
||||
taskId: TASK_EXECUTION.taskId,
|
||||
sourceRevision: TASK_EXECUTION.sourceRevision,
|
||||
taskRevision: TASK_EXECUTION.taskRevision,
|
||||
sourceContentDigest: TASK_EXECUTION.sourceContentDigest,
|
||||
executorType: TASK_EXECUTION.executorType,
|
||||
planSchema: TASK_EXECUTION.planSchema,
|
||||
planJson: {
|
||||
command: TASK_EXECUTION.command,
|
||||
environment: TASK_EXECUTION.environment,
|
||||
...(TASK_EXECUTION.workingDirectory === undefined
|
||||
? {}
|
||||
: { workingDirectory: TASK_EXECUTION.workingDirectory }),
|
||||
...(TASK_EXECUTION.timeoutMs === undefined
|
||||
? {}
|
||||
: { timeoutMs: TASK_EXECUTION.timeoutMs }),
|
||||
...(TASK_EXECUTION.placement === undefined
|
||||
? {}
|
||||
: { placement: TASK_EXECUTION.placement }),
|
||||
},
|
||||
contentDigest: TASK_EXECUTION.contentDigest,
|
||||
createdAtMs: String(TASK_EXECUTION.createdAtMs),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function sourcePool(rows) {
|
||||
const queries = [];
|
||||
return {
|
||||
queries,
|
||||
pool: {
|
||||
async query(text, values) {
|
||||
queries.push({ text, values });
|
||||
return { rows };
|
||||
},
|
||||
async connect() {
|
||||
throw new Error('not used');
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function appendPool(kind, options = {}) {
|
||||
const events = [];
|
||||
const queries = [];
|
||||
let connection = 0;
|
||||
return {
|
||||
events,
|
||||
queries,
|
||||
pool: {
|
||||
async query() {
|
||||
throw new Error('not used');
|
||||
},
|
||||
async connect() {
|
||||
connection += 1;
|
||||
return {
|
||||
async query(text, values) {
|
||||
events.push(text.trim().split('\n', 1)[0]);
|
||||
queries.push({ text, values });
|
||||
if (text.includes('WHERE revision.mutation_id = $1')) {
|
||||
return {
|
||||
rows: options.replay
|
||||
? [kind === 'task' ? taskRow(options.replay) : triggerRow(options.replay)]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."task_execution_revisions"')) {
|
||||
return {
|
||||
rows: options.missingExecution ? [] : [taskExecutionRow()],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."trigger_schedules"')) {
|
||||
return {
|
||||
rows: options.missingSchedule
|
||||
? []
|
||||
: [{ triggerRevision: TRIGGER.revision }],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."projects"')) {
|
||||
return {
|
||||
rows: [{
|
||||
status: options.projectStatus ?? 'active',
|
||||
version: options.projectVersion ?? 1,
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."project_role_bindings"')) {
|
||||
return {
|
||||
rows: options.missingBinding
|
||||
? []
|
||||
: [{
|
||||
version: options.bindingVersion ?? 1,
|
||||
state: options.bindingState ?? 'active',
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."security_audit_events"')) {
|
||||
return {
|
||||
rows: options.audit ? [administrationAuditRow(options.audit)] : [],
|
||||
};
|
||||
}
|
||||
if (text.includes('INSERT INTO "ql3"."task_definitions"')) {
|
||||
return { rows: [{ taskId: TASK.taskId }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."task_definitions"') &&
|
||||
text.includes('FOR UPDATE')) {
|
||||
return {
|
||||
rows: [{
|
||||
currentRevision: 1,
|
||||
createdAtMs: '101',
|
||||
updatedAtMs: '101',
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (text.includes('INSERT INTO "ql3"."task_definition_revisions"')) {
|
||||
if (options.retryOnce && connection === 1) {
|
||||
throw Object.assign(new Error('serialization'), { code: '40001' });
|
||||
}
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('INSERT INTO "ql3"."triggers"')) {
|
||||
return { rows: [{ triggerId: TRIGGER.triggerId }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."triggers"') &&
|
||||
text.includes('FOR UPDATE')) {
|
||||
return {
|
||||
rows: [{
|
||||
taskId: TASK.taskId,
|
||||
currentRevision: 1,
|
||||
createdAtMs: '201',
|
||||
updatedAtMs: '201',
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."task_definitions"') &&
|
||||
text.includes('revision.revision = $3')) {
|
||||
return { rows: options.missingTask ? [] : [taskRow(options.task)] };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."task_definitions"') &&
|
||||
text.includes('JOIN "ql3"."task_definition_revisions"')) {
|
||||
return { rows: options.readAbsent ? [] : [taskRow(options.task)] };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."triggers"') &&
|
||||
text.includes('JOIN "ql3"."trigger_revisions"')) {
|
||||
return {
|
||||
rows: options.readAbsent ? [] : [triggerRow(options.trigger)],
|
||||
};
|
||||
}
|
||||
if (text.includes('INSERT INTO "ql3"."trigger_revisions"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
return { rows: [], rowCount: 1 };
|
||||
},
|
||||
release() {
|
||||
events.push(`release:${connection}`);
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('runtime sources read normalized immutable facts without write authority', async () => {
|
||||
const taskFixture = sourcePool([taskRow()]);
|
||||
const taskSource = new PostgresTaskDefinitionSource(taskFixture.pool);
|
||||
assert.deepEqual(
|
||||
await taskSource.findCurrentTaskDefinition(TASK.projectId, TASK.taskId),
|
||||
TASK,
|
||||
);
|
||||
assert.equal('appendTaskDefinitionRevision' in taskSource, false);
|
||||
assert.deepEqual(taskFixture.queries[0].values, [TASK.projectId, TASK.taskId]);
|
||||
const listed = sourcePool([taskRow()]);
|
||||
assert.equal(
|
||||
(await new PostgresTaskDefinitionSource(listed.pool).listTaskDefinitions({
|
||||
projectId: TASK.projectId,
|
||||
limit: 1,
|
||||
})).definitions.length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
(listed.queries[0].text.match(/ORDER BY/g) ?? []).length,
|
||||
1,
|
||||
);
|
||||
|
||||
const executionSource = new PostgresTaskExecutionRevisionSource(
|
||||
sourcePool([taskExecutionRow()]).pool,
|
||||
);
|
||||
assert.deepEqual(
|
||||
await executionSource.resolveClusterTaskExecutionRevision({
|
||||
projectId: TASK.projectId,
|
||||
taskId: TASK.taskId,
|
||||
sourceRevision: TASK.revision,
|
||||
}),
|
||||
TASK_EXECUTION,
|
||||
);
|
||||
|
||||
const triggerFixture = sourcePool([triggerRow()]);
|
||||
const triggerSource = new PostgresTriggerSource(triggerFixture.pool);
|
||||
assert.deepEqual(
|
||||
await triggerSource.findCurrentTrigger(TRIGGER.projectId, TRIGGER.triggerId),
|
||||
TRIGGER,
|
||||
);
|
||||
assert.equal('appendTriggerRevision' in triggerSource, false);
|
||||
});
|
||||
|
||||
test('publishes TaskDefinition atomically and replays the exact mutation', async () => {
|
||||
const fixture = appendPool('task');
|
||||
const created = await new PostgresTaskDefinitionRepository(
|
||||
fixture.pool,
|
||||
).appendTaskDefinitionRevision(TASK_COMMAND);
|
||||
assert.deepEqual(created, { status: 'created', definition: TASK });
|
||||
assert.ok(fixture.events.includes('BEGIN ISOLATION LEVEL SERIALIZABLE'));
|
||||
assert.ok(fixture.events.includes('COMMIT'));
|
||||
assert.equal(
|
||||
fixture.events.some((event) =>
|
||||
event.includes('INSERT INTO "ql3"."task_execution_revisions"'),
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(fixture.events.at(-1), 'release:1');
|
||||
|
||||
const replayFixture = appendPool('task', { replay: {} });
|
||||
assert.deepEqual(
|
||||
await new PostgresTaskDefinitionRepository(
|
||||
replayFixture.pool,
|
||||
).appendTaskDefinitionRevision(TASK_COMMAND),
|
||||
{ status: 'existing', definition: TASK },
|
||||
);
|
||||
assert.equal(
|
||||
replayFixture.events.some((event) => event.includes('INSERT INTO')),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('runs TaskDefinition transaction hooks for create and replay before COMMIT', async () => {
|
||||
const createdFixture = appendPool('task');
|
||||
const createdHook = [];
|
||||
await new PostgresTaskDefinitionRepository(
|
||||
createdFixture.pool,
|
||||
).appendTaskDefinitionRevision(TASK_COMMAND, async (client, context) => {
|
||||
createdHook.push(context);
|
||||
await client.query('SELECT 1 AS task_hook');
|
||||
});
|
||||
assert.equal(createdHook.length, 1);
|
||||
assert.equal(createdHook[0].replay, null);
|
||||
assert.deepEqual(createdHook[0].record, TASK);
|
||||
assert.ok(
|
||||
createdFixture.events.indexOf('SELECT 1 AS task_hook') <
|
||||
createdFixture.events.indexOf('COMMIT'),
|
||||
);
|
||||
|
||||
const replayFixture = appendPool('task', { replay: {} });
|
||||
const replayHook = [];
|
||||
await new PostgresTaskDefinitionRepository(
|
||||
replayFixture.pool,
|
||||
).appendTaskDefinitionRevision(TASK_COMMAND, async (_client, context) => {
|
||||
replayHook.push(context);
|
||||
});
|
||||
assert.deepEqual(replayHook[0].replay, TASK);
|
||||
assert.deepEqual(replayHook[0].record, TASK);
|
||||
|
||||
const deniedFixture = appendPool('task');
|
||||
const denial = new Error('task authorization fence changed');
|
||||
await assert.rejects(
|
||||
new PostgresTaskDefinitionRepository(
|
||||
deniedFixture.pool,
|
||||
).appendTaskDefinitionRevision(TASK_COMMAND, async () => {
|
||||
throw denial;
|
||||
}),
|
||||
(error) => error === denial,
|
||||
);
|
||||
assert.equal(deniedFixture.events.includes('ROLLBACK'), true);
|
||||
assert.equal(deniedFixture.events.includes('COMMIT'), false);
|
||||
});
|
||||
|
||||
test('retries serialization and fails closed on TaskDefinition drift or corruption', async () => {
|
||||
const retry = appendPool('task', { retryOnce: true });
|
||||
assert.equal(
|
||||
(
|
||||
await new PostgresTaskDefinitionRepository(
|
||||
retry.pool,
|
||||
).appendTaskDefinitionRevision(TASK_COMMAND)
|
||||
).status,
|
||||
'created',
|
||||
);
|
||||
assert.equal(retry.events.filter((event) => event.startsWith('release:')).length, 2);
|
||||
|
||||
await assert.rejects(
|
||||
new PostgresTaskDefinitionRepository(
|
||||
appendPool('task', {
|
||||
replay: {
|
||||
name: DRIFT_TASK.name,
|
||||
contentDigest: DRIFT_TASK.contentDigest,
|
||||
},
|
||||
}).pool,
|
||||
).appendTaskDefinitionRevision(TASK_COMMAND),
|
||||
TaskDefinitionConflictError,
|
||||
);
|
||||
await assert.rejects(
|
||||
new PostgresTaskDefinitionRepository(
|
||||
appendPool('task', { replay: {}, missingExecution: true }).pool,
|
||||
).appendTaskDefinitionRevision(TASK_COMMAND),
|
||||
TaskDefinitionUnavailableError,
|
||||
);
|
||||
await assert.rejects(
|
||||
new PostgresTaskDefinitionSource(sourcePool([
|
||||
taskRow({ contentDigest: 'b'.repeat(64) }),
|
||||
]).pool).findCurrentTaskDefinition(TASK.projectId, TASK.taskId),
|
||||
TaskDefinitionUnavailableError,
|
||||
);
|
||||
});
|
||||
|
||||
test('publishes Trigger only when its immutable task pin is valid', async () => {
|
||||
const fixture = appendPool('trigger');
|
||||
const created = await new PostgresTriggerRepository(
|
||||
fixture.pool,
|
||||
).appendTriggerRevision(TRIGGER_COMMAND);
|
||||
assert.deepEqual(created, { status: 'created', trigger: TRIGGER });
|
||||
assert.ok(
|
||||
fixture.events.findIndex((event) => event.includes('task_definitions')) <
|
||||
fixture.events.findIndex((event) => event.includes('trigger_revisions')),
|
||||
);
|
||||
assert.equal(
|
||||
fixture.events.some((event) =>
|
||||
event.includes('INSERT INTO "ql3"."trigger_schedules"'),
|
||||
),
|
||||
true,
|
||||
);
|
||||
const createdPin = fixture.queries.find(({ text }) =>
|
||||
text.includes('$4::boolean = false'),
|
||||
);
|
||||
assert.equal(createdPin.values[3], true);
|
||||
|
||||
const replayFixture = appendPool('trigger', { replay: {} });
|
||||
assert.deepEqual(
|
||||
await new PostgresTriggerRepository(
|
||||
replayFixture.pool,
|
||||
).appendTriggerRevision(TRIGGER_COMMAND),
|
||||
{ status: 'existing', trigger: TRIGGER },
|
||||
);
|
||||
const replayPin = replayFixture.queries.find(({ text }) =>
|
||||
text.includes('$4::boolean = false'),
|
||||
);
|
||||
assert.equal(replayPin.values[3], false);
|
||||
|
||||
await assert.rejects(
|
||||
new PostgresTriggerRepository(
|
||||
appendPool('trigger', { missingTask: true }).pool,
|
||||
).appendTriggerRevision(TRIGGER_COMMAND),
|
||||
TriggerConflictError,
|
||||
);
|
||||
await assert.rejects(
|
||||
new PostgresTriggerRepository(
|
||||
appendPool('trigger', { task: { contentDigest: 'b'.repeat(64) } }).pool,
|
||||
).appendTriggerRevision(TRIGGER_COMMAND),
|
||||
TriggerUnavailableError,
|
||||
);
|
||||
});
|
||||
|
||||
test('runs Trigger transaction hooks atomically and preserves hook failures', async () => {
|
||||
const createdFixture = appendPool('trigger');
|
||||
const contexts = [];
|
||||
await new PostgresTriggerRepository(
|
||||
createdFixture.pool,
|
||||
).appendTriggerRevision(TRIGGER_COMMAND, async (client, context) => {
|
||||
contexts.push(context);
|
||||
await client.query('SELECT 1 AS trigger_hook');
|
||||
});
|
||||
assert.equal(contexts.length, 1);
|
||||
assert.equal(contexts[0].replay, null);
|
||||
assert.deepEqual(contexts[0].record, TRIGGER);
|
||||
assert.ok(
|
||||
createdFixture.events.indexOf('SELECT 1 AS trigger_hook') <
|
||||
createdFixture.events.indexOf('COMMIT'),
|
||||
);
|
||||
|
||||
const replayFixture = appendPool('trigger', { replay: {} });
|
||||
await new PostgresTriggerRepository(
|
||||
replayFixture.pool,
|
||||
).appendTriggerRevision(TRIGGER_COMMAND, async (_client, context) => {
|
||||
contexts.push(context);
|
||||
});
|
||||
assert.deepEqual(contexts[1].replay, TRIGGER);
|
||||
|
||||
const deniedFixture = appendPool('trigger');
|
||||
const denial = new Error('trigger authorization fence changed');
|
||||
await assert.rejects(
|
||||
new PostgresTriggerRepository(
|
||||
deniedFixture.pool,
|
||||
).appendTriggerRevision(TRIGGER_COMMAND, async () => {
|
||||
throw denial;
|
||||
}),
|
||||
(error) => error === denial,
|
||||
);
|
||||
assert.equal(deniedFixture.events.includes('ROLLBACK'), true);
|
||||
assert.equal(deniedFixture.events.includes('COMMIT'), false);
|
||||
});
|
||||
|
||||
test('Trigger replay rejects mutation drift and corrupt durable records', async () => {
|
||||
await assert.rejects(
|
||||
new PostgresTriggerRepository(
|
||||
appendPool('trigger', {
|
||||
replay: {
|
||||
enabled: DRIFT_TRIGGER.enabled,
|
||||
contentDigest: DRIFT_TRIGGER.contentDigest,
|
||||
},
|
||||
}).pool,
|
||||
).appendTriggerRevision(TRIGGER_COMMAND),
|
||||
TriggerConflictError,
|
||||
);
|
||||
await assert.rejects(
|
||||
new PostgresTriggerRepository(
|
||||
appendPool('trigger', {
|
||||
replay: {},
|
||||
missingSchedule: true,
|
||||
}).pool,
|
||||
).appendTriggerRevision(TRIGGER_COMMAND),
|
||||
TriggerUnavailableError,
|
||||
);
|
||||
await assert.rejects(
|
||||
new PostgresTriggerSource(sourcePool([
|
||||
triggerRow({ contentDigest: 'b'.repeat(64) }),
|
||||
]).pool).findCurrentTrigger(TRIGGER.projectId, TRIGGER.triggerId),
|
||||
TriggerUnavailableError,
|
||||
);
|
||||
});
|
||||
|
||||
test('atomically fences and audits authorized PostgreSQL Task mutations', async () => {
|
||||
const createdFixture = appendPool('task');
|
||||
assert.deepEqual(
|
||||
await new PostgresTaskDefinitionAdministrationRepository(
|
||||
createdFixture.pool,
|
||||
).appendAuthorizedTaskDefinitionRevision({
|
||||
command: TASK_COMMAND,
|
||||
actor: ACTOR,
|
||||
fence: FENCE,
|
||||
audit: TASK_AUDIT,
|
||||
}),
|
||||
{ status: 'created', definition: TASK },
|
||||
);
|
||||
const createdSql = createdFixture.queries.map(({ text }) => text);
|
||||
assert.ok(
|
||||
createdSql.findIndex((text) => text.includes('project_role_bindings')) <
|
||||
createdSql.findIndex((text) => text.includes('security_audit_events')),
|
||||
);
|
||||
assert.ok(
|
||||
createdSql.findIndex((text) => text.includes('security_audit_events')) <
|
||||
createdSql.indexOf('COMMIT'),
|
||||
);
|
||||
|
||||
const replayFixture = appendPool('task', {
|
||||
replay: {},
|
||||
audit: TASK_AUDIT,
|
||||
});
|
||||
assert.equal(
|
||||
(
|
||||
await new PostgresTaskDefinitionAdministrationRepository(
|
||||
replayFixture.pool,
|
||||
).appendAuthorizedTaskDefinitionRevision({
|
||||
command: TASK_COMMAND,
|
||||
actor: ACTOR,
|
||||
fence: FENCE,
|
||||
audit: { ...TASK_AUDIT, occurredAtMs: 999 },
|
||||
})
|
||||
).status,
|
||||
'existing',
|
||||
);
|
||||
assert.equal(
|
||||
replayFixture.queries.some(({ text }) =>
|
||||
text.includes('INSERT INTO "ql3"."security_audit_events"'),
|
||||
),
|
||||
false,
|
||||
);
|
||||
|
||||
const fencedFixture = appendPool('task', { bindingVersion: 2 });
|
||||
await assert.rejects(
|
||||
new PostgresTaskDefinitionAdministrationRepository(
|
||||
fencedFixture.pool,
|
||||
).appendAuthorizedTaskDefinitionRevision({
|
||||
command: TASK_COMMAND,
|
||||
actor: ACTOR,
|
||||
fence: FENCE,
|
||||
audit: TASK_AUDIT,
|
||||
}),
|
||||
TaskDefinitionAdministrationAuthorizationFenceConflictError,
|
||||
);
|
||||
assert.equal(fencedFixture.events.includes('ROLLBACK'), true);
|
||||
|
||||
await assert.rejects(
|
||||
new PostgresTaskDefinitionAdministrationRepository(
|
||||
appendPool('task', { audit: TASK_AUDIT }).pool,
|
||||
).appendAuthorizedTaskDefinitionRevision({
|
||||
command: TASK_COMMAND,
|
||||
actor: ACTOR,
|
||||
fence: FENCE,
|
||||
audit: TASK_AUDIT,
|
||||
}),
|
||||
TaskDefinitionAdministrationMutationConflictError,
|
||||
);
|
||||
});
|
||||
|
||||
test('atomically fences and audits authorized PostgreSQL Trigger mutations', async () => {
|
||||
const createdFixture = appendPool('trigger');
|
||||
assert.equal(
|
||||
(
|
||||
await new PostgresTriggerAdministrationRepository(
|
||||
createdFixture.pool,
|
||||
).appendAuthorizedTriggerRevision({
|
||||
command: TRIGGER_COMMAND,
|
||||
actor: ACTOR,
|
||||
fence: FENCE,
|
||||
audit: TRIGGER_AUDIT,
|
||||
})
|
||||
).status,
|
||||
'created',
|
||||
);
|
||||
assert.equal(
|
||||
createdFixture.queries.some(({ text }) =>
|
||||
text.includes('INSERT INTO "ql3"."security_audit_events"'),
|
||||
),
|
||||
true,
|
||||
);
|
||||
|
||||
const fencedFixture = appendPool('trigger', { projectVersion: 2 });
|
||||
await assert.rejects(
|
||||
new PostgresTriggerAdministrationRepository(
|
||||
fencedFixture.pool,
|
||||
).appendAuthorizedTriggerRevision({
|
||||
command: TRIGGER_COMMAND,
|
||||
actor: ACTOR,
|
||||
fence: FENCE,
|
||||
audit: TRIGGER_AUDIT,
|
||||
}),
|
||||
TriggerAdministrationAuthorizationFenceConflictError,
|
||||
);
|
||||
assert.equal(fencedFixture.events.includes('ROLLBACK'), true);
|
||||
});
|
||||
|
||||
test('atomically fences, reads and audits current PostgreSQL automation facts', async () => {
|
||||
const taskFixture = appendPool('task');
|
||||
const taskRepository = new PostgresTaskDefinitionAdministrationRepository(
|
||||
taskFixture.pool,
|
||||
);
|
||||
assert.deepEqual(
|
||||
await taskRepository.findAuthorizedCurrentTaskDefinition({
|
||||
projectId: TASK.projectId,
|
||||
taskId: TASK.taskId,
|
||||
actor: ACTOR,
|
||||
fence: FENCE,
|
||||
audit: TASK_READ_AUDIT,
|
||||
}),
|
||||
TASK,
|
||||
);
|
||||
const taskSql = taskFixture.queries.map(({ text }) => text);
|
||||
assert.ok(
|
||||
taskSql.findIndex((text) => text.includes('FROM "ql3"."projects"')) <
|
||||
taskSql.findIndex((text) =>
|
||||
text.includes('JOIN "ql3"."task_definition_revisions"'),
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
taskSql.findIndex((text) =>
|
||||
text.includes('JOIN "ql3"."task_definition_revisions"'),
|
||||
) <
|
||||
taskSql.findIndex((text) =>
|
||||
text.includes('INSERT INTO "ql3"."security_audit_events"'),
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
taskSql.findIndex((text) =>
|
||||
text.includes('INSERT INTO "ql3"."security_audit_events"'),
|
||||
) < taskSql.indexOf('COMMIT'),
|
||||
);
|
||||
|
||||
const triggerFixture = appendPool('trigger');
|
||||
assert.deepEqual(
|
||||
await new PostgresTriggerAdministrationRepository(
|
||||
triggerFixture.pool,
|
||||
).findAuthorizedCurrentTrigger({
|
||||
projectId: TRIGGER.projectId,
|
||||
triggerId: TRIGGER.triggerId,
|
||||
actor: ACTOR,
|
||||
fence: FENCE,
|
||||
audit: TRIGGER_READ_AUDIT,
|
||||
}),
|
||||
TRIGGER,
|
||||
);
|
||||
assert.equal(
|
||||
triggerFixture.queries.some(({ text }) =>
|
||||
text.includes('INSERT INTO "ql3"."security_audit_events"'),
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('reads bounded automation pages and fails closed on fence or audit replay', async () => {
|
||||
const taskFixture = appendPool('task');
|
||||
const tasks = await new PostgresTaskDefinitionAdministrationRepository(
|
||||
taskFixture.pool,
|
||||
).listAuthorizedTaskDefinitions({
|
||||
projectId: TASK.projectId,
|
||||
limit: 1,
|
||||
actor: ACTOR,
|
||||
fence: FENCE,
|
||||
audit: TASK_READ_AUDIT,
|
||||
});
|
||||
assert.deepEqual(tasks, { definitions: [TASK], truncated: false });
|
||||
|
||||
const triggerFixture = appendPool('trigger');
|
||||
const triggers = await new PostgresTriggerAdministrationRepository(
|
||||
triggerFixture.pool,
|
||||
).listAuthorizedTriggers({
|
||||
projectId: TRIGGER.projectId,
|
||||
limit: 1,
|
||||
actor: ACTOR,
|
||||
fence: FENCE,
|
||||
audit: TRIGGER_READ_AUDIT,
|
||||
});
|
||||
assert.deepEqual(triggers, { triggers: [TRIGGER], truncated: false });
|
||||
|
||||
const fenced = appendPool('task', { bindingVersion: 2 });
|
||||
await assert.rejects(
|
||||
new PostgresTaskDefinitionAdministrationRepository(
|
||||
fenced.pool,
|
||||
).findAuthorizedCurrentTaskDefinition({
|
||||
projectId: TASK.projectId,
|
||||
taskId: TASK.taskId,
|
||||
actor: ACTOR,
|
||||
fence: FENCE,
|
||||
audit: TASK_READ_AUDIT,
|
||||
}),
|
||||
TaskDefinitionAdministrationAuthorizationFenceConflictError,
|
||||
);
|
||||
assert.equal(
|
||||
fenced.queries.some(({ text }) =>
|
||||
text.includes('JOIN "ql3"."task_definition_revisions"'),
|
||||
),
|
||||
false,
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
new PostgresTaskDefinitionAdministrationRepository(
|
||||
appendPool('task', { audit: TASK_READ_AUDIT }).pool,
|
||||
).findAuthorizedCurrentTaskDefinition({
|
||||
projectId: TASK.projectId,
|
||||
taskId: TASK.taskId,
|
||||
actor: ACTOR,
|
||||
fence: FENCE,
|
||||
audit: TASK_READ_AUDIT,
|
||||
}),
|
||||
TaskDefinitionAdministrationReadConflictError,
|
||||
);
|
||||
await assert.rejects(
|
||||
new PostgresTriggerAdministrationRepository(
|
||||
appendPool('trigger', { audit: TRIGGER_READ_AUDIT }).pool,
|
||||
).findAuthorizedCurrentTrigger({
|
||||
projectId: TRIGGER.projectId,
|
||||
triggerId: TRIGGER.triggerId,
|
||||
actor: ACTOR,
|
||||
fence: FENCE,
|
||||
audit: TRIGGER_READ_AUDIT,
|
||||
}),
|
||||
TriggerAdministrationReadConflictError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,424 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const path = require('node:path');
|
||||
const { spawnSync } = require('node:child_process');
|
||||
const { test } = require('node:test');
|
||||
|
||||
test('runtime export excludes executable migration DDL modules', () => {
|
||||
const packageDirectory = path.resolve(__dirname, '..');
|
||||
const script = `
|
||||
const runtime = require('@qinglong/cluster-postgres/runtime');
|
||||
const loaded = Object.keys(require.cache)
|
||||
.filter((file) => file.includes('/ql3-cluster-postgres/dist/'))
|
||||
.map((file) => file.replaceAll('\\\\', '/'));
|
||||
process.stdout.write(JSON.stringify({
|
||||
hasRepository: typeof runtime.PostgresRunRepository === 'function',
|
||||
hasSecretAuthority: typeof runtime.PostgresRemoteWorkerSecretDeliveryAuthorityRepository === 'function',
|
||||
hasReadiness: typeof runtime.assertPostgresSchemaReady === 'function',
|
||||
loaded,
|
||||
}));
|
||||
`;
|
||||
const result = spawnSync(process.execPath, ['-e', script], {
|
||||
cwd: packageDirectory,
|
||||
encoding: 'utf8',
|
||||
});
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const report = JSON.parse(result.stdout);
|
||||
assert.equal(report.hasRepository, true);
|
||||
assert.equal(report.hasSecretAuthority, true);
|
||||
assert.equal(report.hasReadiness, true);
|
||||
assert.equal(
|
||||
report.loaded.some(
|
||||
(file) =>
|
||||
/\/dist\/migrations\/pg-\d/.test(file) ||
|
||||
file.endsWith('/dist/migration/migrate.js') ||
|
||||
file.endsWith('/dist/migration/migration.js') ||
|
||||
file.endsWith('/dist/schema/schema.js'),
|
||||
),
|
||||
false,
|
||||
report.loaded.join('\n'),
|
||||
);
|
||||
});
|
||||
|
||||
test('Worker ingress export cannot acquire runtime Secret authority', () => {
|
||||
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
|
||||
assert.equal(
|
||||
ingress.PostgresRemoteWorkerSecretDeliveryAuthorityRepository,
|
||||
undefined,
|
||||
);
|
||||
assert.equal(
|
||||
ingress.PostgresRemoteWorkerCompletionRepository,
|
||||
undefined,
|
||||
);
|
||||
assert.equal(
|
||||
ingress.PostgresRemoteWorkerLeaseControlRepository,
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
test('migration export exposes the reviewed runner through a public subpath', () => {
|
||||
const migration = require('@qinglong/cluster-postgres/migration');
|
||||
assert.equal(typeof migration.runPostgresMigrations, 'function');
|
||||
assert.deepEqual(
|
||||
migration.postgresqlMainMigrationManifest.migrations,
|
||||
migration.postgresqlMainMigrationStream.migrations.map(
|
||||
({ id, checksum }) => ({ id, checksum }),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('admin export exposes administration authority without migration DDL', () => {
|
||||
const packageDirectory = path.resolve(__dirname, '..');
|
||||
const script = `
|
||||
const admin = require('@qinglong/cluster-postgres/admin');
|
||||
const loaded = Object.keys(require.cache)
|
||||
.filter((file) => file.includes('/ql3-cluster-postgres/dist/'))
|
||||
.map((file) => file.replaceAll('\\\\', '/'));
|
||||
process.stdout.write(JSON.stringify({
|
||||
hasIdentityAdministration: typeof admin.PostgresIdentityAdministrationRepository === 'function',
|
||||
hasCredentialAdministration: typeof admin.PostgresApiCredentialAdministrationRepository === 'function',
|
||||
hasAuditQuery: typeof admin.PostgresSecurityAuditQueryRepository === 'function',
|
||||
hasReadiness: typeof admin.assertPostgresAdminSchemaReady === 'function',
|
||||
loaded,
|
||||
}));
|
||||
`;
|
||||
const result = spawnSync(process.execPath, ['-e', script], {
|
||||
cwd: packageDirectory,
|
||||
encoding: 'utf8',
|
||||
});
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const report = JSON.parse(result.stdout);
|
||||
assert.equal(report.hasIdentityAdministration, true);
|
||||
assert.equal(report.hasCredentialAdministration, true);
|
||||
assert.equal(report.hasAuditQuery, true);
|
||||
assert.equal(report.hasReadiness, true);
|
||||
assert.equal(
|
||||
report.loaded.some(
|
||||
(file) =>
|
||||
/\/dist\/migrations\/pg-\d/.test(file) ||
|
||||
file.endsWith('/dist/migration/migrate.js') ||
|
||||
file.endsWith('/dist/migration/migration.js') ||
|
||||
file.endsWith('/dist/schema/schema.js'),
|
||||
),
|
||||
false,
|
||||
report.loaded.join('\n'),
|
||||
);
|
||||
});
|
||||
|
||||
test('Plugin Package install authority is isolated behind its explicit subpath', () => {
|
||||
const root = require('@qinglong/cluster-postgres');
|
||||
const runtime = require('@qinglong/cluster-postgres/runtime');
|
||||
const admin = require('@qinglong/cluster-postgres/admin');
|
||||
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
|
||||
const authority = require('@qinglong/cluster-postgres/plugin-package-install');
|
||||
assert.equal(root.PostgresPluginPackageInstallRepository, undefined);
|
||||
assert.equal(runtime.PostgresPluginPackageInstallRepository, undefined);
|
||||
assert.equal(admin.PostgresPluginPackageInstallRepository, undefined);
|
||||
assert.equal(ingress.PostgresPluginPackageInstallRepository, undefined);
|
||||
assert.equal(
|
||||
typeof authority.PostgresPluginPackageInstallRepository,
|
||||
'function',
|
||||
);
|
||||
});
|
||||
|
||||
test('Package manager exposes inventory read authority without install mutation authority', () => {
|
||||
const manager = require('@qinglong/cluster-postgres/package-manager');
|
||||
assert.equal(
|
||||
typeof manager.PostgresPluginPackageInstallInventoryReader,
|
||||
'function',
|
||||
);
|
||||
assert.equal(manager.PostgresPluginPackageInstallRepository, undefined);
|
||||
const reader = new manager.PostgresPluginPackageInstallInventoryReader({
|
||||
async query() {
|
||||
return { rows: [], rowCount: 0 };
|
||||
},
|
||||
});
|
||||
assert.deepEqual(
|
||||
Object.getOwnPropertyNames(Object.getPrototypeOf(reader)).sort(),
|
||||
['constructor', 'findCurrent', 'listCurrentPage'],
|
||||
);
|
||||
});
|
||||
|
||||
test('Worker credential authorities expose disjoint management and execution capabilities', () => {
|
||||
const manager = require('@qinglong/cluster-postgres/worker-credential-manager');
|
||||
const executor = require('@qinglong/cluster-postgres/worker-credential-executor');
|
||||
|
||||
assert.equal(
|
||||
typeof manager.assertPostgresWorkerCredentialManagerSchemaReady,
|
||||
'function',
|
||||
);
|
||||
assert.equal(
|
||||
typeof manager.PostgresWorkerCredentialManagementPlanRepository,
|
||||
'function',
|
||||
);
|
||||
assert.equal(manager.PostgresWorkerCredentialAdministrationRepository, undefined);
|
||||
assert.equal(
|
||||
manager.PostgresRemoteWorkerSecretDeliveryAuthorityRepository,
|
||||
undefined,
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
typeof executor.assertPostgresWorkerCredentialExecutorSchemaReady,
|
||||
'function',
|
||||
);
|
||||
assert.equal(
|
||||
typeof executor.PostgresWorkerCredentialManagementPlanReader,
|
||||
'function',
|
||||
);
|
||||
assert.equal(
|
||||
typeof executor.PostgresWorkerCredentialAdministrationRepository,
|
||||
'function',
|
||||
);
|
||||
assert.equal(
|
||||
typeof executor.PostgresRemoteWorkerSecretDeliveryAuthorityRepository,
|
||||
'function',
|
||||
);
|
||||
assert.equal(executor.PostgresWorkerCredentialManagementPlanRepository, undefined);
|
||||
});
|
||||
|
||||
test('Package manager inventory reader performs only bounded read queries', async () => {
|
||||
const { PostgresPluginPackageInstallInventoryReader } = require(
|
||||
'@qinglong/cluster-postgres/package-manager'
|
||||
);
|
||||
const queries = [];
|
||||
const reader = new PostgresPluginPackageInstallInventoryReader({
|
||||
async query(text, values) {
|
||||
queries.push({ text, values });
|
||||
return { rows: [], rowCount: 0 };
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(await reader.findCurrent('project-reader', 'package-reader'), null);
|
||||
assert.deepEqual(
|
||||
await reader.listCurrentPage({
|
||||
projectId: 'project-reader',
|
||||
limit: 8,
|
||||
after: { packageName: 'package-before' },
|
||||
}),
|
||||
{ items: [], truncated: false },
|
||||
);
|
||||
assert.equal(queries.length, 2);
|
||||
assert.match(queries[0].text, /SELECT/);
|
||||
assert.deepEqual(queries[0].values, ['project-reader', 'package-reader']);
|
||||
assert.match(queries[1].text, /ORDER BY head\.package_name/);
|
||||
assert.deepEqual(queries[1].values, [
|
||||
'project-reader',
|
||||
'package-before',
|
||||
9,
|
||||
]);
|
||||
});
|
||||
|
||||
test('StepRun authority is limited to runtime composition and explicit subpath', () => {
|
||||
const root = require('@qinglong/cluster-postgres');
|
||||
const runtime = require('@qinglong/cluster-postgres/runtime');
|
||||
const admin = require('@qinglong/cluster-postgres/admin');
|
||||
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
|
||||
const authority = require('@qinglong/cluster-postgres/step-run');
|
||||
assert.equal(root.PostgresStepRunRepository, undefined);
|
||||
assert.equal(
|
||||
runtime.PostgresStepRunRepository,
|
||||
authority.PostgresStepRunRepository,
|
||||
);
|
||||
assert.equal(admin.PostgresStepRunRepository, undefined);
|
||||
assert.equal(ingress.PostgresStepRunRepository, undefined);
|
||||
assert.equal(typeof authority.PostgresStepRunRepository, 'function');
|
||||
});
|
||||
|
||||
test('Tool execution evidence authority is isolated behind its explicit subpath', () => {
|
||||
const root = require('@qinglong/cluster-postgres');
|
||||
const runtime = require('@qinglong/cluster-postgres/runtime');
|
||||
const admin = require('@qinglong/cluster-postgres/admin');
|
||||
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
|
||||
const authority = require('@qinglong/cluster-postgres/tool-execution-evidence');
|
||||
assert.equal(root.PostgresToolExecutionEvidenceRepository, undefined);
|
||||
assert.equal(runtime.PostgresToolExecutionEvidenceRepository, undefined);
|
||||
assert.equal(admin.PostgresToolExecutionEvidenceRepository, undefined);
|
||||
assert.equal(ingress.PostgresToolExecutionEvidenceRepository, undefined);
|
||||
assert.equal(
|
||||
typeof authority.PostgresToolExecutionEvidenceRepository,
|
||||
'function',
|
||||
);
|
||||
});
|
||||
|
||||
test('Tool start barrier authority is limited to runtime composition and explicit subpath', () => {
|
||||
const root = require('@qinglong/cluster-postgres');
|
||||
const runtime = require('@qinglong/cluster-postgres/runtime');
|
||||
const admin = require('@qinglong/cluster-postgres/admin');
|
||||
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
|
||||
const authority = require('@qinglong/cluster-postgres/tool-execution-start-barrier');
|
||||
assert.equal(root.PostgresToolExecutionStartBarrierRepository, undefined);
|
||||
assert.equal(
|
||||
runtime.PostgresToolExecutionStartBarrierRepository,
|
||||
authority.PostgresToolExecutionStartBarrierRepository,
|
||||
);
|
||||
assert.equal(admin.PostgresToolExecutionStartBarrierRepository, undefined);
|
||||
assert.equal(ingress.PostgresToolExecutionStartBarrierRepository, undefined);
|
||||
assert.equal(
|
||||
typeof authority.PostgresToolExecutionStartBarrierRepository,
|
||||
'function',
|
||||
);
|
||||
});
|
||||
|
||||
test('Tool completion authority is limited to runtime composition and explicit subpath', () => {
|
||||
const root = require('@qinglong/cluster-postgres');
|
||||
const runtime = require('@qinglong/cluster-postgres/runtime');
|
||||
const admin = require('@qinglong/cluster-postgres/admin');
|
||||
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
|
||||
const authority = require('@qinglong/cluster-postgres/tool-execution-completion');
|
||||
assert.equal(root.PostgresToolExecutionCompletionRepository, undefined);
|
||||
assert.equal(
|
||||
runtime.PostgresToolExecutionCompletionRepository,
|
||||
authority.PostgresToolExecutionCompletionRepository,
|
||||
);
|
||||
assert.equal(admin.PostgresToolExecutionCompletionRepository, undefined);
|
||||
assert.equal(ingress.PostgresToolExecutionCompletionRepository, undefined);
|
||||
assert.equal(
|
||||
typeof authority.PostgresToolExecutionCompletionRepository,
|
||||
'function',
|
||||
);
|
||||
});
|
||||
|
||||
test('Tool failure completion authority is limited to runtime composition and explicit subpath', () => {
|
||||
const root = require('@qinglong/cluster-postgres');
|
||||
const runtime = require('@qinglong/cluster-postgres/runtime');
|
||||
const admin = require('@qinglong/cluster-postgres/admin');
|
||||
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
|
||||
const authority = require('@qinglong/cluster-postgres/tool-execution-failure-completion');
|
||||
assert.equal(
|
||||
root.PostgresToolExecutionFailureCompletionRepository,
|
||||
undefined,
|
||||
);
|
||||
assert.equal(
|
||||
runtime.PostgresToolExecutionFailureCompletionRepository,
|
||||
authority.PostgresToolExecutionFailureCompletionRepository,
|
||||
);
|
||||
assert.equal(
|
||||
admin.PostgresToolExecutionFailureCompletionRepository,
|
||||
undefined,
|
||||
);
|
||||
assert.equal(
|
||||
ingress.PostgresToolExecutionFailureCompletionRepository,
|
||||
undefined,
|
||||
);
|
||||
assert.equal(
|
||||
typeof authority.PostgresToolExecutionFailureCompletionRepository,
|
||||
'function',
|
||||
);
|
||||
});
|
||||
|
||||
test('Tool result key catalog splits runtime read from admin mutation authority', () => {
|
||||
const root = require('@qinglong/cluster-postgres');
|
||||
const runtime = require('@qinglong/cluster-postgres/runtime');
|
||||
const admin = require('@qinglong/cluster-postgres/admin');
|
||||
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
|
||||
const authority = require('@qinglong/cluster-postgres/tool-result-key-catalog');
|
||||
assert.equal(root.PostgresToolResultKeyCatalogReader, undefined);
|
||||
assert.equal(root.PostgresToolResultKeyCatalogRepository, undefined);
|
||||
assert.equal(
|
||||
runtime.PostgresToolResultKeyCatalogReader,
|
||||
authority.PostgresToolResultKeyCatalogReader,
|
||||
);
|
||||
assert.equal(runtime.PostgresToolResultKeyCatalogRepository, undefined);
|
||||
assert.equal(
|
||||
admin.PostgresToolResultKeyCatalogRepository,
|
||||
authority.PostgresToolResultKeyCatalogRepository,
|
||||
);
|
||||
assert.equal(admin.PostgresToolResultKeyCatalogReader, undefined);
|
||||
assert.equal(ingress.PostgresToolResultKeyCatalogReader, undefined);
|
||||
assert.equal(ingress.PostgresToolResultKeyCatalogRepository, undefined);
|
||||
});
|
||||
|
||||
test('Tool result rekey splits runtime read from admin mutation authority', () => {
|
||||
const root = require('@qinglong/cluster-postgres');
|
||||
const runtime = require('@qinglong/cluster-postgres/runtime');
|
||||
const admin = require('@qinglong/cluster-postgres/admin');
|
||||
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
|
||||
const authority = require('@qinglong/cluster-postgres/tool-result-rekey');
|
||||
assert.equal(root.PostgresToolResultRekeyReader, undefined);
|
||||
assert.equal(root.PostgresToolResultRekeyRepository, undefined);
|
||||
assert.equal(
|
||||
runtime.PostgresToolResultRekeyReader,
|
||||
authority.PostgresToolResultRekeyReader,
|
||||
);
|
||||
assert.equal(runtime.PostgresToolResultRekeyRepository, undefined);
|
||||
assert.equal(
|
||||
admin.PostgresToolResultRekeyRepository,
|
||||
authority.PostgresToolResultRekeyRepository,
|
||||
);
|
||||
assert.equal(admin.PostgresToolResultRekeyReader, undefined);
|
||||
assert.equal(ingress.PostgresToolResultRekeyReader, undefined);
|
||||
assert.equal(ingress.PostgresToolResultRekeyRepository, undefined);
|
||||
});
|
||||
|
||||
test('Approved Action authority is isolated behind its explicit subpath', () => {
|
||||
const root = require('@qinglong/cluster-postgres');
|
||||
const runtime = require('@qinglong/cluster-postgres/runtime');
|
||||
const admin = require('@qinglong/cluster-postgres/admin');
|
||||
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
|
||||
const authority = require('@qinglong/cluster-postgres/approved-action');
|
||||
assert.equal(root.PostgresApprovalRequestRepository, undefined);
|
||||
assert.equal(runtime.PostgresApprovalRequestRepository, undefined);
|
||||
assert.equal(admin.PostgresApprovalRequestRepository, undefined);
|
||||
assert.equal(ingress.PostgresApprovalRequestRepository, undefined);
|
||||
assert.equal(typeof authority.PostgresApprovalRequestRepository, 'function');
|
||||
});
|
||||
|
||||
test('Approved Action execution authority is isolated behind its explicit subpath', () => {
|
||||
const root = require('@qinglong/cluster-postgres');
|
||||
const runtime = require('@qinglong/cluster-postgres/runtime');
|
||||
const admin = require('@qinglong/cluster-postgres/admin');
|
||||
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
|
||||
const authority = require('@qinglong/cluster-postgres/approved-action-execution');
|
||||
assert.equal(root.PostgresApprovedActionExecutionRepository, undefined);
|
||||
assert.equal(runtime.PostgresApprovedActionExecutionRepository, undefined);
|
||||
assert.equal(admin.PostgresApprovedActionExecutionRepository, undefined);
|
||||
assert.equal(ingress.PostgresApprovedActionExecutionRepository, undefined);
|
||||
assert.equal(
|
||||
typeof authority.PostgresApprovedActionExecutionRepository,
|
||||
'function',
|
||||
);
|
||||
});
|
||||
|
||||
test('Plugin Package proposal authority is isolated behind its explicit subpath', () => {
|
||||
const root = require('@qinglong/cluster-postgres');
|
||||
const runtime = require('@qinglong/cluster-postgres/runtime');
|
||||
const admin = require('@qinglong/cluster-postgres/admin');
|
||||
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
|
||||
const authority = require('@qinglong/cluster-postgres/plugin-package-proposal');
|
||||
assert.equal(root.PostgresPluginPackageInstallProposalRepository, undefined);
|
||||
assert.equal(
|
||||
runtime.PostgresPluginPackageInstallProposalRepository,
|
||||
undefined,
|
||||
);
|
||||
assert.equal(admin.PostgresPluginPackageInstallProposalRepository, undefined);
|
||||
assert.equal(
|
||||
ingress.PostgresPluginPackageInstallProposalRepository,
|
||||
undefined,
|
||||
);
|
||||
assert.equal(
|
||||
typeof authority.PostgresPluginPackageInstallProposalRepository,
|
||||
'function',
|
||||
);
|
||||
});
|
||||
|
||||
test('Plugin Package lifecycle authority is limited to package executor and its explicit subpath', () => {
|
||||
const root = require('@qinglong/cluster-postgres');
|
||||
const runtime = require('@qinglong/cluster-postgres/runtime');
|
||||
const admin = require('@qinglong/cluster-postgres/admin');
|
||||
const manager = require('@qinglong/cluster-postgres/package-manager');
|
||||
const executor = require('@qinglong/cluster-postgres/package-executor');
|
||||
const ingress = require('@qinglong/cluster-postgres/worker-ingress');
|
||||
const authority = require('@qinglong/cluster-postgres/plugin-package-lifecycle');
|
||||
assert.equal(root.PostgresPluginPackageLifecycleRepository, undefined);
|
||||
assert.equal(runtime.PostgresPluginPackageLifecycleRepository, undefined);
|
||||
assert.equal(admin.PostgresPluginPackageLifecycleRepository, undefined);
|
||||
assert.equal(manager.PostgresPluginPackageLifecycleRepository, undefined);
|
||||
assert.equal(ingress.PostgresPluginPackageLifecycleRepository, undefined);
|
||||
assert.equal(
|
||||
executor.PostgresPluginPackageLifecycleRepository,
|
||||
authority.PostgresPluginPackageLifecycleRepository,
|
||||
);
|
||||
assert.equal(
|
||||
typeof authority.PostgresPluginPackageLifecycleRepository,
|
||||
'function',
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,176 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
loadPostgresCertificateAuthorityFile,
|
||||
} = require('@qinglong/cluster-postgres/runtime');
|
||||
const {
|
||||
PostgresMigrationProcessConfigError,
|
||||
loadPostgresMigrationProcessConfig,
|
||||
runPostgresMigrationProcess,
|
||||
} = require('@qinglong/cluster-postgres/migration-process');
|
||||
|
||||
const CA_FILE = path.resolve(
|
||||
__dirname,
|
||||
'../../ql3-cluster-control/test/fixtures/mtls/ca-cert.pem',
|
||||
);
|
||||
const CA_BUNDLE = loadPostgresCertificateAuthorityFile(CA_FILE);
|
||||
|
||||
const BASE_ENV = Object.freeze({
|
||||
QL3_POSTGRES_MIGRATION_URL:
|
||||
'postgresql://ql3_migration:do-not-log@postgres-rw.internal:5432/qinglong',
|
||||
QL3_POSTGRES_TLS_SERVERNAME: 'postgres-rw.internal',
|
||||
});
|
||||
|
||||
test('builds a TLS-verified single-connection migration configuration', () => {
|
||||
assert.deepEqual(
|
||||
loadPostgresMigrationProcessConfig({
|
||||
...BASE_ENV,
|
||||
QL3_POSTGRES_TLS_CA_FILE: CA_FILE,
|
||||
}),
|
||||
{
|
||||
connection: {
|
||||
connectionString: BASE_ENV.QL3_POSTGRES_MIGRATION_URL,
|
||||
tls: {
|
||||
mode: 'verify-full',
|
||||
ca: CA_BUNDLE,
|
||||
servername: 'postgres-rw.internal',
|
||||
},
|
||||
},
|
||||
pool: {
|
||||
applicationName: 'qinglong3-cluster-migration',
|
||||
maxConnections: 1,
|
||||
connectionTimeoutMs: 15_000,
|
||||
},
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('loads a discrete operator-managed migration credential', () => {
|
||||
const config = loadPostgresMigrationProcessConfig({
|
||||
QL3_POSTGRES_MIGRATION_HOST: 'ql3-postgres-rw.qinglong3-system.svc',
|
||||
QL3_POSTGRES_MIGRATION_PORT: '5432',
|
||||
QL3_POSTGRES_MIGRATION_DATABASE: 'qinglong',
|
||||
QL3_POSTGRES_MIGRATION_USER: 'ql3_migration',
|
||||
QL3_POSTGRES_MIGRATION_PASSWORD: 'operator-secret',
|
||||
QL3_POSTGRES_TLS_SERVERNAME: 'ql3-postgres-rw.qinglong3-system.svc',
|
||||
});
|
||||
assert.deepEqual(config.connection, {
|
||||
host: 'ql3-postgres-rw.qinglong3-system.svc',
|
||||
port: 5432,
|
||||
database: 'qinglong',
|
||||
user: 'ql3_migration',
|
||||
password: 'operator-secret',
|
||||
tls: {
|
||||
mode: 'verify-full',
|
||||
servername: 'ql3-postgres-rw.qinglong3-system.svc',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test('requires an explicit second gate before disabling TLS', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
loadPostgresMigrationProcessConfig({
|
||||
...BASE_ENV,
|
||||
QL3_POSTGRES_TLS_MODE: 'disable',
|
||||
}),
|
||||
PostgresMigrationProcessConfigError,
|
||||
);
|
||||
assert.deepEqual(
|
||||
loadPostgresMigrationProcessConfig({
|
||||
...BASE_ENV,
|
||||
QL3_POSTGRES_TLS_MODE: 'disable',
|
||||
QL3_POSTGRES_ALLOW_INSECURE: 'true',
|
||||
}).connection.tls,
|
||||
{ mode: 'disable' },
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects URL TLS overrides and unbounded identity fields', () => {
|
||||
for (const environment of [
|
||||
{},
|
||||
{
|
||||
...BASE_ENV,
|
||||
QL3_POSTGRES_MIGRATION_URL: `${BASE_ENV.QL3_POSTGRES_MIGRATION_URL}?sslmode=disable`,
|
||||
},
|
||||
{
|
||||
...BASE_ENV,
|
||||
QL3_POSTGRES_MIGRATION_HOST: 'postgres-rw.internal',
|
||||
},
|
||||
{ ...BASE_ENV, QL3_POSTGRES_TLS_SERVERNAME: undefined },
|
||||
{ ...BASE_ENV, QL3_POSTGRES_TLS_SERVERNAME: '127.0.0.1' },
|
||||
{ ...BASE_ENV, QL3_POSTGRES_TLS_SERVERNAME: 'unsafe/name' },
|
||||
{ ...BASE_ENV, QL3_POSTGRES_TLS_CA_FILE: 'relative-ca.pem' },
|
||||
{
|
||||
...BASE_ENV,
|
||||
QL3_POSTGRES_TLS_MODE: 'disable',
|
||||
QL3_POSTGRES_ALLOW_INSECURE: 'true',
|
||||
QL3_POSTGRES_TLS_CA_FILE: CA_FILE,
|
||||
},
|
||||
{ ...BASE_ENV, QL3_POSTGRES_APPLICATION_NAME: 'x'.repeat(64) },
|
||||
]) {
|
||||
assert.throws(
|
||||
() => loadPostgresMigrationProcessConfig(environment),
|
||||
PostgresMigrationProcessConfigError,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('runs the reviewed stream on one Pool and always closes it', async () => {
|
||||
const events = [];
|
||||
const pool = { query() {} };
|
||||
const result = await runPostgresMigrationProcess({
|
||||
environment: BASE_ENV,
|
||||
async openDatabase() {
|
||||
events.push('open');
|
||||
return {
|
||||
pool,
|
||||
async close() {
|
||||
events.push('close');
|
||||
},
|
||||
};
|
||||
},
|
||||
async migrate(options) {
|
||||
events.push('migrate');
|
||||
assert.equal(options.pool, pool);
|
||||
},
|
||||
emit(record) {
|
||||
events.push(record.event);
|
||||
},
|
||||
});
|
||||
assert.equal(result, 'migrated');
|
||||
assert.deepEqual(events, [
|
||||
'open',
|
||||
'migration_started',
|
||||
'migrate',
|
||||
'migration_completed',
|
||||
'close',
|
||||
]);
|
||||
});
|
||||
|
||||
test('preserves migration failure while still closing the Pool', async () => {
|
||||
const failure = Object.assign(new Error('migration failed'), {
|
||||
code: 'MIGRATION_FAILED',
|
||||
});
|
||||
const events = [];
|
||||
await assert.rejects(
|
||||
runPostgresMigrationProcess({
|
||||
environment: BASE_ENV,
|
||||
async openDatabase() {
|
||||
return {
|
||||
pool: {},
|
||||
async close() {
|
||||
events.push('close');
|
||||
throw new Error('close failed');
|
||||
},
|
||||
};
|
||||
},
|
||||
async migrate() {
|
||||
throw failure;
|
||||
},
|
||||
}),
|
||||
(error) => error === failure,
|
||||
);
|
||||
assert.deepEqual(events, ['close']);
|
||||
});
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
PluginPackageAutomationPublicationUnavailableError,
|
||||
} = require('@qinglong/runtime-core/plugin-package-automation-publication');
|
||||
const {
|
||||
PostgresPluginPackageAutomationPublicationRepository,
|
||||
} = require('../dist/plugin-package/publication/pluginPackageAutomationPublicationRepository');
|
||||
|
||||
const PROJECT_ID = 'automation-project';
|
||||
const PACKAGE_NAME = 'automation-package';
|
||||
const PUBLICATION_DIGEST = 'a'.repeat(64);
|
||||
|
||||
function pool(query) {
|
||||
return {
|
||||
query,
|
||||
async connect() {
|
||||
throw new Error('transaction client is not expected');
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('delegates the exact automation start decision to the database guard', async () => {
|
||||
const queries = [];
|
||||
const repository =
|
||||
new PostgresPluginPackageAutomationPublicationRepository(pool(
|
||||
async (text, values) => {
|
||||
queries.push({ text, values });
|
||||
return { rows: [{ allowed: true }] };
|
||||
},
|
||||
));
|
||||
|
||||
assert.equal(
|
||||
await repository.isStartAllowed(
|
||||
PROJECT_ID,
|
||||
PACKAGE_NAME,
|
||||
PUBLICATION_DIGEST,
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(queries.length, 1);
|
||||
assert.match(
|
||||
queries[0].text,
|
||||
/"ql3"\."plugin_package_automation_start_allowed"/,
|
||||
);
|
||||
assert.deepEqual(queries[0].values, [
|
||||
PROJECT_ID,
|
||||
PACKAGE_NAME,
|
||||
PUBLICATION_DIGEST,
|
||||
]);
|
||||
});
|
||||
|
||||
test('fails closed when the database guard returns a malformed decision', async () => {
|
||||
const repository =
|
||||
new PostgresPluginPackageAutomationPublicationRepository(pool(
|
||||
async () => {
|
||||
return { rows: [{ allowed: 1 }] };
|
||||
},
|
||||
));
|
||||
|
||||
await assert.rejects(
|
||||
repository.isStartAllowed(
|
||||
PROJECT_ID,
|
||||
PACKAGE_NAME,
|
||||
PUBLICATION_DIGEST,
|
||||
),
|
||||
PluginPackageAutomationPublicationUnavailableError,
|
||||
);
|
||||
});
|
||||
|
||||
test('excludes quarantined and publisher-revoked generations from recovery', async () => {
|
||||
let recoverySql = '';
|
||||
const repository =
|
||||
new PostgresPluginPackageAutomationPublicationRepository(pool(
|
||||
async (text) => {
|
||||
recoverySql = text;
|
||||
return { rows: [] };
|
||||
},
|
||||
));
|
||||
|
||||
assert.deepEqual(await repository.listPendingPage({ limit: 1 }), {
|
||||
candidates: [],
|
||||
truncated: false,
|
||||
});
|
||||
assert.match(recoverySql, /plugin_package_quarantine_events/);
|
||||
assert.match(recoverySql, /plugin_package_publisher_provenance/);
|
||||
assert.match(recoverySql, /plugin_package_publisher_revocation_receipts/);
|
||||
});
|
||||
+184
@@ -0,0 +1,184 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
PostgresPluginPackageIdentityKeysetLedgerConflictError,
|
||||
PostgresPluginPackageIdentityKeysetLedgerRepository,
|
||||
PostgresPluginPackageIdentityKeysetLedgerUnavailableError,
|
||||
} = require('../dist/entrypoints/packageManager');
|
||||
|
||||
function snapshot(generation, overrides = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
generation,
|
||||
digest: String.fromCharCode(64 + generation).repeat(43),
|
||||
issuer: 'https://identity.example.test/',
|
||||
audience: 'qinglong3-package-management',
|
||||
activeKeyIds: [`issuer-key-${generation}`],
|
||||
revokedKeyIds:
|
||||
generation === 1
|
||||
? []
|
||||
: Array.from(
|
||||
{ length: generation - 1 },
|
||||
(_, index) => `issuer-key-${index + 1}`,
|
||||
),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(authority = 'plugin-package-management') {
|
||||
let state;
|
||||
let loseCommitResponse = false;
|
||||
const queries = [];
|
||||
let releases = 0;
|
||||
const client = {
|
||||
async query(text, values = []) {
|
||||
queries.push({ text, values });
|
||||
if (text.startsWith('INSERT')) {
|
||||
state ??= {
|
||||
generation: values[1],
|
||||
digest: values[2],
|
||||
issuer: values[3],
|
||||
audience: values[4],
|
||||
activeKeyIds: JSON.parse(values[5]),
|
||||
revokedKeyIds: JSON.parse(values[6]),
|
||||
};
|
||||
} else if (text.startsWith('SELECT')) {
|
||||
return { rows: state ? [{ ...state }] : [] };
|
||||
} else if (text.startsWith('UPDATE')) {
|
||||
state = {
|
||||
...state,
|
||||
generation: values[1],
|
||||
digest: values[2],
|
||||
activeKeyIds: JSON.parse(values[3]),
|
||||
revokedKeyIds: JSON.parse(values[4]),
|
||||
};
|
||||
} else if (text === 'COMMIT' && loseCommitResponse) {
|
||||
loseCommitResponse = false;
|
||||
throw new Error('response lost after commit');
|
||||
}
|
||||
return { rows: [] };
|
||||
},
|
||||
release() {
|
||||
releases += 1;
|
||||
},
|
||||
};
|
||||
return {
|
||||
repository: new PostgresPluginPackageIdentityKeysetLedgerRepository({
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
async query() {
|
||||
throw new Error('pool query must not bypass the transaction client');
|
||||
},
|
||||
}, authority),
|
||||
queries,
|
||||
state: () => state,
|
||||
releases: () => releases,
|
||||
loseNextCommitResponse() {
|
||||
loseCommitResponse = true;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('serializes first observation, exact replay and append-only rotation', async () => {
|
||||
const value = fixture();
|
||||
await value.repository.observe(snapshot(1));
|
||||
const writesAfterFirst = value.queries.filter(({ text }) =>
|
||||
text.startsWith('UPDATE'),
|
||||
).length;
|
||||
await value.repository.observe(snapshot(1));
|
||||
assert.equal(
|
||||
value.queries.filter(({ text }) => text.startsWith('UPDATE')).length,
|
||||
writesAfterFirst,
|
||||
);
|
||||
await value.repository.observe(snapshot(2));
|
||||
assert.deepEqual(value.state(), {
|
||||
generation: 2,
|
||||
digest: 'B'.repeat(43),
|
||||
issuer: 'https://identity.example.test/',
|
||||
audience: 'qinglong3-package-management',
|
||||
activeKeyIds: ['issuer-key-2'],
|
||||
revokedKeyIds: ['issuer-key-1'],
|
||||
});
|
||||
assert.match(
|
||||
value.queries.find(({ text }) => text.startsWith('INSERT')).text,
|
||||
/clock_timestamp\(\)/,
|
||||
);
|
||||
assert.equal(value.releases(), 3);
|
||||
});
|
||||
|
||||
test('isolates Plugin, Worker, automation and Approval generations by authority key', async () => {
|
||||
const value = fixture('worker-credential-management');
|
||||
await value.repository.observe(
|
||||
snapshot(1, { audience: 'qinglong3-worker-credential-management' }),
|
||||
);
|
||||
const insert = value.queries.find(({ text }) => text.startsWith('INSERT'));
|
||||
assert.equal(insert.values[0], 'worker-credential-management');
|
||||
const automation = fixture('automation-management');
|
||||
await automation.repository.observe(
|
||||
snapshot(1, { audience: 'qinglong3-automation-management' }),
|
||||
);
|
||||
assert.equal(
|
||||
automation.queries.find(({ text }) => text.startsWith('INSERT')).values[0],
|
||||
'automation-management',
|
||||
);
|
||||
const approval = fixture('approval-management');
|
||||
await approval.repository.observe(
|
||||
snapshot(1, { audience: 'qinglong3-approval-management' }),
|
||||
);
|
||||
assert.equal(
|
||||
approval.queries.find(({ text }) => text.startsWith('INSERT')).values[0],
|
||||
'approval-management',
|
||||
);
|
||||
assert.throws(
|
||||
() => fixture('worker-credential-executor'),
|
||||
TypeError,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects rollback, trust-domain rewrite and implicit key removal', async () => {
|
||||
const value = fixture();
|
||||
await value.repository.observe(snapshot(2));
|
||||
await assert.rejects(
|
||||
value.repository.observe(snapshot(1)),
|
||||
PostgresPluginPackageIdentityKeysetLedgerConflictError,
|
||||
);
|
||||
await assert.rejects(
|
||||
value.repository.observe(
|
||||
snapshot(3, { issuer: 'https://other.example.test/' }),
|
||||
),
|
||||
PostgresPluginPackageIdentityKeysetLedgerConflictError,
|
||||
);
|
||||
await assert.rejects(
|
||||
value.repository.observe(
|
||||
snapshot(3, {
|
||||
activeKeyIds: ['issuer-key-3'],
|
||||
revokedKeyIds: [],
|
||||
}),
|
||||
),
|
||||
PostgresPluginPackageIdentityKeysetLedgerConflictError,
|
||||
);
|
||||
});
|
||||
|
||||
test('converges an ambiguous commit and validates bounded snapshots', async () => {
|
||||
const value = fixture();
|
||||
value.loseNextCommitResponse();
|
||||
await assert.rejects(
|
||||
value.repository.observe(snapshot(1)),
|
||||
PostgresPluginPackageIdentityKeysetLedgerUnavailableError,
|
||||
);
|
||||
await value.repository.observe(snapshot(1));
|
||||
assert.equal(value.state().generation, 1);
|
||||
await assert.rejects(
|
||||
value.repository.observe(
|
||||
snapshot(2, { activeKeyIds: ['issuer-key-2', 'issuer-key-2'] }),
|
||||
),
|
||||
TypeError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
new PostgresPluginPackageIdentityKeysetLedgerRepository({ query() {} }),
|
||||
TypeError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
PostgresPluginPackageManagementQuotaRepository,
|
||||
} = require('../dist/entrypoints/packageManager');
|
||||
const {
|
||||
PluginPackageManagementQuotaExceededError,
|
||||
PluginPackageManagementUnavailableError,
|
||||
} = require('@qinglong/runtime-core/plugin-package-management');
|
||||
|
||||
function command(overrides = {}) {
|
||||
return {
|
||||
projectId: 'default',
|
||||
subject: { type: 'user', id: 'cluster-reviewer' },
|
||||
operation: 'plugin-package.inspect',
|
||||
idempotencyKey: 'inspection-1',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('uses one database-clock UPSERT with a bounded in-row replay ledger', async () => {
|
||||
const queries = [];
|
||||
const pool = {
|
||||
async query(text, values) {
|
||||
queries.push({ text, values });
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
admitted: true,
|
||||
consumedCount: '1',
|
||||
resetAtMs: '60000',
|
||||
observedAtMs: '1',
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
};
|
||||
const repository = new PostgresPluginPackageManagementQuotaRepository(pool, {
|
||||
windowMs: 60_000,
|
||||
limits: { 'plugin-package.inspect': 2 },
|
||||
});
|
||||
|
||||
assert.deepEqual(await repository.consume(command()), {
|
||||
remaining: 1,
|
||||
resetAtMs: 60_000,
|
||||
observedAtMs: 1,
|
||||
});
|
||||
assert.equal(queries.length, 1);
|
||||
assert.match(queries[0].text, /clock_timestamp\(\)/);
|
||||
assert.match(queries[0].text, /ON CONFLICT/);
|
||||
assert.match(queries[0].text, /receipt_ids \? \$5::text/);
|
||||
assert.match(queries[0].text, /jsonb_build_array\(\$5::text\)/);
|
||||
assert.doesNotMatch(queries[0].text, /\bBEGIN\b|\bCOMMIT\b/);
|
||||
assert.deepEqual(queries[0].values, [
|
||||
'default',
|
||||
'user',
|
||||
'cluster-reviewer',
|
||||
'plugin-package.inspect',
|
||||
'inspection-1',
|
||||
60_000,
|
||||
2,
|
||||
]);
|
||||
});
|
||||
|
||||
test('preserves one quota unit across exact retries and exposes reset delay', async () => {
|
||||
let calls = 0;
|
||||
const repository = new PostgresPluginPackageManagementQuotaRepository(
|
||||
{
|
||||
async query() {
|
||||
calls += 1;
|
||||
return {
|
||||
rows: [
|
||||
calls <= 2
|
||||
? {
|
||||
admitted: true,
|
||||
consumedCount: '1',
|
||||
resetAtMs: '60000',
|
||||
observedAtMs: String(calls),
|
||||
}
|
||||
: {
|
||||
admitted: false,
|
||||
consumedCount: '1',
|
||||
resetAtMs: '60000',
|
||||
observedAtMs: '1000',
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
limits: { 'plugin-package.inspect': 1 },
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal((await repository.consume(command())).remaining, 0);
|
||||
assert.equal((await repository.consume(command())).remaining, 0);
|
||||
await assert.rejects(
|
||||
repository.consume(command({ idempotencyKey: 'inspection-2' })),
|
||||
(error) =>
|
||||
error instanceof PluginPackageManagementQuotaExceededError &&
|
||||
error.retryAfterMs === 59_000,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects widened commands and maps database failures to unavailable', async () => {
|
||||
const repository = new PostgresPluginPackageManagementQuotaRepository({
|
||||
async query() {
|
||||
throw new Error('secret database diagnostic');
|
||||
},
|
||||
});
|
||||
await assert.rejects(repository.consume(command({ extra: true })), TypeError);
|
||||
await assert.rejects(
|
||||
repository.consume(
|
||||
command({ subject: { type: 'api_app', id: 'cluster-automation' } }),
|
||||
),
|
||||
TypeError,
|
||||
);
|
||||
await assert.rejects(
|
||||
repository.consume(command()),
|
||||
PluginPackageManagementUnavailableError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
new PostgresPluginPackageManagementQuotaRepository(
|
||||
{ query() {} },
|
||||
{ limits: { 'plugin-package.inspect': 1_001 } },
|
||||
),
|
||||
TypeError,
|
||||
);
|
||||
});
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
registerPluginPackageMaterializedRevisionRepositoryContract,
|
||||
} = require('../../../test/contracts/pluginPackageMaterializedRevisionRepositoryContract.cjs');
|
||||
const {
|
||||
PluginPackageResourceMaterializationConflictError,
|
||||
PluginPackageResourceMaterializationUnavailableError,
|
||||
} = require('@qinglong/runtime-core/plugin-package-resource-materialization');
|
||||
const {
|
||||
PostgresPluginPackageMaterializedRevisionRepository,
|
||||
} = require('../dist/plugin-package/installation/pluginPackageMaterializedRevisionRepository');
|
||||
|
||||
function fakePool() {
|
||||
let row;
|
||||
const queries = [];
|
||||
return {
|
||||
queries,
|
||||
pool: {
|
||||
async query(text, values = []) {
|
||||
queries.push({ text, values });
|
||||
if (text.startsWith('INSERT')) {
|
||||
if (row) return { rows: [] };
|
||||
row = {
|
||||
generationDigest: values[0],
|
||||
projectId: values[1],
|
||||
packageName: values[2],
|
||||
generation: values[3],
|
||||
lockDigest: values[4],
|
||||
manifestDigest: values[5],
|
||||
revisionDigest: values[6],
|
||||
revisionJson: JSON.parse(values[7]),
|
||||
createdAtMs: 300,
|
||||
};
|
||||
return { rows: [{ generation_digest: values[0] }] };
|
||||
}
|
||||
if (text.startsWith('SELECT')) {
|
||||
return { rows: row ? [{ ...row }] : [] };
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${text}`);
|
||||
},
|
||||
},
|
||||
corrupt() {
|
||||
row.revisionJson.resources[0].value.name = 'Changed';
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
registerPluginPackageMaterializedRevisionRepositoryContract({
|
||||
name: 'PostgreSQL Plugin Package materialized revision repository',
|
||||
namespace: 'postgres-materialized',
|
||||
profile: 'cluster-control',
|
||||
async createRepository(_t, fixture) {
|
||||
const value = fakePool();
|
||||
return {
|
||||
repository: new PostgresPluginPackageMaterializedRevisionRepository(
|
||||
value.pool,
|
||||
fixture.registry,
|
||||
),
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
test('uses database time and fails closed on corrupted JSON', async () => {
|
||||
const {
|
||||
materializedRevisionFixture,
|
||||
} = require('../../../test/contracts/pluginPackageMaterializedRevisionRepositoryContract.cjs');
|
||||
const fixture = materializedRevisionFixture(
|
||||
'postgres-corrupt',
|
||||
'cluster-control',
|
||||
);
|
||||
const value = fakePool();
|
||||
const repository = new PostgresPluginPackageMaterializedRevisionRepository(
|
||||
value.pool,
|
||||
fixture.registry,
|
||||
);
|
||||
await repository.publish(fixture.revision);
|
||||
assert.match(
|
||||
value.queries.find(({ text }) => text.startsWith('INSERT')).text,
|
||||
/clock_timestamp\(\)/,
|
||||
);
|
||||
value.corrupt();
|
||||
await assert.rejects(
|
||||
repository.find(fixture.revision.generation.generationDigest),
|
||||
PluginPackageResourceMaterializationUnavailableError,
|
||||
);
|
||||
});
|
||||
|
||||
test('maps PostgreSQL uniqueness rejection to a semantic conflict', async () => {
|
||||
const repository =
|
||||
new PostgresPluginPackageMaterializedRevisionRepository({
|
||||
async query() {
|
||||
const error = new Error('duplicate');
|
||||
error.code = '23505';
|
||||
throw error;
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
repository.find('a'.repeat(64)),
|
||||
PluginPackageResourceMaterializationConflictError,
|
||||
);
|
||||
});
|
||||
|
||||
test('publishes storage through package-executor and the explicit subpath', () => {
|
||||
assert.equal(
|
||||
require('@qinglong/cluster-postgres/plugin-package-materialized-revision')
|
||||
.PostgresPluginPackageMaterializedRevisionRepository,
|
||||
PostgresPluginPackageMaterializedRevisionRepository,
|
||||
);
|
||||
assert.equal(
|
||||
require('../dist/entrypoints/packageExecutor')
|
||||
.PostgresPluginPackageMaterializedRevisionRepository,
|
||||
PostgresPluginPackageMaterializedRevisionRepository,
|
||||
);
|
||||
});
|
||||
+204
@@ -0,0 +1,204 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { generateKeyPairSync } = require('node:crypto');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
createPluginPackagePublisherRevocationProposal,
|
||||
PluginPackagePublisherRevocationProposalConflictError,
|
||||
} = require('@qinglong/runtime-core/plugin-package-publisher-revocation-proposal');
|
||||
const {
|
||||
createPluginPackagePublisherTrustSnapshot,
|
||||
} = require('@qinglong/runtime-core/plugin-package-publisher-trust');
|
||||
const {
|
||||
PostgresPluginPackagePublisherRevocationProposalRepository,
|
||||
} = require('../dist/plugin-package/publisher/pluginPackagePublisherRevocationProposalRepository');
|
||||
|
||||
const SUBJECT = Object.freeze({ type: 'user', id: 'usr_owner' });
|
||||
const FENCE = Object.freeze({ projectVersion: 4, bindingVersion: 7 });
|
||||
|
||||
function proposal() {
|
||||
const { publicKey } = generateKeyPairSync('ed25519');
|
||||
const trustSnapshot = createPluginPackagePublisherTrustSnapshot([
|
||||
{
|
||||
publisher: 'publisher-a.example',
|
||||
keyId: 'key-a',
|
||||
publicKeyPem: publicKey.export({ type: 'spki', format: 'pem' }),
|
||||
notBeforeMs: 1,
|
||||
notAfterMs: 10_000,
|
||||
},
|
||||
]);
|
||||
return createPluginPackagePublisherRevocationProposal({
|
||||
actionRef: 'publisher-revoke:publisher-a.example:key-a',
|
||||
authorityProjectId: 'cluster-trust-authority',
|
||||
trustAuthorityId: 'cluster',
|
||||
trustGeneration: 1,
|
||||
trustSnapshot,
|
||||
publisher: 'publisher-a.example',
|
||||
keyId: 'key-a',
|
||||
authorizationMode: 'dual_control',
|
||||
reasonCode: 'suspected_key_compromise',
|
||||
proposedBy: SUBJECT,
|
||||
proposerAssurance: 'multi_factor',
|
||||
proposalFence: FENCE,
|
||||
createdAtMs: 100,
|
||||
});
|
||||
}
|
||||
|
||||
function audit(candidate, overrides = {}) {
|
||||
return {
|
||||
eventId: '32000000-0000-4000-8000-000000000001',
|
||||
requestId: candidate.actionRef,
|
||||
operationId: 'plugin_package.publisher_revocation.propose',
|
||||
projectId: candidate.projectId,
|
||||
subject: candidate.proposedBy,
|
||||
authenticationId: 'auth-owner',
|
||||
outcome: 'allowed',
|
||||
reasons: ['publisher_revocation_proposal'],
|
||||
fence: candidate.proposalFence,
|
||||
occurredAtMs: candidate.createdAtMs,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(candidate, trustOverrides = {}) {
|
||||
let storedProposal = null;
|
||||
let storedAudit = null;
|
||||
let signerLocks = 0;
|
||||
const query = async (text, values = []) => {
|
||||
if (text.includes('pg_advisory_xact_lock(hashtextextended')) {
|
||||
signerLocks += 1;
|
||||
assert.deepEqual(values, [
|
||||
JSON.stringify([
|
||||
candidate.actionInput.publisher,
|
||||
candidate.actionInput.keyId,
|
||||
]),
|
||||
774635229,
|
||||
]);
|
||||
return { rows: [{}], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'FROM "ql3"."plugin_package_publisher_revocation_proposals"',
|
||||
)
|
||||
) {
|
||||
return {
|
||||
rows: storedProposal
|
||||
? [
|
||||
{
|
||||
proposalJson: storedProposal,
|
||||
proposalDigest: storedProposal.proposalDigest,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'FROM "ql3"."plugin_package_publisher_trust_heads"',
|
||||
)
|
||||
) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
generation:
|
||||
trustOverrides.generation ??
|
||||
candidate.actionInput.trustGeneration,
|
||||
effectiveTrustDigest:
|
||||
trustOverrides.effectiveTrustDigest ??
|
||||
candidate.actionInput.previousTrustDigest,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('lock_approval_policy_fence')) {
|
||||
return { rows: [{ matches: true }] };
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'INSERT INTO\n "ql3"."plugin_package_publisher_revocation_proposals"',
|
||||
)
|
||||
) {
|
||||
storedProposal = JSON.parse(values[20]);
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('INSERT INTO "ql3"."security_audit_events"')) {
|
||||
storedAudit = audit(candidate);
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."security_audit_events"')) {
|
||||
return {
|
||||
rows: storedAudit
|
||||
? [
|
||||
{
|
||||
eventId: storedAudit.eventId,
|
||||
requestId: storedAudit.requestId,
|
||||
operationId: storedAudit.operationId,
|
||||
projectId: storedAudit.projectId,
|
||||
subjectType: storedAudit.subject.type,
|
||||
subjectId: storedAudit.subject.id,
|
||||
authenticationId: storedAudit.authenticationId,
|
||||
outcome: storedAudit.outcome,
|
||||
reasons: storedAudit.reasons,
|
||||
projectVersion: storedAudit.fence.projectVersion,
|
||||
bindingVersion: storedAudit.fence.bindingVersion,
|
||||
occurredAtMs: storedAudit.occurredAtMs,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
return { rows: [], rowCount: 0 };
|
||||
};
|
||||
const client = { query, release() {} };
|
||||
const repository =
|
||||
new PostgresPluginPackagePublisherRevocationProposalRepository({
|
||||
query,
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
});
|
||||
repository.signerLocks = () => signerLocks;
|
||||
return repository;
|
||||
}
|
||||
|
||||
test('persists and exactly replays a generation-fenced revocation proposal', async () => {
|
||||
const candidate = proposal();
|
||||
const repository = fixture(candidate);
|
||||
const created = await repository.createProposal({
|
||||
proposal: candidate,
|
||||
audit: audit(candidate),
|
||||
});
|
||||
assert.equal(created.status, 'created');
|
||||
assert.equal(repository.signerLocks(), 1);
|
||||
const replay = await repository.createProposal({
|
||||
proposal: candidate,
|
||||
audit: audit(candidate),
|
||||
});
|
||||
assert.equal(replay.status, 'existing');
|
||||
assert.equal(repository.signerLocks(), 1);
|
||||
assert.deepEqual(
|
||||
await repository.findProposalByActionRef(candidate.actionRef),
|
||||
candidate,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects stale trust generations and mismatched audit authority', async () => {
|
||||
const candidate = proposal();
|
||||
await assert.rejects(
|
||||
fixture(candidate, { generation: 2 }).createProposal({
|
||||
proposal: candidate,
|
||||
audit: audit(candidate),
|
||||
}),
|
||||
PluginPackagePublisherRevocationProposalConflictError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
fixture(candidate).createProposal({
|
||||
proposal: candidate,
|
||||
audit: audit(candidate, {
|
||||
reasons: ['client_supplied_transition'],
|
||||
}),
|
||||
}),
|
||||
PluginPackagePublisherRevocationProposalConflictError,
|
||||
);
|
||||
});
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { generateKeyPairSync } = require('node:crypto');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
createPluginPackagePublisherTrustSnapshot,
|
||||
} = require('@qinglong/runtime-core/plugin-package-publisher-trust');
|
||||
const {
|
||||
PostgresPluginPackagePublisherTrustAuthorityRepository,
|
||||
} = require('../dist/plugin-package/publisher/pluginPackagePublisherTrustAuthorityRepository');
|
||||
|
||||
function snapshot(publisher, keyId) {
|
||||
const { publicKey } = generateKeyPairSync('ed25519');
|
||||
return createPluginPackagePublisherTrustSnapshot([
|
||||
{
|
||||
publisher,
|
||||
keyId,
|
||||
publicKeyPem: publicKey.export({ type: 'spki', format: 'pem' }),
|
||||
notBeforeMs: 1,
|
||||
notAfterMs: 10_000,
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
function fixture() {
|
||||
const snapshots = new Map();
|
||||
let head = null;
|
||||
const queries = [];
|
||||
const query = async (text, values = []) => {
|
||||
queries.push({ text, values });
|
||||
if (
|
||||
text.includes(
|
||||
'INSERT INTO "ql3"."plugin_package_publisher_trust_snapshots"',
|
||||
)
|
||||
) {
|
||||
snapshots.set(values[0], JSON.parse(values[4]));
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'FROM "ql3"."plugin_package_publisher_trust_heads" AS head',
|
||||
)
|
||||
) {
|
||||
if (!head || head.authorityId !== values[0]) return { rows: [] };
|
||||
const effective = snapshots.get(head.effectiveTrustDigest);
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
headJson: head,
|
||||
headDigest: head.headDigest,
|
||||
snapshotJson: effective,
|
||||
snapshotDigest: effective.snapshotDigest,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'INSERT INTO "ql3"."plugin_package_publisher_trust_heads"',
|
||||
)
|
||||
) {
|
||||
if (head && head.authorityId === values[0]) {
|
||||
return { rows: [], rowCount: 0 };
|
||||
}
|
||||
head = JSON.parse(values[6]);
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
return { rows: [], rowCount: 0 };
|
||||
};
|
||||
const client = {
|
||||
query,
|
||||
release() {},
|
||||
};
|
||||
const repository =
|
||||
new PostgresPluginPackagePublisherTrustAuthorityRepository({
|
||||
query,
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
});
|
||||
return { repository, queries, head: () => head };
|
||||
}
|
||||
|
||||
test('observes one base snapshot and returns the durable effective head', async () => {
|
||||
const value = fixture();
|
||||
const base = snapshot('publisher-a.example', 'key-a');
|
||||
const created = await value.repository.observeSnapshot({
|
||||
authorityId: 'cluster',
|
||||
snapshot: base,
|
||||
observedBy: 'package-manager-1',
|
||||
observedAtMs: 100,
|
||||
});
|
||||
assert.equal(created.status, 'created');
|
||||
assert.equal(created.head.generation, 1);
|
||||
assert.equal(created.head.baseSnapshotDigest, base.snapshotDigest);
|
||||
assert.equal(
|
||||
created.head.effectiveTrustDigest,
|
||||
base.snapshotDigest,
|
||||
);
|
||||
|
||||
const replay = await value.repository.observeSnapshot({
|
||||
authorityId: 'cluster',
|
||||
snapshot: base,
|
||||
observedBy: 'package-manager-2',
|
||||
observedAtMs: 200,
|
||||
});
|
||||
assert.equal(replay.status, 'existing');
|
||||
assert.deepEqual(
|
||||
await value.repository.findAuthority('cluster'),
|
||||
{
|
||||
head: value.head(),
|
||||
effectiveSnapshot: base,
|
||||
},
|
||||
);
|
||||
assert.equal(
|
||||
value.queries.filter(({ text }) =>
|
||||
text.includes(
|
||||
'INSERT INTO "ql3"."plugin_package_publisher_trust_heads"',
|
||||
),
|
||||
).length,
|
||||
2,
|
||||
);
|
||||
});
|
||||
|
||||
test('observes changed material as a candidate without advancing the head', async () => {
|
||||
const value = fixture();
|
||||
const base = snapshot('publisher-a.example', 'key-a');
|
||||
await value.repository.observeSnapshot({
|
||||
authorityId: 'cluster',
|
||||
snapshot: base,
|
||||
observedBy: 'package-manager-1',
|
||||
observedAtMs: 100,
|
||||
});
|
||||
const candidate = snapshot('publisher-b.example', 'key-b');
|
||||
const observed = await value.repository.observeSnapshot({
|
||||
authorityId: 'cluster',
|
||||
snapshot: candidate,
|
||||
observedBy: 'package-manager-1',
|
||||
observedAtMs: 200,
|
||||
});
|
||||
assert.equal(observed.status, 'candidate');
|
||||
assert.equal(observed.head.generation, 1);
|
||||
assert.equal(observed.head.baseSnapshotDigest, base.snapshotDigest);
|
||||
assert.equal(observed.head.effectiveTrustDigest, base.snapshotDigest);
|
||||
assert.deepEqual(observed.effectiveSnapshot, base);
|
||||
});
|
||||
+330
@@ -0,0 +1,330 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { generateKeyPairSync } = require('node:crypto');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
approvalRequestDigest,
|
||||
createApprovalRequest,
|
||||
decideApprovalRequest,
|
||||
} = require('@qinglong/runtime-core/approved-action');
|
||||
const {
|
||||
createPluginPackagePublisherTrustSnapshot,
|
||||
} = require('@qinglong/runtime-core/plugin-package-publisher-trust');
|
||||
const {
|
||||
createPluginPackagePublisherTrustTransitionProposal,
|
||||
PluginPackagePublisherTrustTransitionConflictError,
|
||||
} = require('@qinglong/runtime-core/plugin-package-publisher-trust-transition-proposal');
|
||||
const {
|
||||
PostgresPluginPackagePublisherTrustTransitionProposalRepository,
|
||||
} = require('../dist/plugin-package/publisher/pluginPackagePublisherTrustTransitionProposalRepository');
|
||||
|
||||
const SUBJECT = Object.freeze({ type: 'user', id: 'usr_owner' });
|
||||
const FENCE = Object.freeze({ projectVersion: 4, bindingVersion: 7 });
|
||||
|
||||
function transition() {
|
||||
const oldPair = generateKeyPairSync('ed25519');
|
||||
const newPair = generateKeyPairSync('ed25519');
|
||||
const oldDefinition = {
|
||||
publisher: 'publisher-a.example',
|
||||
keyId: 'key-old',
|
||||
publicKeyPem: oldPair.publicKey.export({
|
||||
type: 'spki',
|
||||
format: 'pem',
|
||||
}),
|
||||
notBeforeMs: 1,
|
||||
notAfterMs: 10_000,
|
||||
};
|
||||
const newDefinition = {
|
||||
publisher: 'publisher-a.example',
|
||||
keyId: 'key-new',
|
||||
publicKeyPem: newPair.publicKey.export({
|
||||
type: 'spki',
|
||||
format: 'pem',
|
||||
}),
|
||||
notBeforeMs: 1,
|
||||
notAfterMs: 20_000,
|
||||
};
|
||||
const currentSnapshot =
|
||||
createPluginPackagePublisherTrustSnapshot([oldDefinition]);
|
||||
const materialSnapshot =
|
||||
createPluginPackagePublisherTrustSnapshot([
|
||||
oldDefinition,
|
||||
newDefinition,
|
||||
]);
|
||||
const created =
|
||||
createPluginPackagePublisherTrustTransitionProposal({
|
||||
actionRef: 'publisher-overlap:publisher-a.example:key-new',
|
||||
authorityProjectId: 'cluster-trust-authority',
|
||||
trustAuthorityId: 'cluster',
|
||||
trustGeneration: 1,
|
||||
mode: 'overlap_add',
|
||||
trustSnapshot: currentSnapshot,
|
||||
materialSnapshot,
|
||||
publisher: 'publisher-a.example',
|
||||
keyId: 'key-new',
|
||||
proposedBy: SUBJECT,
|
||||
proposerAssurance: 'multi_factor',
|
||||
proposalFence: FENCE,
|
||||
createdAtMs: 100,
|
||||
});
|
||||
return {
|
||||
...created,
|
||||
currentSnapshot,
|
||||
};
|
||||
}
|
||||
|
||||
function audit(proposal, overrides = {}) {
|
||||
return {
|
||||
eventId: '33000000-0000-4000-8000-000000000001',
|
||||
requestId: proposal.actionRef,
|
||||
operationId: 'plugin_package.publisher_trust_transition.propose',
|
||||
projectId: proposal.projectId,
|
||||
subject: proposal.proposedBy,
|
||||
authenticationId: 'auth-owner',
|
||||
outcome: 'allowed',
|
||||
reasons: ['publisher_trust_transition_proposal'],
|
||||
fence: proposal.proposalFence,
|
||||
occurredAtMs: proposal.createdAtMs,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(value, trustOverrides = {}) {
|
||||
const snapshots = new Map([
|
||||
[value.currentSnapshot.snapshotDigest, value.currentSnapshot],
|
||||
]);
|
||||
let storedProposal = null;
|
||||
let storedAudit = null;
|
||||
let signerLocks = 0;
|
||||
const query = async (text, values = []) => {
|
||||
if (text.includes('pg_advisory_xact_lock(hashtextextended')) {
|
||||
signerLocks += 1;
|
||||
assert.deepEqual(values, [
|
||||
JSON.stringify([
|
||||
value.proposal.actionInput.publisher,
|
||||
value.proposal.actionInput.keyId,
|
||||
]),
|
||||
774635229,
|
||||
]);
|
||||
return { rows: [{}], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'FROM "ql3"."plugin_package_publisher_trust_transition_proposals"',
|
||||
)
|
||||
) {
|
||||
return {
|
||||
rows: storedProposal
|
||||
? [
|
||||
{
|
||||
proposalJson: storedProposal,
|
||||
proposalDigest: storedProposal.proposalDigest,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'FROM "ql3"."plugin_package_publisher_trust_heads" AS head',
|
||||
)
|
||||
) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
generation:
|
||||
trustOverrides.generation ??
|
||||
value.proposal.actionInput.trustGeneration,
|
||||
effectiveTrustDigest:
|
||||
trustOverrides.effectiveTrustDigest ??
|
||||
value.proposal.actionInput.previousTrustDigest,
|
||||
snapshotJson: value.currentSnapshot,
|
||||
snapshotDigest: value.currentSnapshot.snapshotDigest,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'INSERT INTO "ql3"."plugin_package_publisher_trust_snapshots"',
|
||||
)
|
||||
) {
|
||||
const existed = snapshots.has(values[0]);
|
||||
if (!existed) snapshots.set(values[0], JSON.parse(values[4]));
|
||||
return { rows: [], rowCount: existed ? 0 : 1 };
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'FROM "ql3"."plugin_package_publisher_trust_snapshots"',
|
||||
)
|
||||
) {
|
||||
const snapshot = snapshots.get(values[0]);
|
||||
return {
|
||||
rows: snapshot
|
||||
? [
|
||||
{
|
||||
snapshotJson: snapshot,
|
||||
snapshotDigest: snapshot.snapshotDigest,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
if (text.includes('lock_approval_policy_fence')) {
|
||||
return { rows: [{ matches: true }] };
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'"ql3"."plugin_package_publisher_trust_transition_proposals" (',
|
||||
)
|
||||
) {
|
||||
storedProposal = JSON.parse(values[19]);
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('INSERT INTO "ql3"."security_audit_events"')) {
|
||||
storedAudit = audit(value.proposal);
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."security_audit_events"')) {
|
||||
return {
|
||||
rows: storedAudit
|
||||
? [
|
||||
{
|
||||
eventId: storedAudit.eventId,
|
||||
requestId: storedAudit.requestId,
|
||||
operationId: storedAudit.operationId,
|
||||
projectId: storedAudit.projectId,
|
||||
subjectType: storedAudit.subject.type,
|
||||
subjectId: storedAudit.subject.id,
|
||||
authenticationId: storedAudit.authenticationId,
|
||||
outcome: storedAudit.outcome,
|
||||
reasons: storedAudit.reasons,
|
||||
projectVersion: storedAudit.fence.projectVersion,
|
||||
bindingVersion: storedAudit.fence.bindingVersion,
|
||||
occurredAtMs: storedAudit.occurredAtMs,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
return { rows: [], rowCount: 0 };
|
||||
};
|
||||
const client = { query, release() {} };
|
||||
const repository =
|
||||
new PostgresPluginPackagePublisherTrustTransitionProposalRepository({
|
||||
query,
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
});
|
||||
repository.signerLocks = () => signerLocks;
|
||||
repository.snapshots = () => snapshots;
|
||||
return repository;
|
||||
}
|
||||
|
||||
test('persists candidate material and exactly replays an overlap proposal', async () => {
|
||||
const value = transition();
|
||||
const repository = fixture(value);
|
||||
const command = {
|
||||
proposal: value.proposal,
|
||||
candidateSnapshot: value.candidateSnapshot,
|
||||
audit: audit(value.proposal),
|
||||
};
|
||||
const created = await repository.createProposal(command);
|
||||
assert.equal(created.status, 'created');
|
||||
assert.equal(repository.signerLocks(), 1);
|
||||
assert.deepEqual(
|
||||
repository.snapshots().get(value.candidateSnapshot.snapshotDigest),
|
||||
value.candidateSnapshot,
|
||||
);
|
||||
|
||||
const replay = await repository.createProposal(command);
|
||||
assert.equal(replay.status, 'existing');
|
||||
assert.equal(repository.signerLocks(), 1);
|
||||
assert.deepEqual(
|
||||
await repository.findProposalByActionRef(value.proposal.actionRef),
|
||||
value.proposal,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects stale trust generations and client-shaped candidate drift', async () => {
|
||||
const value = transition();
|
||||
await assert.rejects(
|
||||
fixture(value, { generation: 2 }).createProposal({
|
||||
proposal: value.proposal,
|
||||
candidateSnapshot: value.candidateSnapshot,
|
||||
audit: audit(value.proposal),
|
||||
}),
|
||||
PluginPackagePublisherTrustTransitionConflictError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
fixture(value).createProposal({
|
||||
proposal: value.proposal,
|
||||
candidateSnapshot: value.currentSnapshot,
|
||||
audit: audit(value.proposal),
|
||||
}),
|
||||
PluginPackagePublisherTrustTransitionConflictError,
|
||||
);
|
||||
});
|
||||
|
||||
test('lists only separation-of-duty approvals from the durable JSON record', async () => {
|
||||
const value = transition();
|
||||
const pending = createApprovalRequest({
|
||||
id: 'approval-trust-transition-list',
|
||||
projectId: value.proposal.projectId,
|
||||
action: {
|
||||
permission: value.proposal.permission,
|
||||
actionType: value.proposal.actionType,
|
||||
actionRef: value.proposal.actionRef,
|
||||
actionDigest: value.proposal.actionDigest,
|
||||
previewDigest: value.proposal.previewDigest,
|
||||
},
|
||||
risk: 'critical',
|
||||
decisionMode: 'separation_of_duty',
|
||||
requestedBy: SUBJECT,
|
||||
requestedAtMs: 100,
|
||||
expiresAtMs: 10_000,
|
||||
requestFence: FENCE,
|
||||
});
|
||||
const approved = decideApprovalRequest(pending, {
|
||||
expectedVersion: 1,
|
||||
decisionId: 'decision-trust-transition-list',
|
||||
decision: 'approved',
|
||||
reasonCode: 'reviewed',
|
||||
principal: {
|
||||
subject: { type: 'user', id: 'usr_reviewer' },
|
||||
authenticationId: 'auth-reviewer',
|
||||
authenticatedAtMs: 100,
|
||||
expiresAtMs: 10_000,
|
||||
assurance: 'multi_factor',
|
||||
},
|
||||
decidedAtMs: 101,
|
||||
authorizationFence: FENCE,
|
||||
});
|
||||
let inspectedSql = '';
|
||||
const repository =
|
||||
new PostgresPluginPackagePublisherTrustTransitionProposalRepository({
|
||||
async query(text) {
|
||||
inspectedSql = text;
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
requestJson: approved,
|
||||
requestDigest: approvalRequestDigest(approved),
|
||||
},
|
||||
],
|
||||
rowCount: 1,
|
||||
};
|
||||
},
|
||||
async connect() {
|
||||
throw new Error('listApprovedRequests must not open a transaction');
|
||||
},
|
||||
});
|
||||
assert.deepEqual(await repository.listApprovedRequests(4), [approved]);
|
||||
assert.match(
|
||||
inspectedSql,
|
||||
/request\.request_json ->> 'decisionMode'/,
|
||||
);
|
||||
assert.doesNotMatch(inspectedSql, /request\.decision_mode/);
|
||||
});
|
||||
+292
@@ -0,0 +1,292 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { generateKeyPairSync } = require('node:crypto');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
approvalRequestDigest,
|
||||
approvedActionDispatchDigest,
|
||||
consumeApprovalRequest,
|
||||
createApprovalRequest,
|
||||
decideApprovalRequest,
|
||||
} = require('@qinglong/runtime-core/approved-action');
|
||||
const {
|
||||
createPluginPackagePublisherTrustHead,
|
||||
createPluginPackagePublisherTrustSnapshot,
|
||||
} = require('@qinglong/runtime-core/plugin-package-publisher-trust');
|
||||
const {
|
||||
createPluginPackagePublisherTrustTransitionProposal,
|
||||
PluginPackagePublisherTrustTransitionConflictError,
|
||||
} = require('@qinglong/runtime-core/plugin-package-publisher-trust-transition-proposal');
|
||||
const {
|
||||
PostgresPluginPackagePublisherTrustTransitionRepository,
|
||||
} = require('../dist/plugin-package/publisher/pluginPackagePublisherTrustTransitionRepository');
|
||||
|
||||
const REQUESTER = Object.freeze({ type: 'user', id: 'usr_owner' });
|
||||
const REVIEWER = Object.freeze({ type: 'user', id: 'usr_security' });
|
||||
const SYSTEM = Object.freeze({
|
||||
type: 'system',
|
||||
id: 'cluster_package_executor',
|
||||
});
|
||||
const FENCE = Object.freeze({ projectVersion: 4, bindingVersion: 7 });
|
||||
|
||||
function authority(mode = 'overlap_add') {
|
||||
const oldPair = generateKeyPairSync('ed25519');
|
||||
const newPair = generateKeyPairSync('ed25519');
|
||||
const oldDefinition = {
|
||||
publisher: 'publisher-a.example',
|
||||
keyId: 'key-old',
|
||||
publicKeyPem: oldPair.publicKey.export({
|
||||
type: 'spki',
|
||||
format: 'pem',
|
||||
}),
|
||||
notBeforeMs: 1,
|
||||
notAfterMs: 10_000,
|
||||
};
|
||||
const newDefinition = {
|
||||
publisher: 'publisher-a.example',
|
||||
keyId: 'key-new',
|
||||
publicKeyPem: newPair.publicKey.export({
|
||||
type: 'spki',
|
||||
format: 'pem',
|
||||
}),
|
||||
notBeforeMs: 1,
|
||||
notAfterMs: 20_000,
|
||||
};
|
||||
const currentSnapshot =
|
||||
createPluginPackagePublisherTrustSnapshot(
|
||||
mode === 'overlap_add'
|
||||
? [oldDefinition]
|
||||
: [oldDefinition, newDefinition],
|
||||
);
|
||||
const created =
|
||||
createPluginPackagePublisherTrustTransitionProposal({
|
||||
actionRef:
|
||||
mode === 'overlap_add'
|
||||
? 'publisher-overlap:publisher-a.example:key-new'
|
||||
: 'publisher-retire:publisher-a.example:key-old',
|
||||
authorityProjectId: 'cluster-trust-authority',
|
||||
trustAuthorityId: 'cluster',
|
||||
trustGeneration: 1,
|
||||
mode,
|
||||
trustSnapshot: currentSnapshot,
|
||||
...(mode === 'overlap_add'
|
||||
? {
|
||||
materialSnapshot:
|
||||
createPluginPackagePublisherTrustSnapshot([
|
||||
oldDefinition,
|
||||
newDefinition,
|
||||
]),
|
||||
}
|
||||
: {}),
|
||||
publisher: 'publisher-a.example',
|
||||
keyId: mode === 'overlap_add' ? 'key-new' : 'key-old',
|
||||
proposedBy: REQUESTER,
|
||||
proposerAssurance: 'multi_factor',
|
||||
proposalFence: FENCE,
|
||||
createdAtMs: 100,
|
||||
});
|
||||
const action = {
|
||||
permission: created.proposal.permission,
|
||||
actionType: created.proposal.actionType,
|
||||
actionRef: created.proposal.actionRef,
|
||||
actionDigest: created.proposal.actionDigest,
|
||||
previewDigest: created.proposal.previewDigest,
|
||||
};
|
||||
const pending = createApprovalRequest({
|
||||
id: `approval-${mode}`,
|
||||
projectId: created.proposal.projectId,
|
||||
action,
|
||||
risk: 'critical',
|
||||
decisionMode: 'separation_of_duty',
|
||||
requestedBy: REQUESTER,
|
||||
requestedAtMs: 100,
|
||||
expiresAtMs: 1_000,
|
||||
requestFence: FENCE,
|
||||
});
|
||||
const approved = decideApprovalRequest(pending, {
|
||||
expectedVersion: 1,
|
||||
decisionId: `decision-${mode}`,
|
||||
decision: 'approved',
|
||||
reasonCode: 'reviewed',
|
||||
principal: {
|
||||
subject: REVIEWER,
|
||||
authenticationId: 'auth-reviewer',
|
||||
authenticatedAtMs: 101,
|
||||
expiresAtMs: 900,
|
||||
assurance: 'multi_factor',
|
||||
},
|
||||
decidedAtMs: 110,
|
||||
authorizationFence: FENCE,
|
||||
});
|
||||
const consumed = consumeApprovalRequest(approved, {
|
||||
expectedVersion: 2,
|
||||
consumptionId: `consume-${mode}`,
|
||||
dispatchId: `dispatch-${mode}`,
|
||||
action,
|
||||
requestedBy: REQUESTER,
|
||||
consumedBy: SYSTEM,
|
||||
consumedAtMs: 120,
|
||||
authorizationFence: FENCE,
|
||||
});
|
||||
return {
|
||||
...created,
|
||||
currentSnapshot,
|
||||
approval: consumed.request,
|
||||
dispatch: consumed.dispatch,
|
||||
initialHead: createPluginPackagePublisherTrustHead(
|
||||
'cluster',
|
||||
currentSnapshot,
|
||||
50,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(value, options = {}) {
|
||||
let head = value.initialHead;
|
||||
let receipt = null;
|
||||
let signerLocks = 0;
|
||||
const query = async (text, values = []) => {
|
||||
if (
|
||||
text.includes(
|
||||
'FROM "ql3"."approved_action_dispatches" AS dispatch',
|
||||
)
|
||||
) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
proposalJson: value.proposal,
|
||||
proposalDigest: value.proposal.proposalDigest,
|
||||
dispatchJson: value.dispatch,
|
||||
dispatchDigest: approvedActionDispatchDigest(value.dispatch),
|
||||
approvalJson: value.approval,
|
||||
approvalDigest: approvalRequestDigest(value.approval),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('pg_advisory_xact_lock(hashtextextended')) {
|
||||
signerLocks += 1;
|
||||
return { rows: [{}], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'FROM "ql3"."plugin_package_publisher_trust_transition_receipts"',
|
||||
)
|
||||
) {
|
||||
return {
|
||||
rows: receipt
|
||||
? [
|
||||
{
|
||||
receiptJson: receipt,
|
||||
receiptDigest: receipt.receiptDigest,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'FROM "ql3"."plugin_package_publisher_trust_heads" AS head',
|
||||
)
|
||||
) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
headJson: head,
|
||||
headDigest: head.headDigest,
|
||||
effectiveSnapshotJson: value.currentSnapshot,
|
||||
effectiveSnapshotDigest:
|
||||
value.currentSnapshot.snapshotDigest,
|
||||
candidateSnapshotJson: value.candidateSnapshot,
|
||||
candidateSnapshotDigest:
|
||||
value.candidateSnapshot.snapshotDigest,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'FROM "ql3"."plugin_package_publisher_trust_heads"',
|
||||
)
|
||||
) {
|
||||
return {
|
||||
rows: [{ headJson: head, headDigest: head.headDigest }],
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'FROM "ql3"."plugin_package_publisher_provenance" AS provenance',
|
||||
)
|
||||
) {
|
||||
return {
|
||||
rows: options.matchingInstall
|
||||
? [{ installationId: 'install-old' }]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'UPDATE "ql3"."plugin_package_publisher_trust_heads"',
|
||||
)
|
||||
) {
|
||||
head = JSON.parse(values[5]);
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'"ql3"."plugin_package_publisher_trust_transition_receipts" (',
|
||||
)
|
||||
) {
|
||||
receipt = JSON.parse(values[16]);
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
return { rows: [], rowCount: 0 };
|
||||
};
|
||||
const client = { query, release() {} };
|
||||
const repository =
|
||||
new PostgresPluginPackagePublisherTrustTransitionRepository({
|
||||
query,
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
});
|
||||
repository.signerLocks = () => signerLocks;
|
||||
repository.head = () => head;
|
||||
return repository;
|
||||
}
|
||||
|
||||
test('atomically advances overlap trust and exactly replays its receipt', async () => {
|
||||
const value = authority();
|
||||
const repository = fixture(value);
|
||||
const created = await repository.applyApprovedTransition({
|
||||
dispatch: value.dispatch,
|
||||
executedAtMs: 130,
|
||||
});
|
||||
assert.equal(created.status, 'created');
|
||||
assert.equal(created.receipt.mode, 'overlap_add');
|
||||
assert.equal(created.receipt.retirementMatchingInstallations, null);
|
||||
assert.equal(created.head.generation, 2);
|
||||
assert.equal(
|
||||
created.head.effectiveTrustDigest,
|
||||
value.candidateSnapshot.snapshotDigest,
|
||||
);
|
||||
|
||||
const replay = await repository.applyApprovedTransition({
|
||||
dispatch: value.dispatch,
|
||||
executedAtMs: 130,
|
||||
});
|
||||
assert.equal(replay.status, 'existing');
|
||||
assert.equal(replay.receipt.receiptDigest, created.receipt.receiptDigest);
|
||||
assert.equal(repository.signerLocks(), 2);
|
||||
});
|
||||
|
||||
test('blocks safe retirement while a current installation uses the signer', async () => {
|
||||
const value = authority('safe_retire');
|
||||
await assert.rejects(
|
||||
fixture(value, { matchingInstall: true }).applyApprovedTransition({
|
||||
dispatch: value.dispatch,
|
||||
executedAtMs: 130,
|
||||
}),
|
||||
PluginPackagePublisherTrustTransitionConflictError,
|
||||
);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,435 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
createInitialPluginPackageAutomationPublication,
|
||||
} = require('@qinglong/runtime-core/plugin-package-automation-publication');
|
||||
const {
|
||||
createPluginPackageWorkflowAdmissionBundle,
|
||||
createPluginPackageWorkflowExecutionPlan,
|
||||
PluginPackageWorkflowAdmissionNotAllowedError,
|
||||
PluginPackageWorkflowAdmissionUnavailableError,
|
||||
} = require('@qinglong/runtime-core/plugin-package-workflow-execution-plan');
|
||||
const {
|
||||
transitionStepRunMutation,
|
||||
} = require('@qinglong/runtime-core/step-run');
|
||||
const {
|
||||
pluginPackageTaskReconciliationFixture,
|
||||
} = require('../../../test/contracts/pluginPackageTaskReconciliationRepositoryContract.cjs');
|
||||
const {
|
||||
PostgresPluginPackageWorkflowAdmissionRepository,
|
||||
} = require('../dist/plugin-package/workflow/pluginPackageWorkflowAdmissionRepository');
|
||||
|
||||
function fixture(namespace = 'postgres-workflow-admission') {
|
||||
const value = pluginPackageTaskReconciliationFixture(namespace, {
|
||||
workflows: [
|
||||
{
|
||||
schema: 'qinglong/plugin-package-workflow-resource@v1',
|
||||
id: 'daily',
|
||||
name: 'Daily workflow',
|
||||
enabled: true,
|
||||
steps: [
|
||||
{ id: 'collect', task: 'alpha', needs: [] },
|
||||
{ id: 'summarize', task: 'beta', needs: ['collect'] },
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
const publication = createInitialPluginPackageAutomationPublication(
|
||||
value.revision,
|
||||
value.registry,
|
||||
2_000,
|
||||
);
|
||||
const plan = createPluginPackageWorkflowExecutionPlan({
|
||||
planId: `workflow-plan-${namespace}`,
|
||||
runId: `run-${namespace}`,
|
||||
workflowId: 'daily',
|
||||
stepRunIds: {
|
||||
collect: `step-collect-${namespace}`,
|
||||
summarize: `step-summarize-${namespace}`,
|
||||
},
|
||||
publication,
|
||||
revision: value.revision,
|
||||
taskSpecSemanticRegistry: value.registry,
|
||||
plannedAtMs: 3_000,
|
||||
});
|
||||
return { ...value, publication, plan };
|
||||
}
|
||||
|
||||
function poolWithClient(client) {
|
||||
return {
|
||||
async query(text, values) {
|
||||
return client.query(text, values);
|
||||
},
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function admissionRow(bundle) {
|
||||
const { plan, receipt } = bundle;
|
||||
return {
|
||||
planDigest: plan.planDigest,
|
||||
planId: plan.planId,
|
||||
runId: plan.runId,
|
||||
projectId: plan.target.projectId,
|
||||
packageName: plan.target.packageName,
|
||||
installationId: plan.target.installationId,
|
||||
lockDigest: plan.target.lockDigest,
|
||||
generation: plan.target.generation,
|
||||
generationDigest: plan.target.generationDigest,
|
||||
materializedRevisionDigest: plan.target.materializedRevisionDigest,
|
||||
publicationDigest: plan.target.publicationDigest,
|
||||
workflowId: plan.target.workflowId,
|
||||
workflowDefinitionDigest: plan.target.workflowDefinitionDigest,
|
||||
stepCount: plan.steps.length,
|
||||
admittedAtMs: receipt.admittedAtMs,
|
||||
finalRunVersion: receipt.finalRunVersion,
|
||||
finalRunEventSequence: receipt.finalRunEventSequence,
|
||||
receiptDigest: receipt.receiptDigest,
|
||||
planJson: plan,
|
||||
receiptJson: receipt,
|
||||
};
|
||||
}
|
||||
|
||||
function runRow(run) {
|
||||
return {
|
||||
projectId: run.projectId,
|
||||
taskId: run.taskId,
|
||||
taskRevision: run.taskRevision,
|
||||
taskSnapshotRef: run.taskSnapshotRef ?? null,
|
||||
triggerType: run.triggerType,
|
||||
executionOrigin: run.executionOrigin,
|
||||
executionOwner: run.executionOwner,
|
||||
requestId: run.requestId ?? null,
|
||||
status: run.status,
|
||||
version: run.version,
|
||||
eventSequence: run.eventSequence,
|
||||
priority: run.priority,
|
||||
idempotencyKey: run.idempotencyKey ?? null,
|
||||
createdAtMs: run.createdAtMs,
|
||||
startedAtMs: run.startedAtMs ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function eventRow(event) {
|
||||
return {
|
||||
id: event.id,
|
||||
sequence: event.sequence,
|
||||
type: event.type,
|
||||
dedupeKey: event.dedupeKey ?? null,
|
||||
actorType: event.actorType,
|
||||
actorId: event.actorId ?? null,
|
||||
stepRunId: event.stepRunId ?? null,
|
||||
payload: event.payload,
|
||||
createdAtMs: event.createdAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
function stepEvidenceRow(bundle, stepKey, currentStepRun) {
|
||||
const mutation = bundle.stepMutations.find(
|
||||
({ stepRun }) => stepRun.stepKey === stepKey,
|
||||
);
|
||||
const step = bundle.plan.steps.find(
|
||||
(candidate) => candidate.stepKey === stepKey,
|
||||
);
|
||||
assert.ok(mutation);
|
||||
assert.ok(step);
|
||||
return {
|
||||
stepRunId: mutation.stepRun.id,
|
||||
taskId: step.taskId,
|
||||
taskDefinitionRef: step.taskDefinitionRef,
|
||||
taskDefinitionDigest: step.taskDefinitionDigest,
|
||||
needsJson: step.needs,
|
||||
initialStatus: step.initialStatus,
|
||||
mutationId: mutation.mutationId,
|
||||
eventId: mutation.event.id,
|
||||
currentStepKey: (currentStepRun ?? mutation.stepRun).stepKey,
|
||||
currentKind: (currentStepRun ?? mutation.stepRun).kind,
|
||||
currentDefinitionRef: (currentStepRun ?? mutation.stepRun).definitionRef,
|
||||
currentDefinitionDigest:
|
||||
(currentStepRun ?? mutation.stepRun).definitionDigest,
|
||||
currentRequired: (currentStepRun ?? mutation.stepRun).required,
|
||||
currentStatus: (currentStepRun ?? mutation.stepRun).status,
|
||||
currentVersion: (currentStepRun ?? mutation.stepRun).version,
|
||||
currentLastMutationId: (currentStepRun ?? mutation.stepRun).lastMutationId,
|
||||
currentStepRunDigest: (currentStepRun ?? mutation.stepRun).stepRunDigest,
|
||||
currentStepRunJson: currentStepRun ?? mutation.stepRun,
|
||||
mutationDigest: mutation.mutationDigest,
|
||||
eventSequence: mutation.event.sequence,
|
||||
runVersion: mutation.expectedRunVersion + 1,
|
||||
initialStepRunDigest: mutation.stepRun.stepRunDigest,
|
||||
initialStepRunJson: mutation.stepRun,
|
||||
};
|
||||
}
|
||||
|
||||
test('admits the complete Workflow evidence in one SERIALIZABLE transaction', async () => {
|
||||
const value = fixture();
|
||||
const queries = [];
|
||||
let released = false;
|
||||
const client = {
|
||||
async query(text, values) {
|
||||
queries.push({ text, values });
|
||||
if (
|
||||
text.includes('plugin_package_workflow_admissions') &&
|
||||
text.includes('WHERE plan_id')
|
||||
) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text.includes('plugin_package_workflow_admission_snapshot')) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
publicationJson: value.publication,
|
||||
revisionJson: value.revision,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
return { rows: [] };
|
||||
},
|
||||
release() {
|
||||
released = true;
|
||||
},
|
||||
};
|
||||
const repository = new PostgresPluginPackageWorkflowAdmissionRepository(
|
||||
poolWithClient(client),
|
||||
);
|
||||
|
||||
const result = await repository.admit(value.plan);
|
||||
assert.equal(result.status, 'created');
|
||||
assert.equal(result.receipt.finalRunVersion, 3);
|
||||
assert.equal(queries[0].text, 'BEGIN ISOLATION LEVEL SERIALIZABLE');
|
||||
assert.equal(queries.at(-1).text, 'COMMIT');
|
||||
assert.equal(released, true);
|
||||
assert.equal(
|
||||
queries.some(
|
||||
({ text }) =>
|
||||
text.includes('plugin_package_workflow_admissions') &&
|
||||
text.includes('FOR SHARE'),
|
||||
),
|
||||
false,
|
||||
'append-only replay must not require UPDATE authority',
|
||||
);
|
||||
assert.equal(
|
||||
queries.filter(({ text }) => /INSERT INTO "ql3"\."runs"/.test(text)).length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
queries.filter(({ text }) => /INSERT INTO "ql3"\."step_runs"/.test(text))
|
||||
.length,
|
||||
2,
|
||||
);
|
||||
assert.equal(
|
||||
queries.filter(({ text }) => /INSERT INTO "ql3"\."run_events"/.test(text))
|
||||
.length,
|
||||
3,
|
||||
);
|
||||
assert.equal(
|
||||
queries.filter(({ text }) =>
|
||||
/INSERT INTO "ql3"\."step_run_mutations"/.test(text),
|
||||
).length,
|
||||
2,
|
||||
);
|
||||
assert.equal(
|
||||
queries.filter(({ text }) =>
|
||||
/INSERT INTO "ql3"\."plugin_package_workflow_admissions"/.test(text),
|
||||
).length,
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
test('exactly replays durable admission after current StepRun progression', async () => {
|
||||
const value = fixture('postgres-workflow-replay');
|
||||
const bundle = createPluginPackageWorkflowAdmissionBundle(value.plan);
|
||||
const collect = bundle.stepMutations.find(
|
||||
({ stepRun }) => stepRun.stepKey === 'collect',
|
||||
).stepRun;
|
||||
const running = transitionStepRunMutation(
|
||||
collect,
|
||||
{
|
||||
expectedVersion: collect.version,
|
||||
expectedDigest: collect.stepRunDigest,
|
||||
mutationId: 'postgres-workflow-progress-running',
|
||||
to: 'running',
|
||||
atMs: 4_000,
|
||||
},
|
||||
{
|
||||
expectedRunVersion: bundle.run.version,
|
||||
expectedRunEventSequence: bundle.run.eventSequence,
|
||||
eventId: 'postgres-workflow-running-event',
|
||||
dedupeKey: 'postgres-workflow-running-event',
|
||||
actor: { type: 'executor' },
|
||||
},
|
||||
);
|
||||
const succeeded = transitionStepRunMutation(
|
||||
running.stepRun,
|
||||
{
|
||||
expectedVersion: running.stepRun.version,
|
||||
expectedDigest: running.stepRun.stepRunDigest,
|
||||
mutationId: 'postgres-workflow-progress-success',
|
||||
to: 'succeeded',
|
||||
atMs: 5_000,
|
||||
},
|
||||
{
|
||||
expectedRunVersion: bundle.run.version + 1,
|
||||
expectedRunEventSequence: bundle.run.eventSequence + 1,
|
||||
eventId: 'postgres-workflow-success-event',
|
||||
dedupeKey: 'postgres-workflow-success-event',
|
||||
actor: { type: 'executor' },
|
||||
},
|
||||
);
|
||||
const queries = [];
|
||||
const client = {
|
||||
async query(text, values) {
|
||||
queries.push({ text, values });
|
||||
if (
|
||||
text.includes('plugin_package_workflow_admissions') &&
|
||||
text.includes('WHERE plan_id')
|
||||
) {
|
||||
return {
|
||||
rows: [admissionRow(bundle)],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."runs" WHERE id')) {
|
||||
return {
|
||||
rows: [
|
||||
runRow({
|
||||
...bundle.run,
|
||||
version: bundle.run.version + 2,
|
||||
eventSequence: bundle.run.eventSequence + 2,
|
||||
}),
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."run_events"')) {
|
||||
return {
|
||||
rows: [
|
||||
eventRow(bundle.admissionEvent),
|
||||
...bundle.stepMutations.map(({ event }) => eventRow(event)),
|
||||
],
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes('FROM "ql3"."plugin_package_workflow_admission_steps"')
|
||||
) {
|
||||
return {
|
||||
rows: [
|
||||
stepEvidenceRow(
|
||||
bundle,
|
||||
values[1],
|
||||
values[1] === 'collect' ? succeeded.stepRun : undefined,
|
||||
),
|
||||
],
|
||||
};
|
||||
}
|
||||
return { rows: [] };
|
||||
},
|
||||
release() {},
|
||||
};
|
||||
const repository = new PostgresPluginPackageWorkflowAdmissionRepository(
|
||||
poolWithClient(client),
|
||||
);
|
||||
|
||||
assert.deepEqual(await repository.findPlanByPlanId(value.plan.planId), value.plan);
|
||||
assert.deepEqual(await repository.admit(value.plan), {
|
||||
status: 'existing',
|
||||
receipt: bundle.receipt,
|
||||
});
|
||||
assert.equal(
|
||||
queries.some(({ text }) =>
|
||||
text.includes('plugin_package_workflow_admission_snapshot'),
|
||||
),
|
||||
false,
|
||||
);
|
||||
const initialEventsQuery = queries.find(({ text }) =>
|
||||
text.includes('FROM "ql3"."run_events"'),
|
||||
);
|
||||
assert.match(initialEventsQuery.text, /sequence <= \$2/);
|
||||
assert.deepEqual(initialEventsQuery.values, [
|
||||
value.plan.runId,
|
||||
bundle.receipt.finalRunEventSequence,
|
||||
]);
|
||||
assert.equal(queries.at(-1).text, 'COMMIT');
|
||||
});
|
||||
|
||||
test('rolls back without Run evidence when the database guard denies admission', async () => {
|
||||
const value = fixture('postgres-workflow-denied');
|
||||
const queries = [];
|
||||
const client = {
|
||||
async query(text, values) {
|
||||
queries.push({ text, values });
|
||||
if (
|
||||
text.includes('plugin_package_workflow_admissions') &&
|
||||
text.includes('WHERE plan_id')
|
||||
) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text.includes('plugin_package_workflow_admission_snapshot')) {
|
||||
return { rows: [] };
|
||||
}
|
||||
return { rows: [] };
|
||||
},
|
||||
release() {},
|
||||
};
|
||||
const repository = new PostgresPluginPackageWorkflowAdmissionRepository(
|
||||
poolWithClient(client),
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
repository.admit(value.plan),
|
||||
PluginPackageWorkflowAdmissionNotAllowedError,
|
||||
);
|
||||
assert.equal(queries.at(-1).text, 'ROLLBACK');
|
||||
assert.equal(
|
||||
queries.some(({ text }) => /INSERT INTO "ql3"\."runs"/.test(text)),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('surfaces serialization failure as bounded retryable unavailability', async () => {
|
||||
const value = fixture('postgres-workflow-serialization');
|
||||
const serializationFailure = Object.assign(
|
||||
new Error('could not serialize access'),
|
||||
{ code: '40001' },
|
||||
);
|
||||
const client = {
|
||||
async query(text) {
|
||||
if (
|
||||
text.includes('plugin_package_workflow_admissions') &&
|
||||
text.includes('WHERE plan_id')
|
||||
) {
|
||||
throw serializationFailure;
|
||||
}
|
||||
return { rows: [] };
|
||||
},
|
||||
release() {},
|
||||
};
|
||||
const repository = new PostgresPluginPackageWorkflowAdmissionRepository(
|
||||
poolWithClient(client),
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
repository.admit(value.plan),
|
||||
PluginPackageWorkflowAdmissionUnavailableError,
|
||||
);
|
||||
});
|
||||
|
||||
test('publishes Workflow admission only through its explicit cluster subpath', () => {
|
||||
const authority = require('@qinglong/cluster-postgres/plugin-package-workflow-admission');
|
||||
const root = require('../dist');
|
||||
const runtime = require('@qinglong/cluster-postgres/runtime');
|
||||
assert.equal(
|
||||
authority.PostgresPluginPackageWorkflowAdmissionRepository,
|
||||
PostgresPluginPackageWorkflowAdmissionRepository,
|
||||
);
|
||||
assert.equal(
|
||||
root.PostgresPluginPackageWorkflowAdmissionRepository,
|
||||
undefined,
|
||||
);
|
||||
assert.equal(
|
||||
runtime.PostgresPluginPackageWorkflowAdmissionRepository,
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,497 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
createInitialPluginPackageAutomationPublication,
|
||||
} = require('@qinglong/runtime-core/plugin-package-automation-publication');
|
||||
const {
|
||||
createPluginPackageWorkflowAdmissionBundle,
|
||||
createPluginPackageWorkflowExecutionPlan,
|
||||
} = require('@qinglong/runtime-core/plugin-package-workflow-execution-plan');
|
||||
const {
|
||||
InvalidPluginPackageWorkflowFrontierError,
|
||||
PluginPackageWorkflowFrontierUnavailableError,
|
||||
resolvePluginPackageWorkflowFrontier,
|
||||
} = require('@qinglong/runtime-core/plugin-package-workflow-frontier');
|
||||
const {
|
||||
transitionStepRunMutation,
|
||||
} = require('@qinglong/runtime-core/step-run');
|
||||
const {
|
||||
pluginPackageTaskReconciliationFixture,
|
||||
} = require('../../../test/contracts/pluginPackageTaskReconciliationRepositoryContract.cjs');
|
||||
const {
|
||||
PostgresPluginPackageWorkflowFrontierRepository,
|
||||
} = require('../dist/plugin-package/workflow/pluginPackageWorkflowFrontierRepository');
|
||||
|
||||
function fixture(namespace) {
|
||||
const value = pluginPackageTaskReconciliationFixture(namespace, {
|
||||
workflows: [
|
||||
{
|
||||
schema: 'qinglong/plugin-package-workflow-resource@v1',
|
||||
id: 'daily',
|
||||
name: 'Daily workflow',
|
||||
enabled: true,
|
||||
steps: [
|
||||
{ id: 'collect', task: 'alpha', needs: [] },
|
||||
{ id: 'summarize', task: 'beta', needs: ['collect'] },
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
const publication = createInitialPluginPackageAutomationPublication(
|
||||
value.revision,
|
||||
value.registry,
|
||||
2_000,
|
||||
);
|
||||
const plan = createPluginPackageWorkflowExecutionPlan({
|
||||
planId: `wf-plan-${namespace}`,
|
||||
runId: `wf-run-${namespace}`,
|
||||
workflowId: 'daily',
|
||||
stepRunIds: {
|
||||
collect: `wf-collect-${namespace}`,
|
||||
summarize: `wf-summary-${namespace}`,
|
||||
},
|
||||
publication,
|
||||
revision: value.revision,
|
||||
taskSpecSemanticRegistry: value.registry,
|
||||
plannedAtMs: 3_000,
|
||||
});
|
||||
const bundle = createPluginPackageWorkflowAdmissionBundle(plan);
|
||||
return { ...value, publication, plan, bundle };
|
||||
}
|
||||
|
||||
function transition(stepRun, to, runVersion, atMs) {
|
||||
return transitionStepRunMutation(
|
||||
stepRun,
|
||||
{
|
||||
expectedVersion: stepRun.version,
|
||||
expectedDigest: stepRun.stepRunDigest,
|
||||
mutationId: `${stepRun.stepKey}-${to}-${stepRun.version}`,
|
||||
to,
|
||||
atMs,
|
||||
...(to === 'failed' ? { resultCode: 'task_failed' } : {}),
|
||||
},
|
||||
{
|
||||
expectedRunVersion: runVersion,
|
||||
expectedRunEventSequence: runVersion,
|
||||
eventId: `${stepRun.stepKey}-${to}-event-${stepRun.version}`,
|
||||
dedupeKey: `${stepRun.stepKey}-${to}-event-${stepRun.version}`,
|
||||
actor: { type: 'executor' },
|
||||
},
|
||||
).stepRun;
|
||||
}
|
||||
|
||||
function progressedSnapshot(value, collectTerminal = 'succeeded') {
|
||||
const collectInitial = value.bundle.stepMutations.find(
|
||||
({ stepRun }) => stepRun.stepKey === 'collect',
|
||||
).stepRun;
|
||||
const summarize = value.bundle.stepMutations.find(
|
||||
({ stepRun }) => stepRun.stepKey === 'summarize',
|
||||
).stepRun;
|
||||
const collectRunning = transition(collectInitial, 'running', 3, 4_000);
|
||||
const collect = transition(
|
||||
collectRunning,
|
||||
collectTerminal,
|
||||
4,
|
||||
5_000,
|
||||
);
|
||||
return {
|
||||
run: Object.freeze({
|
||||
...value.bundle.run,
|
||||
version: 5,
|
||||
eventSequence: 5,
|
||||
}),
|
||||
stepRuns: Object.freeze([collect, summarize]),
|
||||
};
|
||||
}
|
||||
|
||||
function runRow(run) {
|
||||
return {
|
||||
id: run.id,
|
||||
projectId: run.projectId,
|
||||
taskId: run.taskId,
|
||||
taskRevision: run.taskRevision,
|
||||
taskName: run.taskName ?? null,
|
||||
taskSnapshotRef: run.taskSnapshotRef ?? null,
|
||||
legacyCronId: run.legacyCronId ?? null,
|
||||
parentRunId: run.parentRunId ?? null,
|
||||
retryOfRunId: run.retryOfRunId ?? null,
|
||||
triggerId: run.triggerId ?? null,
|
||||
triggerType: run.triggerType,
|
||||
executionOrigin: run.executionOrigin,
|
||||
executionOwner: run.executionOwner,
|
||||
triggeredBy: run.triggeredBy ?? null,
|
||||
requestId: run.requestId ?? null,
|
||||
scheduledForMs: run.scheduledForMs ?? null,
|
||||
status: run.status,
|
||||
version: run.version,
|
||||
eventSequence: run.eventSequence,
|
||||
priority: run.priority,
|
||||
idempotencyKey: run.idempotencyKey ?? null,
|
||||
inputRef: run.inputRef ?? null,
|
||||
outputRef: run.outputRef ?? null,
|
||||
createdAtMs: run.createdAtMs,
|
||||
queuedAtMs: run.queuedAtMs ?? null,
|
||||
startedAtMs: run.startedAtMs ?? null,
|
||||
finishedAtMs: run.finishedAtMs ?? null,
|
||||
cancelRequestedAtMs: run.cancelRequestedAtMs ?? null,
|
||||
cancelReason: run.cancelReason ?? null,
|
||||
errorCode: run.errorCode ?? null,
|
||||
errorSummary: run.errorSummary ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function stepRunRow(stepRun) {
|
||||
return {
|
||||
id: stepRun.id,
|
||||
runId: stepRun.runId,
|
||||
parentStepRunId: stepRun.parentStepRunId,
|
||||
stepKey: stepRun.stepKey,
|
||||
kind: stepRun.kind,
|
||||
definitionRef: stepRun.definitionRef,
|
||||
definitionDigest: stepRun.definitionDigest,
|
||||
required: stepRun.required,
|
||||
status: stepRun.status,
|
||||
version: stepRun.version,
|
||||
attemptCount: stepRun.attemptCount,
|
||||
inputRef: stepRun.inputRef ?? null,
|
||||
outputRef: stepRun.outputRef ?? null,
|
||||
approvalRequestId: stepRun.approvalRequestId ?? null,
|
||||
readyAtMs: stepRun.readyAtMs ?? null,
|
||||
startedAtMs: stepRun.startedAtMs ?? null,
|
||||
finishedAtMs: stepRun.finishedAtMs ?? null,
|
||||
resultCode: stepRun.resultCode ?? null,
|
||||
errorSummary: stepRun.errorSummary ?? null,
|
||||
createdAtMs: stepRun.createdAtMs,
|
||||
updatedAtMs: stepRun.updatedAtMs,
|
||||
lastMutationId: stepRun.lastMutationId,
|
||||
stepRunDigest: stepRun.stepRunDigest,
|
||||
stepRunJson: stepRun,
|
||||
};
|
||||
}
|
||||
|
||||
function transactionPool(value, snapshot, options = {}) {
|
||||
const queries = [];
|
||||
let connections = 0;
|
||||
let releases = 0;
|
||||
const pool = {
|
||||
async query(text, values) {
|
||||
queries.push({ connection: 0, text, values });
|
||||
if (text.includes('plugin_package_workflow_admissions')) {
|
||||
return {
|
||||
rows: options.candidateRows ?? [
|
||||
{
|
||||
runId: value.plan.runId,
|
||||
planDigest: value.plan.planDigest,
|
||||
admittedAtMs: 3_000,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
return { rows: [] };
|
||||
},
|
||||
async connect() {
|
||||
connections += 1;
|
||||
const connection = connections;
|
||||
let serializationFailureRemaining =
|
||||
connection <= (options.serializationFailures ?? 0) ? 1 : 0;
|
||||
return {
|
||||
async query(text, values) {
|
||||
queries.push({ connection, text, values });
|
||||
if (
|
||||
serializationFailureRemaining > 0 &&
|
||||
text.includes('plugin_package_workflow_admissions')
|
||||
) {
|
||||
serializationFailureRemaining -= 1;
|
||||
throw Object.assign(new Error('could not serialize access'), {
|
||||
code: '40001',
|
||||
});
|
||||
}
|
||||
if (
|
||||
text.includes('plugin_package_workflow_admissions') &&
|
||||
text.includes('plan_json')
|
||||
) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
planDigest: value.plan.planDigest,
|
||||
planJson: value.plan,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."runs"')) {
|
||||
return { rows: [runRow(snapshot.run)] };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."step_runs"')) {
|
||||
return { rows: snapshot.stepRuns.map(stepRunRow) };
|
||||
}
|
||||
if (text.includes('transaction_timestamp()')) {
|
||||
return { rows: [{ observedAtMs: options.observedAtMs ?? 6_000 }] };
|
||||
}
|
||||
return { rows: [], rowCount: 1 };
|
||||
},
|
||||
release() {
|
||||
releases += 1;
|
||||
queries.push({ connection, text: 'RELEASE' });
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
return {
|
||||
pool,
|
||||
queries,
|
||||
get connections() {
|
||||
return connections;
|
||||
},
|
||||
get releases() {
|
||||
return releases;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('bounds and keyset-pages only actionable Workflow candidates', async () => {
|
||||
const value = fixture('pg-frontier-page');
|
||||
const snapshot = progressedSnapshot(value);
|
||||
const secondDigest = 'f'.repeat(64);
|
||||
const db = transactionPool(value, snapshot, {
|
||||
candidateRows: [
|
||||
{
|
||||
runId: value.plan.runId,
|
||||
planDigest: value.plan.planDigest,
|
||||
admittedAtMs: 3_000,
|
||||
},
|
||||
{
|
||||
runId: 'wf-run-next',
|
||||
planDigest: secondDigest,
|
||||
admittedAtMs: 3_001,
|
||||
},
|
||||
],
|
||||
});
|
||||
const repository =
|
||||
new PostgresPluginPackageWorkflowFrontierRepository(db.pool);
|
||||
|
||||
assert.deepEqual(
|
||||
await repository.listCandidates({
|
||||
limit: 1,
|
||||
after: {
|
||||
admittedAtMs: 2_999,
|
||||
planDigest: '0'.repeat(64),
|
||||
},
|
||||
}),
|
||||
{
|
||||
candidates: [
|
||||
{
|
||||
runId: value.plan.runId,
|
||||
planDigest: value.plan.planDigest,
|
||||
admittedAtMs: 3_000,
|
||||
},
|
||||
],
|
||||
truncated: true,
|
||||
next: {
|
||||
admittedAtMs: 3_000,
|
||||
planDigest: value.plan.planDigest,
|
||||
},
|
||||
},
|
||||
);
|
||||
const candidateQuery = db.queries[0];
|
||||
assert.deepEqual(candidateQuery.values, [2_999, '0'.repeat(64), 2]);
|
||||
assert.match(candidateQuery.text, /jsonb_array_elements_text/);
|
||||
assert.match(
|
||||
candidateQuery.text,
|
||||
/run\.cancel_requested_at_ms IS NULL/,
|
||||
);
|
||||
assert.match(candidateQuery.text, /current\.status = 'pending'/);
|
||||
assert.match(candidateQuery.text, /dependency\.status IN/);
|
||||
assert.match(candidateQuery.text, /LIMIT \$3/);
|
||||
await assert.rejects(
|
||||
repository.listCandidates({ limit: 65 }),
|
||||
InvalidPluginPackageWorkflowFrontierError,
|
||||
);
|
||||
assert.equal(db.queries.length, 1);
|
||||
});
|
||||
|
||||
test('advances a ready dependency with locked rows and one aggregate Run CAS', async () => {
|
||||
const value = fixture('pg-frontier-advance');
|
||||
const snapshot = progressedSnapshot(value);
|
||||
const db = transactionPool(value, snapshot);
|
||||
const repository =
|
||||
new PostgresPluginPackageWorkflowFrontierRepository(db.pool);
|
||||
|
||||
const result = await repository.advance(value.plan.runId);
|
||||
assert.equal(result.status, 'advanced');
|
||||
assert.equal(result.stepMutationCount, 1);
|
||||
assert.equal(result.terminalStatus, null);
|
||||
assert.equal(result.runVersion, 6);
|
||||
assert.deepEqual(result.readyStepRunIds, [
|
||||
value.plan.steps.find(({ stepKey }) => stepKey === 'summarize').stepRunId,
|
||||
]);
|
||||
|
||||
const sql = db.queries.map(({ text }) => text);
|
||||
assert.equal(sql[0], 'BEGIN ISOLATION LEVEL SERIALIZABLE');
|
||||
assert.match(
|
||||
sql.find((text) => text.includes('FROM "ql3"."runs"')),
|
||||
/FOR UPDATE/,
|
||||
);
|
||||
assert.match(
|
||||
sql.find((text) => text.includes('FROM "ql3"."step_runs"')),
|
||||
/FOR UPDATE/,
|
||||
);
|
||||
assert.equal(
|
||||
sql.filter((text) => text.includes('UPDATE "ql3"."runs"')).length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
sql.filter((text) => text.includes('UPDATE "ql3"."step_runs"')).length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
sql.filter((text) => text.includes('INSERT INTO "ql3"."run_events"'))
|
||||
.length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
sql.filter((text) =>
|
||||
text.includes('INSERT INTO "ql3"."step_run_mutations"'),
|
||||
).length,
|
||||
1,
|
||||
);
|
||||
assert.equal(sql.at(-2), 'COMMIT');
|
||||
assert.equal(sql.at(-1), 'RELEASE');
|
||||
});
|
||||
|
||||
test('skips blocked work and terminalizes the aggregate in one transaction', async () => {
|
||||
const value = fixture('pg-frontier-terminal');
|
||||
const snapshot = progressedSnapshot(value, 'failed');
|
||||
const db = transactionPool(value, snapshot);
|
||||
const repository =
|
||||
new PostgresPluginPackageWorkflowFrontierRepository(db.pool);
|
||||
|
||||
const result = await repository.advance(value.plan.runId);
|
||||
assert.equal(result.status, 'terminal');
|
||||
assert.equal(result.stepMutationCount, 1);
|
||||
assert.equal(result.terminalStatus, 'failed');
|
||||
assert.equal(result.runVersion, 7);
|
||||
const runUpdate = db.queries.find(({ text }) =>
|
||||
text.includes('UPDATE "ql3"."runs"'),
|
||||
);
|
||||
assert.deepEqual(runUpdate.values.slice(0, 4), [
|
||||
'failed',
|
||||
2,
|
||||
6_000,
|
||||
'workflow_step_failed',
|
||||
]);
|
||||
assert.equal(
|
||||
db.queries.filter(({ text }) =>
|
||||
text.includes('INSERT INTO "ql3"."run_events"'),
|
||||
).length,
|
||||
2,
|
||||
);
|
||||
assert.equal(db.queries.at(-2).text, 'COMMIT');
|
||||
});
|
||||
|
||||
test('retries a serialization race with a fresh client and converges', async () => {
|
||||
const value = fixture('pg-frontier-retry');
|
||||
const snapshot = progressedSnapshot(value);
|
||||
const db = transactionPool(value, snapshot, {
|
||||
serializationFailures: 1,
|
||||
});
|
||||
const repository =
|
||||
new PostgresPluginPackageWorkflowFrontierRepository(db.pool);
|
||||
|
||||
assert.equal((await repository.advance(value.plan.runId)).status, 'advanced');
|
||||
assert.equal(db.connections, 2);
|
||||
assert.equal(db.releases, 2);
|
||||
assert.equal(
|
||||
db.queries.filter(({ text }) => text === 'ROLLBACK').length,
|
||||
1,
|
||||
);
|
||||
|
||||
const unavailableDb = transactionPool(value, snapshot, {
|
||||
serializationFailures: 99,
|
||||
});
|
||||
await assert.rejects(
|
||||
new PostgresPluginPackageWorkflowFrontierRepository(
|
||||
unavailableDb.pool,
|
||||
).advance(value.plan.runId),
|
||||
PluginPackageWorkflowFrontierUnavailableError,
|
||||
);
|
||||
assert.equal(unavailableDb.connections, 3);
|
||||
assert.equal(unavailableDb.releases, 3);
|
||||
});
|
||||
|
||||
test('returns settled for an already terminal aggregate without writes', async () => {
|
||||
const value = fixture('pg-frontier-settled');
|
||||
const progressed = progressedSnapshot(value);
|
||||
const frontier = resolvePluginPackageWorkflowFrontier({
|
||||
plan: value.plan,
|
||||
run: progressed.run,
|
||||
stepRuns: progressed.stepRuns,
|
||||
observedAtMs: 6_000,
|
||||
});
|
||||
const summarizeReady = frontier.stepMutations[0].stepRun;
|
||||
const summarizeRunning = transition(summarizeReady, 'running', 6, 7_000);
|
||||
const summarizeSucceeded = transition(
|
||||
summarizeRunning,
|
||||
'succeeded',
|
||||
7,
|
||||
8_000,
|
||||
);
|
||||
const terminalResolution = resolvePluginPackageWorkflowFrontier({
|
||||
plan: value.plan,
|
||||
run: {
|
||||
...progressed.run,
|
||||
version: 8,
|
||||
eventSequence: 8,
|
||||
},
|
||||
stepRuns: [progressed.stepRuns[0], summarizeSucceeded],
|
||||
observedAtMs: 9_000,
|
||||
});
|
||||
const terminal = terminalResolution.terminalTransition;
|
||||
assert.ok(terminal);
|
||||
const snapshot = {
|
||||
run: Object.freeze({
|
||||
...progressed.run,
|
||||
status: terminal.status,
|
||||
version: 9,
|
||||
eventSequence: 9,
|
||||
finishedAtMs: terminal.finishedAtMs,
|
||||
errorCode: terminal.errorCode ?? undefined,
|
||||
}),
|
||||
stepRuns: Object.freeze([
|
||||
progressed.stepRuns[0],
|
||||
summarizeSucceeded,
|
||||
]),
|
||||
};
|
||||
const db = transactionPool(value, snapshot, { observedAtMs: 9_000 });
|
||||
|
||||
const result =
|
||||
await new PostgresPluginPackageWorkflowFrontierRepository(
|
||||
db.pool,
|
||||
).advance(value.plan.runId);
|
||||
assert.equal(result.status, 'settled');
|
||||
assert.equal(result.terminalStatus, 'succeeded');
|
||||
assert.equal(
|
||||
db.queries.some(({ text }) => text.includes('UPDATE "ql3"')),
|
||||
false,
|
||||
);
|
||||
assert.equal(db.queries.at(-2).text, 'COMMIT');
|
||||
});
|
||||
|
||||
test('publishes Workflow frontier only through its explicit cluster subpath', () => {
|
||||
const authority = require('@qinglong/cluster-postgres/plugin-package-workflow-frontier');
|
||||
const root = require('../dist');
|
||||
const runtime = require('@qinglong/cluster-postgres/runtime');
|
||||
assert.equal(
|
||||
authority.PostgresPluginPackageWorkflowFrontierRepository,
|
||||
PostgresPluginPackageWorkflowFrontierRepository,
|
||||
);
|
||||
assert.equal(
|
||||
root.PostgresPluginPackageWorkflowFrontierRepository,
|
||||
undefined,
|
||||
);
|
||||
assert.equal(
|
||||
runtime.PostgresPluginPackageWorkflowFrontierRepository,
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
+501
@@ -0,0 +1,501 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
compileClusterCommandTaskDefinition,
|
||||
} = require('@qinglong/runtime-core/cluster-execution-revision');
|
||||
const {
|
||||
createInitialPluginPackageAutomationPublication,
|
||||
} = require('@qinglong/runtime-core/plugin-package-automation-publication');
|
||||
const {
|
||||
planPluginPackageTaskReconciliation,
|
||||
pluginPackageTaskReconciliationTaskIds,
|
||||
} = require('@qinglong/runtime-core/plugin-package-task-reconciliation');
|
||||
const {
|
||||
createPluginPackageWorkflowAdmissionBundle,
|
||||
createPluginPackageWorkflowExecutionPlan,
|
||||
} = require('@qinglong/runtime-core/plugin-package-workflow-execution-plan');
|
||||
const {
|
||||
InvalidPluginPackageWorkflowTaskAttemptAdmissionError,
|
||||
PluginPackageWorkflowTaskAttemptAdmissionConflictError,
|
||||
} = require('@qinglong/runtime-core/plugin-package-workflow-task-attempt-admission');
|
||||
const {
|
||||
pluginPackageTaskReconciliationFixture,
|
||||
} = require('../../../test/contracts/pluginPackageTaskReconciliationRepositoryContract.cjs');
|
||||
const {
|
||||
PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository,
|
||||
} = require('../dist/plugin-package/workflow/pluginPackageWorkflowTaskAttemptAdmissionRepository');
|
||||
|
||||
function fixture(namespace) {
|
||||
const value = pluginPackageTaskReconciliationFixture(namespace, {
|
||||
workflows: [
|
||||
{
|
||||
schema: 'qinglong/plugin-package-workflow-resource@v1',
|
||||
id: 'daily',
|
||||
name: 'Daily workflow',
|
||||
enabled: true,
|
||||
steps: [
|
||||
{ id: 'collect', task: 'alpha', needs: [] },
|
||||
{ id: 'summarize', task: 'beta', needs: ['collect'] },
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
const publication = createInitialPluginPackageAutomationPublication(
|
||||
value.revision,
|
||||
value.registry,
|
||||
2_000,
|
||||
);
|
||||
const plan = createPluginPackageWorkflowExecutionPlan({
|
||||
planId: `wf-attempt-plan-${namespace}`,
|
||||
runId: `wf-attempt-run-${namespace}`,
|
||||
workflowId: 'daily',
|
||||
stepRunIds: {
|
||||
collect: `wf-attempt-collect-${namespace}`,
|
||||
summarize: `wf-attempt-summary-${namespace}`,
|
||||
},
|
||||
publication,
|
||||
revision: value.revision,
|
||||
taskSpecSemanticRegistry: value.registry,
|
||||
plannedAtMs: 3_000,
|
||||
});
|
||||
const admission = createPluginPackageWorkflowAdmissionBundle(plan);
|
||||
const reconciliationPlan = planPluginPackageTaskReconciliation({
|
||||
revision: value.revision,
|
||||
previousReceipt: null,
|
||||
facts: pluginPackageTaskReconciliationTaskIds(
|
||||
value.revision,
|
||||
null,
|
||||
value.registry,
|
||||
).map((taskId) => ({
|
||||
taskId,
|
||||
packageName: null,
|
||||
current: null,
|
||||
})),
|
||||
committedAtMs: 2_500,
|
||||
taskSpecSemanticRegistry: value.registry,
|
||||
});
|
||||
const taskDefinition = reconciliationPlan.writes.find(
|
||||
({ definition }) =>
|
||||
definition.taskId === `pkg:${value.packageName}:alpha`,
|
||||
).definition;
|
||||
const execution = compileClusterCommandTaskDefinition(
|
||||
taskDefinition,
|
||||
value.registry,
|
||||
);
|
||||
const stepRun = admission.stepMutations.find(
|
||||
({ stepRun: candidate }) => candidate.stepKey === 'collect',
|
||||
).stepRun;
|
||||
return {
|
||||
...value,
|
||||
plan,
|
||||
admission,
|
||||
reconciliation: reconciliationPlan.receipt,
|
||||
execution,
|
||||
stepRun,
|
||||
};
|
||||
}
|
||||
|
||||
function runRow(run) {
|
||||
return {
|
||||
id: run.id,
|
||||
projectId: run.projectId,
|
||||
taskId: run.taskId,
|
||||
taskRevision: run.taskRevision,
|
||||
taskName: run.taskName ?? null,
|
||||
taskSnapshotRef: run.taskSnapshotRef ?? null,
|
||||
legacyCronId: run.legacyCronId ?? null,
|
||||
parentRunId: run.parentRunId ?? null,
|
||||
retryOfRunId: run.retryOfRunId ?? null,
|
||||
triggerId: run.triggerId ?? null,
|
||||
triggerType: run.triggerType,
|
||||
executionOrigin: run.executionOrigin,
|
||||
executionOwner: run.executionOwner,
|
||||
triggeredBy: run.triggeredBy ?? null,
|
||||
requestId: run.requestId ?? null,
|
||||
scheduledForMs: run.scheduledForMs ?? null,
|
||||
status: run.status,
|
||||
version: run.version,
|
||||
eventSequence: run.eventSequence,
|
||||
priority: run.priority,
|
||||
idempotencyKey: run.idempotencyKey ?? null,
|
||||
inputRef: run.inputRef ?? null,
|
||||
outputRef: run.outputRef ?? null,
|
||||
createdAtMs: run.createdAtMs,
|
||||
queuedAtMs: run.queuedAtMs ?? null,
|
||||
startedAtMs: run.startedAtMs ?? null,
|
||||
finishedAtMs: run.finishedAtMs ?? null,
|
||||
cancelRequestedAtMs: run.cancelRequestedAtMs ?? null,
|
||||
cancelReason: run.cancelReason ?? null,
|
||||
errorCode: run.errorCode ?? null,
|
||||
errorSummary: run.errorSummary ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function stepRunRow(stepRun) {
|
||||
return {
|
||||
id: stepRun.id,
|
||||
runId: stepRun.runId,
|
||||
parentStepRunId: stepRun.parentStepRunId ?? null,
|
||||
stepKey: stepRun.stepKey,
|
||||
kind: stepRun.kind,
|
||||
definitionRef: stepRun.definitionRef,
|
||||
definitionDigest: stepRun.definitionDigest,
|
||||
required: stepRun.required,
|
||||
status: stepRun.status,
|
||||
version: stepRun.version,
|
||||
attemptCount: stepRun.attemptCount,
|
||||
inputRef: stepRun.inputRef ?? null,
|
||||
outputRef: stepRun.outputRef ?? null,
|
||||
approvalRequestId: stepRun.approvalRequestId ?? null,
|
||||
readyAtMs: stepRun.readyAtMs ?? null,
|
||||
startedAtMs: stepRun.startedAtMs ?? null,
|
||||
finishedAtMs: stepRun.finishedAtMs ?? null,
|
||||
resultCode: stepRun.resultCode ?? null,
|
||||
errorSummary: stepRun.errorSummary ?? null,
|
||||
createdAtMs: stepRun.createdAtMs,
|
||||
updatedAtMs: stepRun.updatedAtMs,
|
||||
lastMutationId: stepRun.lastMutationId,
|
||||
stepRunDigest: stepRun.stepRunDigest,
|
||||
stepRunJson: stepRun,
|
||||
};
|
||||
}
|
||||
|
||||
function snapshotRow(value) {
|
||||
const execution = value.execution;
|
||||
return {
|
||||
planJson: value.plan,
|
||||
reconciliationJson: value.reconciliation,
|
||||
executionProjectId: execution.projectId,
|
||||
executionTaskId: execution.taskId,
|
||||
executionSourceRevision: execution.sourceRevision,
|
||||
executionTaskRevision: execution.taskRevision,
|
||||
executionSourceContentDigest: execution.sourceContentDigest,
|
||||
executionExecutorType: execution.executorType,
|
||||
executionPlanSchema: execution.planSchema,
|
||||
executionPlanJson: {
|
||||
command: execution.command,
|
||||
environment: execution.environment,
|
||||
...(execution.workingDirectory === undefined
|
||||
? {}
|
||||
: { workingDirectory: execution.workingDirectory }),
|
||||
...(execution.timeoutMs === undefined
|
||||
? {}
|
||||
: { timeoutMs: execution.timeoutMs }),
|
||||
...(execution.placement === undefined
|
||||
? {}
|
||||
: { placement: execution.placement }),
|
||||
},
|
||||
executionContentDigest: execution.contentDigest,
|
||||
executionCreatedAtMs: execution.createdAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
function mockPool(value, options = {}) {
|
||||
const queries = [];
|
||||
let connections = 0;
|
||||
let releases = 0;
|
||||
return {
|
||||
pool: {
|
||||
async connect() {
|
||||
connections += 1;
|
||||
const connection = connections;
|
||||
let retryFailure =
|
||||
connection <= (options.serializationFailures ?? 0);
|
||||
return {
|
||||
async query(text, values) {
|
||||
queries.push({ connection, text, values });
|
||||
if (
|
||||
retryFailure &&
|
||||
text.includes(
|
||||
'plugin_package_workflow_task_attempt_snapshot',
|
||||
)
|
||||
) {
|
||||
retryFailure = false;
|
||||
throw Object.assign(new Error('serialization failure'), {
|
||||
code: '40001',
|
||||
});
|
||||
}
|
||||
if (
|
||||
text.includes('FROM "ql3"."step_runs" AS current')
|
||||
) {
|
||||
return {
|
||||
rows: options.candidateRows ?? [
|
||||
{
|
||||
runId: value.plan.runId,
|
||||
stepRunId: value.stepRun.id,
|
||||
readyAtMs: value.stepRun.readyAtMs,
|
||||
planDigest: value.plan.planDigest,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."runs"')) {
|
||||
return {
|
||||
rows: [
|
||||
runRow(options.run ?? value.admission.run),
|
||||
],
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes('FROM "ql3"."step_runs"') &&
|
||||
text.includes('FOR UPDATE')
|
||||
) {
|
||||
return { rows: [stepRunRow(value.stepRun)] };
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'plugin_package_workflow_task_attempt_admissions',
|
||||
) &&
|
||||
text.includes('receipt_json') &&
|
||||
text.includes('WHERE run_id')
|
||||
) {
|
||||
return {
|
||||
rows: options.existingReceipt
|
||||
? [
|
||||
{
|
||||
receiptDigest:
|
||||
options.existingReceipt.receiptDigest,
|
||||
receiptJson: options.existingReceipt,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes(
|
||||
'plugin_package_workflow_task_attempt_snapshot',
|
||||
)
|
||||
) {
|
||||
return {
|
||||
rows: options.snapshotMissing
|
||||
? []
|
||||
: [snapshotRow(value)],
|
||||
};
|
||||
}
|
||||
if (text.includes('transaction_timestamp()')) {
|
||||
return {
|
||||
rows: [{ admittedAtMs: options.admittedAtMs ?? 4_000 }],
|
||||
};
|
||||
}
|
||||
if (text.includes('MAX(attempt)')) {
|
||||
return {
|
||||
rows: [
|
||||
{ attemptNumber: options.attemptNumber ?? 1 },
|
||||
],
|
||||
};
|
||||
}
|
||||
return { rows: [], rowCount: 1 };
|
||||
},
|
||||
release() {
|
||||
releases += 1;
|
||||
queries.push({ connection, text: 'RELEASE' });
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
queries,
|
||||
get connections() {
|
||||
return connections;
|
||||
},
|
||||
get releases() {
|
||||
return releases;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('bounds and keyset-pages only ready uncancelled Task attempts', async () => {
|
||||
const value = fixture('pg-wta-page');
|
||||
const db = mockPool(value, {
|
||||
candidateRows: [
|
||||
{
|
||||
runId: value.plan.runId,
|
||||
stepRunId: value.stepRun.id,
|
||||
readyAtMs: value.stepRun.readyAtMs,
|
||||
planDigest: value.plan.planDigest,
|
||||
},
|
||||
{
|
||||
runId: 'wf-attempt-run-next',
|
||||
stepRunId: 'wf-attempt-step-next',
|
||||
readyAtMs: value.stepRun.readyAtMs + 1,
|
||||
planDigest: 'f'.repeat(64),
|
||||
},
|
||||
],
|
||||
});
|
||||
const repository =
|
||||
new PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository(
|
||||
db.pool,
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
await repository.listCandidates({
|
||||
limit: 1,
|
||||
after: {
|
||||
readyAtMs: value.stepRun.readyAtMs - 1,
|
||||
stepRunId: 'previous-step',
|
||||
},
|
||||
}),
|
||||
{
|
||||
candidates: [
|
||||
{
|
||||
runId: value.plan.runId,
|
||||
stepRunId: value.stepRun.id,
|
||||
readyAtMs: value.stepRun.readyAtMs,
|
||||
planDigest: value.plan.planDigest,
|
||||
},
|
||||
],
|
||||
truncated: true,
|
||||
next: {
|
||||
readyAtMs: value.stepRun.readyAtMs,
|
||||
stepRunId: value.stepRun.id,
|
||||
},
|
||||
},
|
||||
);
|
||||
const query = db.queries[0];
|
||||
assert.deepEqual(query.values, [
|
||||
'previous-step',
|
||||
value.stepRun.readyAtMs - 1,
|
||||
2,
|
||||
]);
|
||||
assert.match(query.text, /run\.cancel_requested_at_ms IS NULL/);
|
||||
assert.match(query.text, /current\.status = 'ready'/);
|
||||
assert.match(query.text, /NOT EXISTS/);
|
||||
assert.match(query.text, /LIMIT \$3/);
|
||||
await assert.rejects(
|
||||
repository.listCandidates({ limit: 65 }),
|
||||
InvalidPluginPackageWorkflowTaskAttemptAdmissionError,
|
||||
);
|
||||
});
|
||||
|
||||
test('atomically admits the exact cluster execution and replays the epoch', async () => {
|
||||
const value = fixture('pg-wta-create');
|
||||
const db = mockPool(value);
|
||||
const repository =
|
||||
new PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository(
|
||||
db.pool,
|
||||
);
|
||||
|
||||
const created = await repository.admit(
|
||||
value.plan.runId,
|
||||
value.stepRun.id,
|
||||
);
|
||||
assert.equal(created.status, 'created');
|
||||
assert.equal(created.receipt.resourceTaskId, 'alpha');
|
||||
assert.equal(
|
||||
created.receipt.taskId,
|
||||
`pkg:${value.packageName}:alpha`,
|
||||
);
|
||||
assert.equal(created.receipt.executorType, 'remote_worker');
|
||||
assert.equal(created.receipt.executionDigest, value.execution.contentDigest);
|
||||
assert.equal(created.receipt.attemptNumber, 1);
|
||||
assert.equal(created.receipt.runVersion, value.admission.run.version + 1);
|
||||
|
||||
const sql = db.queries.map(({ text }) => text);
|
||||
assert.equal(sql[0], 'BEGIN ISOLATION LEVEL SERIALIZABLE');
|
||||
assert.match(
|
||||
sql.find((text) => text.includes('FROM "ql3"."runs"')),
|
||||
/FOR UPDATE/,
|
||||
);
|
||||
assert.match(
|
||||
sql.find(
|
||||
(text) =>
|
||||
text.includes('FROM "ql3"."step_runs"') &&
|
||||
text.includes('FOR UPDATE'),
|
||||
),
|
||||
/FOR UPDATE/,
|
||||
);
|
||||
assert.ok(
|
||||
sql.some((text) =>
|
||||
text.includes('plugin_package_workflow_task_attempt_snapshot'),
|
||||
),
|
||||
);
|
||||
assert.equal(
|
||||
sql.filter((text) => text.includes('UPDATE "ql3"."runs"')).length,
|
||||
1,
|
||||
);
|
||||
assert.ok(
|
||||
sql.some((text) => text.includes('INSERT INTO "ql3"."run_attempts"')),
|
||||
);
|
||||
assert.ok(
|
||||
sql.some((text) => text.includes('INSERT INTO "ql3"."run_events"')),
|
||||
);
|
||||
assert.ok(
|
||||
sql.some((text) =>
|
||||
text.includes(
|
||||
'"ql3"."plugin_package_workflow_task_attempt_admissions"',
|
||||
),
|
||||
),
|
||||
);
|
||||
assert.equal(sql.at(-2), 'COMMIT');
|
||||
assert.equal(sql.at(-1), 'RELEASE');
|
||||
|
||||
const replayDb = mockPool(value, {
|
||||
existingReceipt: created.receipt,
|
||||
run: {
|
||||
...value.admission.run,
|
||||
version: created.receipt.runVersion,
|
||||
eventSequence: created.receipt.runEventSequence,
|
||||
},
|
||||
});
|
||||
const replay =
|
||||
new PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository(
|
||||
replayDb.pool,
|
||||
);
|
||||
assert.deepEqual(
|
||||
await replay.admit(value.plan.runId, value.stepRun.id),
|
||||
{ status: 'existing', receipt: created.receipt },
|
||||
);
|
||||
assert.equal(
|
||||
replayDb.queries.some(({ text }) =>
|
||||
text.includes('plugin_package_workflow_task_attempt_snapshot'),
|
||||
),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('retries serialization once and fails closed without a snapshot', async () => {
|
||||
const value = fixture('pg-wta-retry');
|
||||
const retried = mockPool(value, { serializationFailures: 1 });
|
||||
const repository =
|
||||
new PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository(
|
||||
retried.pool,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await repository.admit(value.plan.runId, value.stepRun.id)
|
||||
).status,
|
||||
'created',
|
||||
);
|
||||
assert.equal(retried.connections, 2);
|
||||
assert.equal(retried.releases, 2);
|
||||
assert.ok(retried.queries.some(({ text }) => text === 'ROLLBACK'));
|
||||
|
||||
const missing = mockPool(value, { snapshotMissing: true });
|
||||
await assert.rejects(
|
||||
new PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository(
|
||||
missing.pool,
|
||||
).admit(value.plan.runId, value.stepRun.id),
|
||||
PluginPackageWorkflowTaskAttemptAdmissionConflictError,
|
||||
);
|
||||
assert.ok(missing.queries.some(({ text }) => text === 'ROLLBACK'));
|
||||
assert.equal(
|
||||
missing.queries.some(({ text }) =>
|
||||
text.includes('INSERT INTO "ql3"."run_attempts"'),
|
||||
),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('is available only through the explicit package subpath', () => {
|
||||
assert.match(
|
||||
require.resolve(
|
||||
'@qinglong/cluster-postgres/plugin-package-workflow-task-attempt-admission',
|
||||
),
|
||||
/pluginPackageWorkflowTaskAttemptAdmissionRepository\.js$/,
|
||||
);
|
||||
const root = require('@qinglong/cluster-postgres');
|
||||
assert.equal(
|
||||
root.PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository,
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,281 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { EventEmitter } = require('node:events');
|
||||
const test = require('node:test');
|
||||
|
||||
const { types } = require('pg');
|
||||
const {
|
||||
POSTGRES_AVAILABILITY_SQLSTATE_CLASSES,
|
||||
POSTGRES_AVAILABILITY_SQLSTATES,
|
||||
POSTGRES_AVAILABILITY_SYSTEM_ERROR_CODES,
|
||||
PgPoolBinding,
|
||||
createPostgresDatabaseOpener,
|
||||
isPostgresAvailabilityError,
|
||||
} = require('../dist');
|
||||
|
||||
function options(overrides = {}) {
|
||||
return {
|
||||
role: 'runtime',
|
||||
connection: {
|
||||
connectionString: 'postgresql://ql3:secret@127.0.0.1:5432/ql3',
|
||||
tls: { mode: 'disable' },
|
||||
},
|
||||
onPoolError() {},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('creates and closes a pg.Pool without eagerly opening a connection', async () => {
|
||||
let passwordReads = 0;
|
||||
const openDatabase = createPostgresDatabaseOpener(
|
||||
options({
|
||||
connection: {
|
||||
host: '127.0.0.1',
|
||||
database: 'ql3',
|
||||
user: 'ql3',
|
||||
password() {
|
||||
passwordReads += 1;
|
||||
return 'secret';
|
||||
},
|
||||
tls: { mode: 'disable' },
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
assert.equal(passwordReads, 0);
|
||||
const database = await openDatabase();
|
||||
assert.equal(passwordReads, 0);
|
||||
await Promise.all([database.close(), database.close()]);
|
||||
assert.equal(passwordReads, 0);
|
||||
});
|
||||
|
||||
test('rejects connection-string TLS overrides and unsafe application names', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
createPostgresDatabaseOpener(
|
||||
options({
|
||||
connection: {
|
||||
connectionString:
|
||||
'postgresql://ql3:secret@127.0.0.1/ql3?sslmode=disable',
|
||||
tls: { mode: 'verify-full' },
|
||||
},
|
||||
}),
|
||||
),
|
||||
/explicit tls option/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
createPostgresDatabaseOpener(
|
||||
options({ pool: { applicationName: 'ql3 runtime -c role=admin' } }),
|
||||
),
|
||||
/safe identifier/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
createPostgresDatabaseOpener(
|
||||
options({
|
||||
connection: {
|
||||
connectionString: 'postgresql://ql3:secret@database.internal/ql3',
|
||||
tls: { mode: 'verify-full' },
|
||||
},
|
||||
}),
|
||||
),
|
||||
/explicit DNS servername/,
|
||||
);
|
||||
});
|
||||
|
||||
test('enforces role-specific bounded pool sizes', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
createPostgresDatabaseOpener(options({ pool: { maxConnections: 65 } })),
|
||||
/between 1 and 64/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
createPostgresDatabaseOpener(
|
||||
options({
|
||||
role: 'migration',
|
||||
pool: { maxConnections: 5 },
|
||||
}),
|
||||
),
|
||||
/between 1 and 4/,
|
||||
);
|
||||
for (const role of [
|
||||
'ai-maintenance',
|
||||
'ai-credential-tester',
|
||||
'automation-manager',
|
||||
'approval-manager',
|
||||
'worker-credential-manager',
|
||||
'worker-credential-executor',
|
||||
]) {
|
||||
assert.doesNotThrow(() => createPostgresDatabaseOpener(options({ role })));
|
||||
assert.throws(
|
||||
() =>
|
||||
createPostgresDatabaseOpener(
|
||||
options({ role, pool: { maxConnections: 5 } }),
|
||||
),
|
||||
/between 1 and 4/,
|
||||
);
|
||||
}
|
||||
assert.throws(
|
||||
() =>
|
||||
createPostgresDatabaseOpener(
|
||||
options({
|
||||
role: 'worker-ingress',
|
||||
pool: { maxConnections: 17 },
|
||||
}),
|
||||
),
|
||||
/between 1 and 16/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
createPostgresDatabaseOpener(
|
||||
options({
|
||||
role: 'admin',
|
||||
pool: { maxConnections: 5 },
|
||||
}),
|
||||
),
|
||||
/between 1 and 4/,
|
||||
);
|
||||
assert.throws(
|
||||
() => createPostgresDatabaseOpener(options({ role: 'unknown' })),
|
||||
/database role is invalid/,
|
||||
);
|
||||
});
|
||||
|
||||
test('keeps PostgreSQL bigint parsing as strings and omits pg-native', () => {
|
||||
assert.equal(types.getTypeParser(20)('9007199254740993'), '9007199254740993');
|
||||
assert.throws(() => require.resolve('pg-native'), {
|
||||
code: 'MODULE_NOT_FOUND',
|
||||
});
|
||||
});
|
||||
|
||||
test('classifies only explicit PostgreSQL and transport availability codes', () => {
|
||||
assert.deepEqual(POSTGRES_AVAILABILITY_SQLSTATE_CLASSES, ['08']);
|
||||
for (const code of [
|
||||
...POSTGRES_AVAILABILITY_SQLSTATES,
|
||||
...POSTGRES_AVAILABILITY_SYSTEM_ERROR_CODES,
|
||||
]) {
|
||||
assert.equal(
|
||||
isPostgresAvailabilityError(Object.assign(new Error(code), { code })),
|
||||
true,
|
||||
code,
|
||||
);
|
||||
}
|
||||
assert.equal(
|
||||
isPostgresAvailabilityError(
|
||||
Object.assign(new Error('vendor connection exception'), {
|
||||
code: '08999',
|
||||
}),
|
||||
),
|
||||
true,
|
||||
);
|
||||
for (const code of ['23505', '40001', '40P01', '55P03', '57014']) {
|
||||
assert.equal(
|
||||
isPostgresAvailabilityError(Object.assign(new Error(code), { code })),
|
||||
false,
|
||||
code,
|
||||
);
|
||||
}
|
||||
assert.equal(isPostgresAvailabilityError(new Error('uncoded')), false);
|
||||
assert.equal(isPostgresAvailabilityError({ code: '08006' }), false);
|
||||
});
|
||||
|
||||
test('reports query availability without replacing the original rejection', async () => {
|
||||
const connectionLost = Object.assign(new Error('connection lost'), {
|
||||
code: '08006',
|
||||
});
|
||||
const duplicate = Object.assign(new Error('duplicate'), { code: '23505' });
|
||||
const observed = [];
|
||||
let currentError = connectionLost;
|
||||
const pool = new PgPoolBinding(
|
||||
{
|
||||
async query() {
|
||||
throw currentError;
|
||||
},
|
||||
},
|
||||
(error) => {
|
||||
observed.push(error);
|
||||
throw new Error('availability listener failure');
|
||||
},
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
pool.query('SELECT 1'),
|
||||
(error) => error === connectionLost,
|
||||
);
|
||||
assert.deepEqual(observed, [connectionLost]);
|
||||
|
||||
currentError = duplicate;
|
||||
await assert.rejects(pool.query('INSERT'), (error) => error === duplicate);
|
||||
assert.deepEqual(observed, [connectionLost]);
|
||||
});
|
||||
|
||||
test('reports availability from connect and bound-client query failures', async () => {
|
||||
const connectError = Object.assign(new Error('network unreachable'), {
|
||||
code: 'ENETUNREACH',
|
||||
});
|
||||
const readOnlyError = Object.assign(new Error('read-only transaction'), {
|
||||
code: '25006',
|
||||
});
|
||||
const observed = [];
|
||||
let connectFails = true;
|
||||
const driverClient = Object.assign(new EventEmitter(), {
|
||||
async query() {
|
||||
throw readOnlyError;
|
||||
},
|
||||
release() {},
|
||||
});
|
||||
const pool = new PgPoolBinding(
|
||||
{
|
||||
async connect() {
|
||||
if (connectFails) throw connectError;
|
||||
return driverClient;
|
||||
},
|
||||
},
|
||||
(error) => observed.push(error),
|
||||
);
|
||||
|
||||
await assert.rejects(pool.connect(), (error) => error === connectError);
|
||||
connectFails = false;
|
||||
const client = await pool.connect();
|
||||
await assert.rejects(
|
||||
client.query('SELECT 1'),
|
||||
(error) => error === readOnlyError,
|
||||
);
|
||||
client.release();
|
||||
assert.deepEqual(observed, [connectError, readOnlyError]);
|
||||
});
|
||||
|
||||
test('contains checked-out client error events and removes its listener on release', async () => {
|
||||
const administratorShutdown = Object.assign(
|
||||
new Error('terminating connection due to administrator command'),
|
||||
{ code: '57P01' },
|
||||
);
|
||||
const observed = [];
|
||||
let releases = 0;
|
||||
const driverClient = Object.assign(new EventEmitter(), {
|
||||
async query() {
|
||||
return { rows: [], rowCount: 0 };
|
||||
},
|
||||
release() {
|
||||
releases += 1;
|
||||
},
|
||||
});
|
||||
const pool = new PgPoolBinding(
|
||||
{
|
||||
async connect() {
|
||||
return driverClient;
|
||||
},
|
||||
},
|
||||
(error) => observed.push(error),
|
||||
);
|
||||
|
||||
const client = await pool.connect();
|
||||
assert.equal(driverClient.listenerCount('error'), 1);
|
||||
assert.equal(driverClient.emit('error', administratorShutdown), true);
|
||||
assert.deepEqual(observed, [administratorShutdown]);
|
||||
|
||||
client.release();
|
||||
assert.equal(releases, 1);
|
||||
assert.equal(driverClient.listenerCount('error'), 0);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,174 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PostgresMigrationLeaderUnavailableError,
|
||||
PostgresMigrationStreamStore,
|
||||
} = require('../dist');
|
||||
const { runMigrationStream } = require('@qinglong/runtime-core');
|
||||
|
||||
const CHECKSUM = 'a'.repeat(64);
|
||||
|
||||
function createPool(lockResults = [true, true]) {
|
||||
const records = new Map();
|
||||
const calls = [];
|
||||
let released = 0;
|
||||
let lockIndex = 0;
|
||||
|
||||
function selectRecord(values, source) {
|
||||
const record = source.get(values[0]);
|
||||
return { rows: record ? [{ ...record }] : [] };
|
||||
}
|
||||
|
||||
return {
|
||||
calls,
|
||||
records,
|
||||
get released() {
|
||||
return released;
|
||||
},
|
||||
pool: {
|
||||
async query(text, values = []) {
|
||||
calls.push({ scope: 'pool', text, values: [...values] });
|
||||
if (text.startsWith('SELECT\n stream_id')) {
|
||||
if (values.length === 0) {
|
||||
return {
|
||||
rows: [...records.values()].map((record) => ({ ...record })),
|
||||
};
|
||||
}
|
||||
return selectRecord(values, records);
|
||||
}
|
||||
throw new Error(`unexpected pool query: ${text}`);
|
||||
},
|
||||
async connect() {
|
||||
let staged = null;
|
||||
return {
|
||||
async query(text, values = []) {
|
||||
calls.push({ scope: 'client', text, values: [...values] });
|
||||
if (text === 'BEGIN') {
|
||||
staged = new Map(
|
||||
[...records].map(([id, record]) => [id, { ...record }]),
|
||||
);
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text === 'COMMIT') {
|
||||
records.clear();
|
||||
for (const [id, record] of staged) records.set(id, record);
|
||||
staged = null;
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text === 'ROLLBACK') {
|
||||
staged = null;
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text.startsWith('SELECT set_config')) return { rows: [] };
|
||||
if (text.startsWith('SELECT pg_try_advisory_xact_lock')) {
|
||||
return { rows: [{ acquired: lockResults[lockIndex++] ?? true }] };
|
||||
}
|
||||
if (text.startsWith('CREATE SCHEMA')) return { rows: [] };
|
||||
if (text.startsWith('CREATE TABLE')) return { rows: [] };
|
||||
if (text.startsWith('SELECT\n stream_id')) {
|
||||
return selectRecord(values, staged);
|
||||
}
|
||||
if (text.startsWith('INSERT INTO')) {
|
||||
const [migrationId, streamId, dialect, checksum, appliedAtMs] =
|
||||
values;
|
||||
staged.set(migrationId, {
|
||||
migrationId,
|
||||
streamId,
|
||||
dialect,
|
||||
checksum,
|
||||
appliedAtMs: String(appliedAtMs),
|
||||
});
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text === 'CREATE TABLE ql3.run_probe(id integer)') {
|
||||
return { rows: [] };
|
||||
}
|
||||
throw new Error(`unexpected client query: ${text}`);
|
||||
},
|
||||
release() {
|
||||
released += 1;
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function stream(up) {
|
||||
return {
|
||||
id: 'postgresql-main',
|
||||
dialect: 'postgresql',
|
||||
migrationIdScheme: 'postgres-prefixed',
|
||||
checksumScheme: 'sha256',
|
||||
migrations: [
|
||||
{
|
||||
id: 'pg-0001-schema-history',
|
||||
checksum: CHECKSUM,
|
||||
up,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
test('serializes PostgreSQL history bootstrap and migration in advisory-lock transactions', async () => {
|
||||
const state = createPool();
|
||||
let upCalls = 0;
|
||||
await runMigrationStream({
|
||||
stream: stream(async (context) => {
|
||||
upCalls += 1;
|
||||
await context.query('CREATE TABLE ql3.run_probe(id integer)');
|
||||
}),
|
||||
store: new PostgresMigrationStreamStore(state.pool),
|
||||
clock: () => 123,
|
||||
});
|
||||
|
||||
assert.equal(upCalls, 1);
|
||||
assert.equal(state.released, 2);
|
||||
assert.deepEqual(state.records.get('pg-0001-schema-history'), {
|
||||
migrationId: 'pg-0001-schema-history',
|
||||
streamId: 'postgresql-main',
|
||||
dialect: 'postgresql',
|
||||
checksum: CHECKSUM,
|
||||
appliedAtMs: '123',
|
||||
});
|
||||
assert.equal(
|
||||
state.calls.filter(({ text }) =>
|
||||
text.startsWith('SELECT pg_try_advisory_xact_lock'),
|
||||
).length,
|
||||
2,
|
||||
);
|
||||
assert.equal(state.calls.filter(({ text }) => text === 'COMMIT').length, 2);
|
||||
});
|
||||
|
||||
test('fails closed before migration work when another migration leader owns the lock', async () => {
|
||||
const state = createPool([true, false]);
|
||||
let upCalls = 0;
|
||||
await assert.rejects(
|
||||
runMigrationStream({
|
||||
stream: stream(async () => {
|
||||
upCalls += 1;
|
||||
}),
|
||||
store: new PostgresMigrationStreamStore(state.pool),
|
||||
}),
|
||||
PostgresMigrationLeaderUnavailableError,
|
||||
);
|
||||
assert.equal(upCalls, 0);
|
||||
assert.equal(state.records.size, 0);
|
||||
assert.equal(state.calls.filter(({ text }) => text === 'ROLLBACK').length, 1);
|
||||
assert.equal(state.released, 2);
|
||||
});
|
||||
|
||||
test('rolls PostgreSQL migration work and history back together', async () => {
|
||||
const state = createPool();
|
||||
await assert.rejects(
|
||||
runMigrationStream({
|
||||
stream: stream(async () => {
|
||||
throw new Error('migration work failed');
|
||||
}),
|
||||
store: new PostgresMigrationStreamStore(state.pool),
|
||||
}),
|
||||
/migration work failed/,
|
||||
);
|
||||
assert.equal(state.records.size, 0);
|
||||
assert.equal(state.calls.filter(({ text }) => text === 'ROLLBACK').length, 1);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,222 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
ProjectPolicyUnavailableError,
|
||||
ProjectRoleBindingMutationConflictError,
|
||||
ProjectRoleBindingVersionConflictError,
|
||||
} = require('@qinglong/runtime-core/project-policy');
|
||||
const {
|
||||
PostgresProjectPolicyRepository,
|
||||
} = require('../dist/security/projectPolicyRepository');
|
||||
|
||||
const BINDING = Object.freeze({
|
||||
projectId: 'default',
|
||||
subject: Object.freeze({ type: 'user', id: 'usr_primary' }),
|
||||
version: 1,
|
||||
state: 'active',
|
||||
role: 'owner',
|
||||
mutationId: 'grant-owner-1',
|
||||
changedBy: Object.freeze({ type: 'system', id: 'bootstrap' }),
|
||||
createdAtMs: 1,
|
||||
});
|
||||
|
||||
function bindingRow(overrides = {}) {
|
||||
return {
|
||||
projectId: 'default',
|
||||
subjectType: 'user',
|
||||
subjectId: 'usr_primary',
|
||||
version: 1,
|
||||
state: 'active',
|
||||
role: 'owner',
|
||||
mutationId: 'grant-owner-1',
|
||||
changedByType: 'system',
|
||||
changedById: 'bootstrap',
|
||||
createdAtMs: '1',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function snapshotRow(overrides = {}) {
|
||||
return {
|
||||
projectId: 'default',
|
||||
projectName: 'Default',
|
||||
projectSlug: 'default',
|
||||
projectStatus: 'active',
|
||||
projectVersion: 1,
|
||||
projectCreatedAtMs: '0',
|
||||
projectUpdatedAtMs: '0',
|
||||
bindingProjectId: 'default',
|
||||
bindingSubjectType: 'user',
|
||||
bindingSubjectId: 'usr_primary',
|
||||
bindingVersion: 1,
|
||||
bindingState: 'active',
|
||||
bindingRole: 'owner',
|
||||
bindingMutationId: 'grant-owner-1',
|
||||
bindingChangedByType: 'system',
|
||||
bindingChangedById: 'bootstrap',
|
||||
bindingCreatedAtMs: '1',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('resolves one normalized Project and latest RoleBinding snapshot', async () => {
|
||||
const queries = [];
|
||||
const repository = new PostgresProjectPolicyRepository({
|
||||
async query(text, values) {
|
||||
queries.push({ text, values });
|
||||
return { rows: [snapshotRow()] };
|
||||
},
|
||||
async connect() {
|
||||
throw new Error('not used');
|
||||
},
|
||||
});
|
||||
assert.deepEqual(
|
||||
await repository.resolve('default', { type: 'user', id: 'usr_primary' }),
|
||||
{
|
||||
project: {
|
||||
id: 'default',
|
||||
name: 'Default',
|
||||
slug: 'default',
|
||||
status: 'active',
|
||||
version: 1,
|
||||
createdAtMs: 0,
|
||||
updatedAtMs: 0,
|
||||
},
|
||||
binding: BINDING,
|
||||
},
|
||||
);
|
||||
assert.deepEqual(queries[0].values, ['default', 'user', 'usr_primary']);
|
||||
assert.match(queries[0].text, /LEFT JOIN LATERAL/);
|
||||
assert.match(queries[0].text, /ORDER BY candidate\.version DESC/);
|
||||
await assert.rejects(
|
||||
repository.resolve('x'.repeat(129), {
|
||||
type: 'user',
|
||||
id: 'usr_primary',
|
||||
}),
|
||||
);
|
||||
assert.equal(queries.length, 1);
|
||||
});
|
||||
|
||||
function appendPool(options = {}) {
|
||||
const events = [];
|
||||
let connection = 0;
|
||||
return {
|
||||
events,
|
||||
pool: {
|
||||
async query() {
|
||||
throw new Error('not used');
|
||||
},
|
||||
async connect() {
|
||||
connection += 1;
|
||||
let inserted = false;
|
||||
return {
|
||||
async query(text) {
|
||||
events.push(text.split('\n', 1)[0]);
|
||||
if (text.startsWith('SELECT id FROM'))
|
||||
return { rows: [{ id: 'default' }] };
|
||||
if (text.includes('mutation_id = $2')) {
|
||||
return {
|
||||
rows: options.replay ? [bindingRow(options.replay)] : [],
|
||||
};
|
||||
}
|
||||
if (text.startsWith('SELECT version FROM')) {
|
||||
return {
|
||||
rows:
|
||||
options.currentVersion === undefined
|
||||
? []
|
||||
: [{ version: options.currentVersion }],
|
||||
};
|
||||
}
|
||||
if (text.startsWith('INSERT INTO')) {
|
||||
if (options.retryOnce && connection === 1 && !inserted) {
|
||||
inserted = true;
|
||||
throw Object.assign(new Error('serialization'), {
|
||||
code: '40001',
|
||||
});
|
||||
}
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
return { rows: [] };
|
||||
},
|
||||
release() {
|
||||
events.push(`release:${connection}`);
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('appends under a serializable Project lock and replays exactly', async () => {
|
||||
const insertedFixture = appendPool();
|
||||
const inserted = await new PostgresProjectPolicyRepository(
|
||||
insertedFixture.pool,
|
||||
).append({ expectedCurrentVersion: 0, binding: BINDING });
|
||||
assert.deepEqual(inserted, { status: 'inserted', binding: BINDING });
|
||||
assert.equal(
|
||||
insertedFixture.events.some((event) =>
|
||||
event.startsWith('BEGIN ISOLATION LEVEL SERIALIZABLE'),
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.ok(
|
||||
insertedFixture.events.findIndex((event) => event.startsWith('SELECT id')) <
|
||||
insertedFixture.events.findIndex((event) => event.startsWith('INSERT')),
|
||||
);
|
||||
assert.ok(insertedFixture.events.includes('COMMIT'));
|
||||
|
||||
const replayFixture = appendPool({ replay: {} });
|
||||
const replay = await new PostgresProjectPolicyRepository(
|
||||
replayFixture.pool,
|
||||
).append({ expectedCurrentVersion: 0, binding: BINDING });
|
||||
assert.deepEqual(replay, { status: 'existing', binding: BINDING });
|
||||
assert.equal(
|
||||
replayFixture.events.some((event) => event.startsWith('INSERT')),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects stale versions and conflicting mutation replay', async () => {
|
||||
await assert.rejects(
|
||||
new PostgresProjectPolicyRepository(
|
||||
appendPool({ currentVersion: 1 }).pool,
|
||||
).append({ expectedCurrentVersion: 0, binding: BINDING }),
|
||||
ProjectRoleBindingVersionConflictError,
|
||||
);
|
||||
await assert.rejects(
|
||||
new PostgresProjectPolicyRepository(
|
||||
appendPool({ replay: { role: 'viewer' } }).pool,
|
||||
).append({ expectedCurrentVersion: 0, binding: BINDING }),
|
||||
ProjectRoleBindingMutationConflictError,
|
||||
);
|
||||
});
|
||||
|
||||
test('retries serialization failures but fails closed on corrupt rows', async () => {
|
||||
const retryFixture = appendPool({ retryOnce: true });
|
||||
assert.equal(
|
||||
(
|
||||
await new PostgresProjectPolicyRepository(retryFixture.pool).append({
|
||||
expectedCurrentVersion: 0,
|
||||
binding: BINDING,
|
||||
})
|
||||
).status,
|
||||
'inserted',
|
||||
);
|
||||
assert.equal(
|
||||
retryFixture.events.filter((event) => event.startsWith('release:')).length,
|
||||
2,
|
||||
);
|
||||
|
||||
const repository = new PostgresProjectPolicyRepository({
|
||||
async query() {
|
||||
return { rows: [snapshotRow({ projectVersion: 'not-a-number' })] };
|
||||
},
|
||||
async connect() {
|
||||
throw new Error('not used');
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
repository.resolve('default', { type: 'user', id: 'usr_primary' }),
|
||||
ProjectPolicyUnavailableError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,170 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
createProjectToolDefinitionSnapshot,
|
||||
ProjectToolDefinitionSnapshotConflictError,
|
||||
} = require('@qinglong/runtime-core/project-tool-definition-snapshot');
|
||||
const {
|
||||
PostgresProjectToolDefinitionSnapshotRepository,
|
||||
} = require('../dist/tool-execution/projectToolDefinitionSnapshotRepository');
|
||||
|
||||
function fakePool() {
|
||||
const queries = [];
|
||||
const snapshots = new Map();
|
||||
const client = {
|
||||
async query(text, values = []) {
|
||||
queries.push({ text, values });
|
||||
if (
|
||||
text.startsWith('BEGIN') ||
|
||||
text.startsWith('SELECT set_config') ||
|
||||
text === 'COMMIT' ||
|
||||
text === 'ROLLBACK'
|
||||
) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."plugin_package_install_heads"')) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (
|
||||
text.startsWith('INSERT INTO "ql3"."project_tool_definition_snapshots"')
|
||||
) {
|
||||
const key = `${values[0]}:${values[1]}`;
|
||||
if (snapshots.has(key)) return { rows: [] };
|
||||
snapshots.set(key, {
|
||||
projectId: values[0],
|
||||
activeVectorDigest: values[1],
|
||||
definitionsDigest: values[2],
|
||||
snapshotDigest: values[3],
|
||||
snapshotJson: JSON.parse(values[4]),
|
||||
committedAtMs: 500,
|
||||
});
|
||||
return { rows: [{ activeVectorDigest: values[1] }] };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."project_tool_definition_snapshots"')) {
|
||||
const row = snapshots.get(`${values[0]}:${values[1]}`);
|
||||
return { rows: row ? [{ ...row }] : [] };
|
||||
}
|
||||
if (
|
||||
text.includes('FROM "ql3"."project_tool_definition_snapshot_sources"')
|
||||
) {
|
||||
return { rows: [] };
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${text}`);
|
||||
},
|
||||
release() {},
|
||||
};
|
||||
return {
|
||||
queries,
|
||||
pool: {
|
||||
query: (...args) => client.query(...args),
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
},
|
||||
corrupt(snapshot) {
|
||||
snapshots.get(
|
||||
`${snapshot.projectId}:${snapshot.activeVectorDigest}`,
|
||||
).snapshotJson.snapshotDigest = 'f'.repeat(64);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('publishes and exactly replays one empty PostgreSQL snapshot', async () => {
|
||||
const value = fakePool();
|
||||
const repository = new PostgresProjectToolDefinitionSnapshotRepository(
|
||||
value.pool,
|
||||
);
|
||||
const snapshot = createProjectToolDefinitionSnapshot({
|
||||
projectId: 'project-postgres-snapshot',
|
||||
contributions: [],
|
||||
});
|
||||
|
||||
assert.equal(await repository.findCurrent(snapshot.projectId), null);
|
||||
const created = await repository.publish(snapshot);
|
||||
assert.equal(created.status, 'created');
|
||||
assert.deepEqual(created.record.snapshot, snapshot);
|
||||
assert.equal((await repository.publish(snapshot)).status, 'existing');
|
||||
assert.deepEqual(
|
||||
await repository.findCurrent(snapshot.projectId),
|
||||
created.record,
|
||||
);
|
||||
assert.match(
|
||||
value.queries.find(({ text }) =>
|
||||
text.startsWith('INSERT INTO "ql3"."project_tool_definition_snapshots"'),
|
||||
).text,
|
||||
/clock_timestamp\(\)/,
|
||||
);
|
||||
assert.equal(
|
||||
value.queries.some(({ text }) =>
|
||||
text.startsWith('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY'),
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('fails closed for corrupted PostgreSQL snapshot JSON', async () => {
|
||||
const value = fakePool();
|
||||
const repository = new PostgresProjectToolDefinitionSnapshotRepository(
|
||||
value.pool,
|
||||
);
|
||||
const snapshot = createProjectToolDefinitionSnapshot({
|
||||
projectId: 'project-postgres-corrupt',
|
||||
contributions: [],
|
||||
});
|
||||
await repository.publish(snapshot);
|
||||
value.corrupt(snapshot);
|
||||
await assert.rejects(
|
||||
repository.findCurrent(snapshot.projectId),
|
||||
/snapshot is unavailable/,
|
||||
);
|
||||
});
|
||||
|
||||
test('maps PostgreSQL constraint rejection to snapshot conflict', async () => {
|
||||
const repository = new PostgresProjectToolDefinitionSnapshotRepository({
|
||||
async query() {
|
||||
return { rows: [] };
|
||||
},
|
||||
async connect() {
|
||||
return {
|
||||
async query(text) {
|
||||
if (
|
||||
text.startsWith('BEGIN') ||
|
||||
text.startsWith('SELECT set_config')
|
||||
) {
|
||||
return { rows: [] };
|
||||
}
|
||||
const error = new Error('constraint');
|
||||
error.code = '23505';
|
||||
throw error;
|
||||
},
|
||||
release() {},
|
||||
};
|
||||
},
|
||||
});
|
||||
const snapshot = createProjectToolDefinitionSnapshot({
|
||||
projectId: 'project-postgres-conflict',
|
||||
contributions: [],
|
||||
});
|
||||
await assert.rejects(
|
||||
repository.publish(snapshot),
|
||||
ProjectToolDefinitionSnapshotConflictError,
|
||||
);
|
||||
});
|
||||
|
||||
test('publishes PostgreSQL snapshot storage only through executor subpaths', () => {
|
||||
assert.equal(
|
||||
require('@qinglong/cluster-postgres/project-tool-definition-snapshot')
|
||||
.PostgresProjectToolDefinitionSnapshotRepository,
|
||||
PostgresProjectToolDefinitionSnapshotRepository,
|
||||
);
|
||||
assert.equal(
|
||||
require('../dist/entrypoints/packageExecutor')
|
||||
.PostgresProjectToolDefinitionSnapshotRepository,
|
||||
PostgresProjectToolDefinitionSnapshotRepository,
|
||||
);
|
||||
assert.equal(
|
||||
require('../dist').PostgresProjectToolDefinitionSnapshotRepository,
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,384 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
RemoteRunActivationUnavailableError,
|
||||
} = require('@qinglong/runtime-core/remote-activation');
|
||||
const {
|
||||
createStepRunRecord,
|
||||
} = require('@qinglong/runtime-core/step-run');
|
||||
const {
|
||||
PostgresRemoteRunActivationRepository,
|
||||
} = require('../dist/entrypoints/runtime');
|
||||
|
||||
const SESSION_ID = '018f5c64-9b9d-7f1a-8c2d-1234567890ac';
|
||||
const LEASE_TOKEN = 'worker_generated_lease_capability_0000000000000001';
|
||||
const LEASE_DIGEST = createHash('sha256').update(LEASE_TOKEN).digest('hex');
|
||||
|
||||
function command() {
|
||||
return {
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
workerId: 'edge-1',
|
||||
workerSessionId: SESSION_ID,
|
||||
workerGeneration: 2,
|
||||
offerId: 'offer-1',
|
||||
leaseGeneration: 3,
|
||||
leaseToken: LEASE_TOKEN,
|
||||
expectedLeaseVersion: 4,
|
||||
eventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a1',
|
||||
};
|
||||
}
|
||||
|
||||
function fixture({ failEvent = false, timeoutMs = 5_000, omitTimeout = false } = {}) {
|
||||
const calls = [];
|
||||
const client = {
|
||||
async query(sql, params = []) {
|
||||
const normalized = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ sql: normalized, params });
|
||||
if (
|
||||
normalized === 'BEGIN' ||
|
||||
normalized.startsWith('SET LOCAL') ||
|
||||
normalized === 'COMMIT' ||
|
||||
normalized === 'ROLLBACK' ||
|
||||
normalized.startsWith('SELECT pg_advisory_xact_lock')
|
||||
) return { rows: [], rowCount: 0 };
|
||||
if (normalized.includes('FROM "ql3"."worker_sessions"')) {
|
||||
return {
|
||||
rows: [{
|
||||
workerId: 'edge-1',
|
||||
sessionId: SESSION_ID,
|
||||
workerGeneration: 2,
|
||||
workerStatus: 'online',
|
||||
workerLeaseExpiresAtMs: 2_000,
|
||||
}],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('INNER JOIN "ql3"."run_attempts"')) {
|
||||
return {
|
||||
rows: [{
|
||||
runId: 'run-1',
|
||||
runStatus: 'dispatching',
|
||||
executionOwner: 'runtime',
|
||||
cancelRequestedAtMs: null,
|
||||
cancelReason: null,
|
||||
runErrorCode: null,
|
||||
runVersion: 3,
|
||||
eventSequence: 7,
|
||||
planJson: {
|
||||
command: { kind: 'argv', file: '/bin/true', args: [] },
|
||||
environment: [],
|
||||
...(omitTimeout ? {} : { timeoutMs }),
|
||||
},
|
||||
attemptId: 'attempt-1',
|
||||
attemptRunId: 'run-1',
|
||||
attemptStatus: 'claimed',
|
||||
executorType: 'remote_worker',
|
||||
attemptWorkerId: 'edge-1',
|
||||
attemptWorkerSessionId: SESSION_ID,
|
||||
attemptWorkerGeneration: 2,
|
||||
attemptLeaseGeneration: 3,
|
||||
attemptLeaseVersion: 4,
|
||||
attemptLeaseTokenDigest: LEASE_DIGEST,
|
||||
attemptOfferId: 'offer-1',
|
||||
callbackSequence: 0,
|
||||
callbackTokenDigest: null,
|
||||
executorHandle: null,
|
||||
logArtifactId: null,
|
||||
deadlineAtMs: null,
|
||||
startedAtMs: null,
|
||||
finishedAtMs: null,
|
||||
attemptErrorCode: null,
|
||||
}],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."run_dispatch_leases"')) {
|
||||
return {
|
||||
rows: [{
|
||||
attemptId: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
leaseStatus: 'leased',
|
||||
leaseVersion: 4,
|
||||
leaseGeneration: 3,
|
||||
workerId: 'edge-1',
|
||||
workerSessionId: SESSION_ID,
|
||||
workerGeneration: 2,
|
||||
leaseTokenDigest: LEASE_DIGEST,
|
||||
offerId: 'offer-1',
|
||||
leaseExpiresAtMs: 2_000,
|
||||
}],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: 1_000 }], rowCount: 1 };
|
||||
}
|
||||
if (normalized.startsWith('UPDATE "ql3"."run_attempts"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (normalized.startsWith('UPDATE "ql3"."runs"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (normalized.startsWith('INSERT INTO "ql3"."run_events"')) {
|
||||
if (failEvent) throw new Error('injected event failure');
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`Unexpected SQL: ${normalized}`);
|
||||
},
|
||||
release() {
|
||||
calls.push({ sql: 'RELEASE', params: [] });
|
||||
},
|
||||
};
|
||||
const pool = {
|
||||
async connect() { return client; },
|
||||
};
|
||||
return { repository: new PostgresRemoteRunActivationRepository(pool), calls };
|
||||
}
|
||||
|
||||
test('locks every authority before database time and persists digest-only starting', async () => {
|
||||
const { repository, calls } = fixture();
|
||||
const activation = await repository.acknowledgeStarting(command());
|
||||
assert.equal(activation.status, 'applied');
|
||||
assert.equal(activation.snapshot.attemptStatus, 'starting');
|
||||
assert.equal(activation.snapshot.deadlineAtMs, 6_000);
|
||||
const index = (needle) => calls.findIndex(({ sql }) => sql.includes(needle));
|
||||
assert.ok(index('pg_advisory_xact_lock') < index('worker_sessions'));
|
||||
assert.ok(index('worker_sessions') < index('INNER JOIN "ql3"."run_attempts"'));
|
||||
assert.ok(index('INNER JOIN "ql3"."run_attempts"') < index('run_dispatch_leases'));
|
||||
assert.ok(index('run_dispatch_leases') < index('statement_timestamp()'));
|
||||
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), true);
|
||||
assert.equal(JSON.stringify(calls).includes(LEASE_TOKEN), false);
|
||||
assert.equal(JSON.stringify(calls).includes(LEASE_DIGEST), true);
|
||||
const attemptUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."run_attempts"'));
|
||||
assert.equal(attemptUpdate.params.at(-1), 6_000);
|
||||
});
|
||||
|
||||
test('keeps the durable deadline absent when the immutable revision has no timeout', async () => {
|
||||
const { repository, calls } = fixture({ omitTimeout: true });
|
||||
const activation = await repository.acknowledgeStarting(command());
|
||||
assert.equal(activation.snapshot.deadlineAtMs, undefined);
|
||||
const attemptUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."run_attempts"'));
|
||||
assert.equal(attemptUpdate.params.at(-1), null);
|
||||
});
|
||||
|
||||
test('rolls the transaction back and exposes storage failure as unavailable', async () => {
|
||||
const { repository, calls } = fixture({ failEvent: true });
|
||||
await assert.rejects(
|
||||
repository.acknowledgeStarting(command()),
|
||||
(error) =>
|
||||
error instanceof RemoteRunActivationUnavailableError &&
|
||||
error.cause?.message === 'injected event failure',
|
||||
);
|
||||
assert.equal(calls.some(({ sql }) => sql === 'ROLLBACK'), true);
|
||||
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), false);
|
||||
});
|
||||
|
||||
function workflowFixture() {
|
||||
const stepRun = createStepRunRecord({
|
||||
id: 'workflow-step-1',
|
||||
runId: 'run-1',
|
||||
stepKey: 'collect',
|
||||
kind: 'task',
|
||||
definitionRef: 'pkg:demo:alpha',
|
||||
definitionDigest: 'a'.repeat(64),
|
||||
required: true,
|
||||
initialStatus: 'ready',
|
||||
mutationId: 'workflow-step-created',
|
||||
createdAtMs: 500,
|
||||
});
|
||||
const calls = [];
|
||||
const client = {
|
||||
async query(sql, params = []) {
|
||||
const normalized = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ sql: normalized, params });
|
||||
if (
|
||||
normalized === 'BEGIN' ||
|
||||
normalized.startsWith('SET LOCAL') ||
|
||||
normalized === 'COMMIT' ||
|
||||
normalized === 'ROLLBACK' ||
|
||||
normalized.startsWith('SELECT pg_advisory_xact_lock')
|
||||
) {
|
||||
return { rows: [], rowCount: 0 };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."worker_sessions"')) {
|
||||
return {
|
||||
rows: [{
|
||||
workerId: 'edge-1',
|
||||
sessionId: SESSION_ID,
|
||||
workerGeneration: 2,
|
||||
workerStatus: 'online',
|
||||
workerLeaseExpiresAtMs: 2_000,
|
||||
}],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('INNER JOIN "ql3"."run_attempts"')) {
|
||||
return {
|
||||
rows: [{
|
||||
runId: 'run-1',
|
||||
runStatus: 'running',
|
||||
executionOwner: 'runtime',
|
||||
cancelRequestedAtMs: null,
|
||||
cancelReason: null,
|
||||
runErrorCode: null,
|
||||
runVersion: 4,
|
||||
eventSequence: 4,
|
||||
planJson: {
|
||||
command: { kind: 'argv', file: '/bin/true', args: [] },
|
||||
environment: [],
|
||||
timeoutMs: 5_000,
|
||||
},
|
||||
attemptId: 'attempt-1',
|
||||
attemptRunId: 'run-1',
|
||||
attemptStepRunId: stepRun.id,
|
||||
attemptStatus: 'starting',
|
||||
executorType: 'remote_worker',
|
||||
attemptWorkerId: 'edge-1',
|
||||
attemptWorkerSessionId: SESSION_ID,
|
||||
attemptWorkerGeneration: 2,
|
||||
attemptLeaseGeneration: 3,
|
||||
attemptLeaseVersion: 4,
|
||||
attemptLeaseTokenDigest: LEASE_DIGEST,
|
||||
attemptOfferId: 'offer-1',
|
||||
callbackSequence: 0,
|
||||
callbackTokenDigest: null,
|
||||
executorHandle: null,
|
||||
logArtifactId: null,
|
||||
deadlineAtMs: 6_000,
|
||||
startedAtMs: null,
|
||||
finishedAtMs: null,
|
||||
attemptErrorCode: null,
|
||||
workflowAttemptId: 'attempt-1',
|
||||
workflowStepRunId: stepRun.id,
|
||||
admittedWorkflowStepVersion: stepRun.version,
|
||||
admittedWorkflowStepDigest: stepRun.stepRunDigest,
|
||||
}],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."step_runs"')) {
|
||||
return {
|
||||
rows: [{
|
||||
workflowStepVersion: stepRun.version,
|
||||
workflowStepDigest: stepRun.stepRunDigest,
|
||||
workflowStepJson: stepRun,
|
||||
}],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."run_dispatch_leases"')) {
|
||||
return {
|
||||
rows: [{
|
||||
attemptId: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
leaseStatus: 'leased',
|
||||
leaseVersion: 4,
|
||||
leaseGeneration: 3,
|
||||
workerId: 'edge-1',
|
||||
workerSessionId: SESSION_ID,
|
||||
workerGeneration: 2,
|
||||
leaseTokenDigest: LEASE_DIGEST,
|
||||
offerId: 'offer-1',
|
||||
leaseExpiresAtMs: 2_000,
|
||||
}],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: 1_000 }], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
normalized.startsWith('UPDATE ') ||
|
||||
normalized.startsWith('INSERT INTO ')
|
||||
) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`Unexpected SQL: ${normalized}`);
|
||||
},
|
||||
release() {
|
||||
calls.push({ sql: 'RELEASE', params: [] });
|
||||
},
|
||||
};
|
||||
return {
|
||||
calls,
|
||||
repository: new PostgresRemoteRunActivationRepository({
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
}),
|
||||
stepRun,
|
||||
};
|
||||
}
|
||||
|
||||
test('activates a Workflow Task Attempt by advancing StepRun, not the aggregate Run', async () => {
|
||||
const { repository, calls, stepRun } = workflowFixture();
|
||||
const result = await repository.acknowledgeRunning({
|
||||
...command(),
|
||||
attemptEventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a2',
|
||||
runEventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a3',
|
||||
executorHandle: 'worker-handle-1',
|
||||
callbackSequence: 1,
|
||||
callbackTokenDigest: 'b'.repeat(64),
|
||||
});
|
||||
assert.equal(result.status, 'applied');
|
||||
assert.equal(result.snapshot.runStatus, 'running');
|
||||
assert.equal(result.snapshot.attemptStatus, 'running');
|
||||
|
||||
const runUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"'));
|
||||
assert.equal(runUpdate.params[1], 'running');
|
||||
assert.equal(runUpdate.params.at(-1), 'running');
|
||||
const stepUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."step_runs"'));
|
||||
assert.ok(stepUpdate);
|
||||
assert.equal(stepUpdate.params[0], 'running');
|
||||
assert.equal(stepUpdate.params[2], stepRun.attemptCount + 1);
|
||||
assert.ok(
|
||||
calls.some(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."step_run_mutations"')),
|
||||
);
|
||||
const attemptEvent = calls.find(
|
||||
({ sql, params }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"') &&
|
||||
params[3] === 'workflow.task_attempt.running',
|
||||
);
|
||||
assert.equal(attemptEvent.params[7], stepRun.id);
|
||||
assert.match(attemptEvent.params[8], /"execution_scope":"workflow_task"/);
|
||||
});
|
||||
|
||||
test('fails a Workflow Task before StepRun start without terminalizing its aggregate Run', async () => {
|
||||
const { repository, calls, stepRun } = workflowFixture();
|
||||
const result = await repository.failStart({
|
||||
...command(),
|
||||
attemptEventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a2',
|
||||
runEventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a3',
|
||||
});
|
||||
|
||||
assert.equal(result.status, 'applied');
|
||||
assert.equal(result.snapshot.runStatus, 'running');
|
||||
assert.equal(result.snapshot.attemptStatus, 'failed');
|
||||
assert.equal(result.snapshot.errorCode, 'EXECUTOR_START_FAILED');
|
||||
|
||||
const runUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"'));
|
||||
assert.equal(runUpdate.params[1], 'running');
|
||||
assert.equal(runUpdate.params[2], null);
|
||||
assert.equal(runUpdate.params[3], null);
|
||||
assert.equal(runUpdate.params[4], null);
|
||||
const stepUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."step_runs"'));
|
||||
assert.ok(stepUpdate);
|
||||
assert.equal(stepUpdate.params[0], 'failed');
|
||||
assert.equal(stepUpdate.params[2], stepRun.attemptCount);
|
||||
assert.equal(stepUpdate.params[5], stepRun.readyAtMs);
|
||||
assert.equal(stepUpdate.params[6], null);
|
||||
assert.equal(stepUpdate.params[7], 1_000);
|
||||
assert.ok(
|
||||
calls.some(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."step_run_mutations"')),
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,130 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PostgresRemoteRunActivationRepository,
|
||||
} = require('@qinglong/cluster-postgres/runtime');
|
||||
|
||||
test('pins the shared running status parameter to PostgreSQL varchar', async () => {
|
||||
const token = 'worker_generated_lease_capability_0000000000000001';
|
||||
const digest = createHash('sha256').update(token).digest('hex');
|
||||
const client = {
|
||||
async query(text) {
|
||||
const sql = String(text);
|
||||
if (sql === 'BEGIN' || sql === 'COMMIT' || sql === 'ROLLBACK') {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (sql.startsWith('SET LOCAL')) return { rows: [] };
|
||||
if (sql.includes('pg_advisory_xact_lock')) return { rows: [{}] };
|
||||
if (sql.includes('FROM "ql3"."worker_sessions"')) {
|
||||
return {
|
||||
rows: [{
|
||||
workerId: 'worker-live',
|
||||
sessionId: '019f7094-a853-72f3-82ab-dfa08e6bd1c1',
|
||||
workerGeneration: 1,
|
||||
workerStatus: 'online',
|
||||
workerLeaseExpiresAtMs: '9000',
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (sql.includes('FROM "ql3"."runs" AS run')) {
|
||||
return {
|
||||
rows: [{
|
||||
runId: 'run-live',
|
||||
runStatus: 'dispatching',
|
||||
executionOwner: 'runtime',
|
||||
cancelRequestedAtMs: null,
|
||||
cancelReason: null,
|
||||
runErrorCode: null,
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
planJson: { command: {}, environment: [] },
|
||||
attemptId: 'attempt-live',
|
||||
attemptRunId: 'run-live',
|
||||
attemptStepRunId: null,
|
||||
attemptStatus: 'starting',
|
||||
executorType: 'remote_worker',
|
||||
attemptWorkerId: 'worker-live',
|
||||
attemptWorkerSessionId: '019f7094-a853-72f3-82ab-dfa08e6bd1c1',
|
||||
attemptWorkerGeneration: 1,
|
||||
attemptLeaseGeneration: 1,
|
||||
attemptLeaseVersion: 0,
|
||||
attemptLeaseTokenDigest: digest,
|
||||
attemptOfferId: 'offer-live',
|
||||
callbackSequence: 0,
|
||||
callbackTokenDigest: null,
|
||||
executorHandle: null,
|
||||
logArtifactId: null,
|
||||
deadlineAtMs: null,
|
||||
startedAtMs: null,
|
||||
finishedAtMs: null,
|
||||
attemptErrorCode: null,
|
||||
workflowAttemptId: null,
|
||||
workflowStepRunId: null,
|
||||
admittedWorkflowStepVersion: null,
|
||||
admittedWorkflowStepDigest: null,
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (sql.includes('FROM "ql3"."run_dispatch_leases"')) {
|
||||
return {
|
||||
rows: [{
|
||||
attemptId: 'attempt-live',
|
||||
runId: 'run-live',
|
||||
leaseStatus: 'leased',
|
||||
leaseVersion: 0,
|
||||
leaseGeneration: 1,
|
||||
workerId: 'worker-live',
|
||||
workerSessionId: '019f7094-a853-72f3-82ab-dfa08e6bd1c1',
|
||||
workerGeneration: 1,
|
||||
leaseTokenDigest: digest,
|
||||
offerId: 'offer-live',
|
||||
leaseExpiresAtMs: '9000',
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (sql.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: '1000' }] };
|
||||
}
|
||||
if (sql.includes('UPDATE "ql3"."run_attempts"')) {
|
||||
return { rowCount: 1, rows: [] };
|
||||
}
|
||||
if (sql.includes('UPDATE "ql3"."runs"')) {
|
||||
assert.match(sql, /SET status = \$2::varchar/);
|
||||
assert.match(sql, /WHEN \$2::varchar = 'running'/);
|
||||
return { rowCount: 1, rows: [] };
|
||||
}
|
||||
if (sql.includes('INSERT INTO "ql3"."run_events"')) {
|
||||
return { rowCount: 1, rows: [] };
|
||||
}
|
||||
throw new Error(`unexpected query: ${sql}`);
|
||||
},
|
||||
release() {},
|
||||
};
|
||||
const repository = new PostgresRemoteRunActivationRepository({
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
async query() {
|
||||
throw new Error('not used');
|
||||
},
|
||||
});
|
||||
const result = await repository.acknowledgeRunning({
|
||||
runId: 'run-live',
|
||||
attemptId: 'attempt-live',
|
||||
workerId: 'worker-live',
|
||||
workerSessionId: '019f7094-a853-72f3-82ab-dfa08e6bd1c1',
|
||||
workerGeneration: 1,
|
||||
offerId: 'offer-live',
|
||||
leaseGeneration: 1,
|
||||
leaseToken: token,
|
||||
expectedLeaseVersion: 0,
|
||||
attemptEventId: '019f7094-a853-72f3-82ab-dfa08e6bd1c2',
|
||||
runEventId: '019f7094-a853-72f3-82ab-dfa08e6bd1c3',
|
||||
executorHandle: 'ql3lp1.test',
|
||||
logArtifactId: 'wlog-0123456789abcdef0123456789abcd',
|
||||
callbackSequence: 1,
|
||||
callbackTokenDigest: 'b'.repeat(64),
|
||||
});
|
||||
assert.equal(result.status, 'applied');
|
||||
});
|
||||
@@ -0,0 +1,94 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PostgresRemoteWorkerAttestationEvidenceProvider,
|
||||
} = require('../dist');
|
||||
|
||||
function target() {
|
||||
return {
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
attemptNumber: 1,
|
||||
executorType: 'remote-worker',
|
||||
executorHandle: 'remote:handle-1',
|
||||
callbackSequence: 5,
|
||||
workerId: 'edge-1',
|
||||
workerSessionId: '018f5c64-9b9d-7f1a-8c2d-1234567890ac',
|
||||
workerGeneration: 2,
|
||||
leaseTokenDigest: 'a'.repeat(64),
|
||||
leaseGeneration: 3,
|
||||
leaseVersion: 4,
|
||||
offerId: 'offer-1',
|
||||
};
|
||||
}
|
||||
|
||||
function attestation(state, receivedAtMs = 9_000) {
|
||||
return {
|
||||
attestationId: '018f5c64-9b9d-7f1a-8c2d-1234567890ab',
|
||||
...target(),
|
||||
sequence: 1,
|
||||
state,
|
||||
journalRevision: 6,
|
||||
receivedAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
function provider(value, observedAtMs = '10000') {
|
||||
return new PostgresRemoteWorkerAttestationEvidenceProvider(
|
||||
{ async query() { return { rows: [{ observedAtMs }] }; } },
|
||||
{
|
||||
async findLatestExact(observed) {
|
||||
assert.deepEqual(observed, {
|
||||
runId: 'run-1', attemptId: 'attempt-1', workerId: 'edge-1',
|
||||
workerSessionId: target().workerSessionId, workerGeneration: 2,
|
||||
leaseTokenDigest: 'a'.repeat(64), leaseGeneration: 3,
|
||||
leaseVersion: 4, offerId: 'offer-1', callbackSequence: 5,
|
||||
executorHandle: 'remote:handle-1',
|
||||
});
|
||||
return value;
|
||||
},
|
||||
async submit() { throw new Error('not used'); },
|
||||
},
|
||||
{ runningFreshnessMs: 2_000 },
|
||||
);
|
||||
}
|
||||
|
||||
test('treats only exact stopped attestation as authoritative not-running', async () => {
|
||||
const context = { signal: new AbortController().signal };
|
||||
assert.deepEqual(await provider(attestation('stopped')).inspect(target(), context), {
|
||||
status: 'not_running',
|
||||
});
|
||||
assert.deepEqual(await provider(attestation('running')).inspect(target(), context), {
|
||||
status: 'running',
|
||||
});
|
||||
});
|
||||
|
||||
test('keeps missing, stale, malformed-time and cancelled evidence fail-closed', async () => {
|
||||
const context = { signal: new AbortController().signal };
|
||||
for (const candidate of [
|
||||
provider(null),
|
||||
provider(attestation('running', 1_000)),
|
||||
provider(attestation('running'), 'corrupt'),
|
||||
]) {
|
||||
assert.deepEqual(await candidate.inspect(target(), context), {
|
||||
status: 'unknown',
|
||||
reason: 'provider_unavailable',
|
||||
});
|
||||
}
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
assert.deepEqual(await provider(attestation('running')).inspect(target(), {
|
||||
signal: controller.signal,
|
||||
}), { status: 'unknown', reason: 'provider_unavailable' });
|
||||
});
|
||||
|
||||
test('rejects an unbounded attestation freshness window', () => {
|
||||
assert.throws(
|
||||
() => new PostgresRemoteWorkerAttestationEvidenceProvider(
|
||||
{ async query() { return { rows: [] }; } },
|
||||
{ async findLatestExact() { return null; }, async submit() {} },
|
||||
{ runningFreshnessMs: 300_001 },
|
||||
),
|
||||
/freshness is invalid/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,606 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
RemoteWorkerCompletionFenceRejectedError,
|
||||
RemoteWorkerCompletionUnavailableError,
|
||||
} = require('@qinglong/runtime-core/remote-worker-completion');
|
||||
const {
|
||||
createStepRunRecord,
|
||||
transitionStepRunRecord,
|
||||
} = require('@qinglong/runtime-core/step-run');
|
||||
const { PostgresRemoteWorkerCompletionRepository } = require('../dist/entrypoints/runtime');
|
||||
|
||||
const SESSION_ID = '018f5c64-9b9d-7f1a-8c2d-1234567890ac';
|
||||
const LEASE_TOKEN = 'worker_generated_lease_capability_0000000000000001';
|
||||
const LEASE_DIGEST = createHash('sha256').update(LEASE_TOKEN).digest('hex');
|
||||
const LOG_ARTIFACT_ID = `wlog-${'a'.repeat(30)}`;
|
||||
const ARTIFACT_SHA256 = 'b'.repeat(64);
|
||||
const CALLBACK_DIGEST = 'c'.repeat(64);
|
||||
|
||||
function fence() {
|
||||
return {
|
||||
workerId: 'worker-1',
|
||||
workerSessionId: SESSION_ID,
|
||||
workerGeneration: 2,
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
offerId: 'offer-1',
|
||||
leaseGeneration: 3,
|
||||
leaseToken: LEASE_TOKEN,
|
||||
expectedLeaseVersion: 4,
|
||||
};
|
||||
}
|
||||
|
||||
function uploadCommand() {
|
||||
return {
|
||||
...fence(),
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
byteLength: 10,
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
|
||||
function completionCommand(overrides = {}) {
|
||||
return {
|
||||
...fence(),
|
||||
callbackSequence: 1,
|
||||
callbackTokenDigest: CALLBACK_DIGEST,
|
||||
result: {
|
||||
outcome: 'succeeded',
|
||||
startedAtMs: 100,
|
||||
finishedAtMs: 200,
|
||||
exitCode: 0,
|
||||
},
|
||||
artifact: {
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
byteLength: 10,
|
||||
sha256: ARTIFACT_SHA256,
|
||||
truncated: false,
|
||||
},
|
||||
attemptEventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a1',
|
||||
runEventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a2',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function aggregate(overrides = {}) {
|
||||
return {
|
||||
runId: 'run-1',
|
||||
projectId: 'project-1',
|
||||
runStatus: 'dispatching',
|
||||
executionOwner: 'runtime',
|
||||
cancelRequestedAtMs: null,
|
||||
cancelReason: null,
|
||||
runErrorCode: null,
|
||||
runVersion: 3,
|
||||
eventSequence: 7,
|
||||
runCreatedAtMs: 40,
|
||||
runStartedAtMs: null,
|
||||
attemptId: 'attempt-1',
|
||||
attemptRunId: 'run-1',
|
||||
attemptStatus: 'starting',
|
||||
executorType: 'remote_worker',
|
||||
attemptWorkerId: 'worker-1',
|
||||
attemptWorkerSessionId: SESSION_ID,
|
||||
attemptWorkerGeneration: 2,
|
||||
attemptLeaseGeneration: 3,
|
||||
attemptLeaseVersion: 4,
|
||||
attemptLeaseTokenDigest: LEASE_DIGEST,
|
||||
attemptOfferId: 'offer-1',
|
||||
callbackSequence: 0,
|
||||
callbackTokenDigest: null,
|
||||
logArtifactId: null,
|
||||
attemptCreatedAtMs: 50,
|
||||
startedAtMs: null,
|
||||
finishedAtMs: null,
|
||||
exitCode: null,
|
||||
attemptErrorCode: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function lease(overrides = {}) {
|
||||
return {
|
||||
attemptId: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
leaseStatus: 'leased',
|
||||
leaseVersion: 4,
|
||||
leaseGeneration: 3,
|
||||
workerId: 'worker-1',
|
||||
workerSessionId: SESSION_ID,
|
||||
workerGeneration: 2,
|
||||
leaseTokenDigest: LEASE_DIGEST,
|
||||
offerId: 'offer-1',
|
||||
leaseExpiresAtMs: 2_000,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function replayPayload(command, overrides = {}) {
|
||||
return {
|
||||
attempt_id: command.attemptId,
|
||||
lease_generation: command.leaseGeneration,
|
||||
from_status: 'running',
|
||||
to_status: 'succeeded',
|
||||
callback_sequence: command.callbackSequence,
|
||||
callback_token_digest: command.callbackTokenDigest,
|
||||
worker_started_at_ms: command.result.startedAtMs,
|
||||
worker_finished_at_ms: command.result.finishedAtMs,
|
||||
exit_code: command.result.exitCode,
|
||||
log_artifact_id: command.artifact.logArtifactId,
|
||||
artifact_byte_length: command.artifact.byteLength,
|
||||
artifact_sha256: command.artifact.sha256,
|
||||
artifact_truncated: command.artifact.truncated,
|
||||
error_code: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function workflowReadyStep() {
|
||||
return createStepRunRecord({
|
||||
id: 'workflow-step-1',
|
||||
runId: 'run-1',
|
||||
stepKey: 'collect',
|
||||
kind: 'task',
|
||||
definitionRef: 'pkg:demo:alpha',
|
||||
definitionDigest: 'a'.repeat(64),
|
||||
required: true,
|
||||
initialStatus: 'ready',
|
||||
mutationId: 'workflow-step-created',
|
||||
createdAtMs: 60,
|
||||
});
|
||||
}
|
||||
|
||||
function workflowAggregate(stepRun, overrides = {}) {
|
||||
return aggregate({
|
||||
runStatus: 'running',
|
||||
runVersion: 4,
|
||||
eventSequence: 9,
|
||||
runStartedAtMs: 50,
|
||||
attemptStepRunId: stepRun.id,
|
||||
workflowAttemptId: 'attempt-1',
|
||||
workflowStepRunId: stepRun.id,
|
||||
admittedWorkflowStepVersion: 1,
|
||||
admittedWorkflowStepDigest: workflowReadyStep().stepRunDigest,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
function fixture(options = {}) {
|
||||
const calls = [];
|
||||
const client = {
|
||||
async query(sql, params = []) {
|
||||
const normalized = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ sql: normalized, params });
|
||||
if (
|
||||
normalized === 'BEGIN' ||
|
||||
normalized.startsWith('SET LOCAL') ||
|
||||
normalized === 'COMMIT' ||
|
||||
normalized === 'ROLLBACK' ||
|
||||
normalized.startsWith('SELECT pg_advisory_xact_lock')
|
||||
)
|
||||
return { rows: [], rowCount: 0 };
|
||||
if (normalized.includes('FROM "ql3"."worker_sessions"')) {
|
||||
return {
|
||||
rows: [
|
||||
options.worker ?? {
|
||||
workerId: 'worker-1',
|
||||
sessionId: SESSION_ID,
|
||||
workerGeneration: 2,
|
||||
workerStatus: 'online',
|
||||
workerLeaseExpiresAtMs: 2_000,
|
||||
},
|
||||
],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('INNER JOIN "ql3"."run_attempts"')) {
|
||||
return { rows: [options.aggregate ?? aggregate()], rowCount: 1 };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."step_runs"')) {
|
||||
return {
|
||||
rows: options.stepRun
|
||||
? [
|
||||
{
|
||||
workflowStepVersion: options.stepRun.version,
|
||||
workflowStepDigest: options.stepRun.stepRunDigest,
|
||||
workflowStepJson: options.stepRun,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
rowCount: options.stepRun ? 1 : 0,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."run_dispatch_leases"')) {
|
||||
return { rows: [options.lease ?? lease()], rowCount: 1 };
|
||||
}
|
||||
if (normalized.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: options.nowMs ?? 1_000 }], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
normalized.startsWith('SELECT payload') &&
|
||||
normalized.includes('FROM "ql3"."run_events"')
|
||||
) {
|
||||
return {
|
||||
rows: options.replayPayload
|
||||
? [{ payload: options.replayPayload }]
|
||||
: [],
|
||||
rowCount: options.replayPayload ? 1 : 0,
|
||||
};
|
||||
}
|
||||
if (
|
||||
normalized.startsWith('UPDATE "ql3"."run_dispatch_leases"') ||
|
||||
normalized.startsWith('UPDATE "ql3"."run_attempts"') ||
|
||||
normalized.startsWith('UPDATE "ql3"."runs"') ||
|
||||
normalized.startsWith('UPDATE "ql3"."step_runs"')
|
||||
)
|
||||
return { rows: [], rowCount: options.updateRowCount ?? 1 };
|
||||
if (
|
||||
normalized.startsWith('INSERT INTO "ql3"."run_events"') ||
|
||||
normalized.startsWith('INSERT INTO "ql3"."step_run_mutations"')
|
||||
) {
|
||||
if (options.failEvent) throw new Error('injected event failure');
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`Unexpected SQL: ${normalized}`);
|
||||
},
|
||||
release() {
|
||||
calls.push({ sql: 'RELEASE', params: [] });
|
||||
},
|
||||
};
|
||||
return {
|
||||
repository: new PostgresRemoteWorkerCompletionRepository({
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
}),
|
||||
calls,
|
||||
};
|
||||
}
|
||||
|
||||
test('authorizes Artifact upload only after the shared Attempt lock and DB clock', async () => {
|
||||
const { repository, calls } = fixture();
|
||||
await repository.authorizeArtifactUpload(uploadCommand());
|
||||
const index = (needle) => calls.findIndex(({ sql }) => sql.includes(needle));
|
||||
assert.ok(index('pg_advisory_xact_lock') < index('worker_sessions'));
|
||||
assert.ok(
|
||||
index('worker_sessions') < index('INNER JOIN "ql3"."run_attempts"'),
|
||||
);
|
||||
assert.ok(
|
||||
index('INNER JOIN "ql3"."run_attempts"') < index('run_dispatch_leases'),
|
||||
);
|
||||
assert.ok(index('run_dispatch_leases') < index('statement_timestamp()'));
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql === 'COMMIT'),
|
||||
true,
|
||||
);
|
||||
assert.equal(JSON.stringify(calls).includes(LEASE_TOKEN), false);
|
||||
assert.equal(JSON.stringify(calls).includes(LEASE_DIGEST), false);
|
||||
});
|
||||
|
||||
test('completes directly from the durable starting crash window in one transaction', async () => {
|
||||
const { repository, calls } = fixture();
|
||||
assert.deepEqual(await repository.complete(completionCommand()), {
|
||||
status: 'applied',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
callbackSequence: 1,
|
||||
});
|
||||
const attemptUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."run_attempts"'),
|
||||
);
|
||||
assert.equal(attemptUpdate.params[9], 'succeeded');
|
||||
assert.equal(attemptUpdate.params[10], 5);
|
||||
assert.equal(attemptUpdate.params[11], 1);
|
||||
assert.equal(attemptUpdate.params[12], CALLBACK_DIGEST);
|
||||
assert.equal(attemptUpdate.params[13], LOG_ARTIFACT_ID);
|
||||
assert.equal(attemptUpdate.params[14], 100);
|
||||
assert.equal(attemptUpdate.params[15], 1_000);
|
||||
assert.equal(attemptUpdate.params[16], 0);
|
||||
assert.equal(attemptUpdate.params[19], 'starting');
|
||||
assert.equal(
|
||||
calls.filter(({ sql }) => sql.startsWith('INSERT INTO "ql3"."run_events"'))
|
||||
.length,
|
||||
2,
|
||||
);
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql === 'COMMIT'),
|
||||
true,
|
||||
);
|
||||
assert.equal(JSON.stringify(calls).includes(LEASE_TOKEN), false);
|
||||
assert.equal(JSON.stringify(calls).includes(ARTIFACT_SHA256), true);
|
||||
});
|
||||
|
||||
test('completes a Workflow Task from the starting crash window without terminalizing its parent Run', async () => {
|
||||
const stepRun = workflowReadyStep();
|
||||
const { repository, calls } = fixture({
|
||||
stepRun,
|
||||
aggregate: workflowAggregate(stepRun),
|
||||
});
|
||||
assert.equal(
|
||||
(await repository.complete(completionCommand())).status,
|
||||
'applied',
|
||||
);
|
||||
|
||||
const runUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"'),
|
||||
);
|
||||
assert.match(runUpdate.sql, /SET version = \$2, event_sequence = \$3/);
|
||||
assert.equal(runUpdate.sql.includes('SET status ='), false);
|
||||
assert.deepEqual(runUpdate.params, ['run-1', 7, 12, 4]);
|
||||
|
||||
const stepUpdates = calls.filter(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."step_runs"'),
|
||||
);
|
||||
assert.equal(stepUpdates.length, 2);
|
||||
assert.equal(stepUpdates[0].params[0], 'running');
|
||||
assert.equal(stepUpdates[0].params[2], 1);
|
||||
assert.equal(stepUpdates[1].params[0], 'succeeded');
|
||||
assert.equal(stepUpdates[1].params[2], 1);
|
||||
assert.equal(
|
||||
calls.filter(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."step_run_mutations"'),
|
||||
).length,
|
||||
2,
|
||||
);
|
||||
|
||||
const attemptEvent = calls.find(
|
||||
({ sql, params }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"') &&
|
||||
params[3] === 'workflow.task_attempt.succeeded',
|
||||
);
|
||||
assert.equal(attemptEvent.params[2], 11);
|
||||
assert.equal(attemptEvent.params[7], stepRun.id);
|
||||
assert.match(attemptEvent.params[8], /"execution_scope":"workflow_task"/);
|
||||
assert.match(attemptEvent.params[8], /"step_run_id":"workflow-step-1"/);
|
||||
});
|
||||
|
||||
test('makes parent Workflow cancellation win over a successful stopped Worker completion', async () => {
|
||||
const ready = workflowReadyStep();
|
||||
const running = transitionStepRunRecord(ready, {
|
||||
expectedVersion: ready.version,
|
||||
expectedDigest: ready.stepRunDigest,
|
||||
mutationId: 'workflow-step-running-before-stop',
|
||||
to: 'running',
|
||||
atMs: 100,
|
||||
});
|
||||
const { repository, calls } = fixture({
|
||||
stepRun: running,
|
||||
aggregate: workflowAggregate(running, {
|
||||
attemptStatus: 'running',
|
||||
callbackSequence: 1,
|
||||
callbackTokenDigest: CALLBACK_DIGEST,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
cancelRequestedAtMs: 150,
|
||||
cancelReason: 'user',
|
||||
startedAtMs: 100,
|
||||
}),
|
||||
});
|
||||
assert.equal(
|
||||
(await repository.complete(completionCommand())).status,
|
||||
'applied',
|
||||
);
|
||||
|
||||
const attemptUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."run_attempts"'),
|
||||
);
|
||||
assert.equal(attemptUpdate.params[9], 'cancelled');
|
||||
assert.equal(attemptUpdate.params[17], 'EXECUTION_CANCELLED');
|
||||
const runUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"'),
|
||||
);
|
||||
assert.match(runUpdate.sql, /SET version = \$2, event_sequence = \$3/);
|
||||
assert.equal(runUpdate.sql.includes('SET status ='), false);
|
||||
const stepUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."step_runs"'),
|
||||
);
|
||||
assert.equal(stepUpdate.params[0], 'cancelled');
|
||||
const attemptEvent = calls.find(
|
||||
({ sql, params }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"') &&
|
||||
params[3] === 'workflow.task_attempt.cancelled',
|
||||
);
|
||||
assert.equal(attemptEvent.params[7], running.id);
|
||||
assert.match(attemptEvent.params[8], /"error_code":"EXECUTION_CANCELLED"/);
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql === 'COMMIT'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('makes timeout intent win over a successful Worker exit', async () => {
|
||||
const running = aggregate({
|
||||
runStatus: 'running',
|
||||
attemptStatus: 'running',
|
||||
cancelRequestedAtMs: 900,
|
||||
cancelReason: 'timeout',
|
||||
callbackSequence: 1,
|
||||
callbackTokenDigest: CALLBACK_DIGEST,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
runStartedAtMs: 100,
|
||||
startedAtMs: 100,
|
||||
});
|
||||
const { repository, calls } = fixture({ aggregate: running });
|
||||
await repository.complete(completionCommand());
|
||||
const attemptUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."run_attempts"'),
|
||||
);
|
||||
assert.equal(attemptUpdate.params[9], 'timed_out');
|
||||
assert.equal(attemptUpdate.params[17], 'EXECUTION_TIMED_OUT');
|
||||
const runUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"'),
|
||||
);
|
||||
assert.equal(runUpdate.params[1], 'timed_out');
|
||||
});
|
||||
|
||||
test('makes a Workflow Task deadline win without timing out its parent Run', async () => {
|
||||
const ready = workflowReadyStep();
|
||||
const running = transitionStepRunRecord(ready, {
|
||||
expectedVersion: ready.version,
|
||||
expectedDigest: ready.stepRunDigest,
|
||||
mutationId: 'workflow-step-running',
|
||||
to: 'running',
|
||||
atMs: 100,
|
||||
});
|
||||
const { repository, calls } = fixture({
|
||||
stepRun: running,
|
||||
aggregate: workflowAggregate(running, {
|
||||
attemptStatus: 'running',
|
||||
callbackSequence: 1,
|
||||
callbackTokenDigest: CALLBACK_DIGEST,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
deadlineAtMs: 950,
|
||||
startedAtMs: 100,
|
||||
}),
|
||||
});
|
||||
await repository.complete(completionCommand());
|
||||
|
||||
const attemptUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."run_attempts"'),
|
||||
);
|
||||
assert.equal(attemptUpdate.params[9], 'timed_out');
|
||||
assert.equal(attemptUpdate.params[17], 'EXECUTION_TIMED_OUT');
|
||||
const stepUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."step_runs"'),
|
||||
);
|
||||
assert.equal(stepUpdate.params[0], 'timed_out');
|
||||
assert.equal(stepUpdate.params[8], 'execution_timed_out');
|
||||
const runUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"'),
|
||||
);
|
||||
assert.equal(runUpdate.sql.includes('SET status ='), false);
|
||||
});
|
||||
|
||||
test('replays a completed Workflow Task from immutable Attempt state after its parent Run is cancelled', async () => {
|
||||
const command = completionCommand();
|
||||
const ready = workflowReadyStep();
|
||||
const running = transitionStepRunRecord(ready, {
|
||||
expectedVersion: ready.version,
|
||||
expectedDigest: ready.stepRunDigest,
|
||||
mutationId: 'workflow-step-running',
|
||||
to: 'running',
|
||||
atMs: 100,
|
||||
});
|
||||
const succeeded = transitionStepRunRecord(running, {
|
||||
expectedVersion: running.version,
|
||||
expectedDigest: running.stepRunDigest,
|
||||
mutationId: command.runEventId,
|
||||
to: 'succeeded',
|
||||
atMs: 1_000,
|
||||
});
|
||||
const aggregateRow = workflowAggregate(succeeded, {
|
||||
runStatus: 'cancelled',
|
||||
cancelRequestedAtMs: 1_200,
|
||||
cancelReason: 'user',
|
||||
runErrorCode: 'EXECUTION_CANCELLED',
|
||||
attemptStatus: 'succeeded',
|
||||
attemptLeaseVersion: 5,
|
||||
callbackSequence: 1,
|
||||
callbackTokenDigest: CALLBACK_DIGEST,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
startedAtMs: 100,
|
||||
finishedAtMs: 1_000,
|
||||
exitCode: 0,
|
||||
});
|
||||
const exact = fixture({
|
||||
aggregate: aggregateRow,
|
||||
stepRun: succeeded,
|
||||
lease: lease({ leaseStatus: 'completed', leaseVersion: 5 }),
|
||||
replayPayload: replayPayload(command, {
|
||||
execution_scope: 'workflow_task',
|
||||
step_run_id: succeeded.id,
|
||||
}),
|
||||
});
|
||||
assert.equal(
|
||||
(await exact.repository.complete(command)).status,
|
||||
'already_completed',
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts only an event-authenticated exact completed replay', async () => {
|
||||
const command = completionCommand();
|
||||
const terminal = aggregate({
|
||||
runStatus: 'succeeded',
|
||||
attemptStatus: 'succeeded',
|
||||
attemptLeaseVersion: 5,
|
||||
callbackSequence: 1,
|
||||
callbackTokenDigest: CALLBACK_DIGEST,
|
||||
logArtifactId: LOG_ARTIFACT_ID,
|
||||
runStartedAtMs: 100,
|
||||
startedAtMs: 100,
|
||||
finishedAtMs: 1_000,
|
||||
exitCode: 0,
|
||||
});
|
||||
const exact = fixture({
|
||||
aggregate: terminal,
|
||||
lease: lease({ leaseStatus: 'completed', leaseVersion: 5 }),
|
||||
replayPayload: replayPayload(command),
|
||||
});
|
||||
assert.equal(
|
||||
(await exact.repository.complete(command)).status,
|
||||
'already_completed',
|
||||
);
|
||||
|
||||
const drifted = fixture({
|
||||
aggregate: terminal,
|
||||
lease: lease({ leaseStatus: 'completed', leaseVersion: 5 }),
|
||||
replayPayload: replayPayload(command, { artifact_sha256: 'd'.repeat(64) }),
|
||||
});
|
||||
await assert.rejects(
|
||||
drifted.repository.complete(command),
|
||||
(error) =>
|
||||
error instanceof RemoteWorkerCompletionFenceRejectedError &&
|
||||
error.reason === 'replay_mismatch',
|
||||
);
|
||||
|
||||
const invalidOrigin = fixture({
|
||||
aggregate: terminal,
|
||||
lease: lease({ leaseStatus: 'completed', leaseVersion: 5 }),
|
||||
replayPayload: replayPayload(command, { from_status: 'queued' }),
|
||||
});
|
||||
await assert.rejects(
|
||||
invalidOrigin.repository.complete(command),
|
||||
(error) =>
|
||||
error instanceof RemoteWorkerCompletionFenceRejectedError &&
|
||||
error.reason === 'replay_mismatch',
|
||||
);
|
||||
});
|
||||
|
||||
test('fences future Worker timestamps and rolls storage failures back', async () => {
|
||||
const future = fixture();
|
||||
await assert.rejects(
|
||||
future.repository.complete(
|
||||
completionCommand({
|
||||
result: {
|
||||
outcome: 'succeeded',
|
||||
startedAtMs: 100,
|
||||
finishedAtMs: 302_000,
|
||||
exitCode: 0,
|
||||
},
|
||||
}),
|
||||
),
|
||||
(error) =>
|
||||
error instanceof RemoteWorkerCompletionFenceRejectedError &&
|
||||
error.reason === 'state_mismatch',
|
||||
);
|
||||
|
||||
const failed = fixture({ failEvent: true });
|
||||
await assert.rejects(
|
||||
failed.repository.complete(completionCommand()),
|
||||
(error) =>
|
||||
error instanceof RemoteWorkerCompletionUnavailableError &&
|
||||
error.cause?.message === 'injected event failure',
|
||||
);
|
||||
assert.equal(
|
||||
failed.calls.some(({ sql }) => sql === 'ROLLBACK'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
failed.calls.some(({ sql }) => sql === 'COMMIT'),
|
||||
false,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,324 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
RemoteWorkerLeaseControlFenceRejectedError,
|
||||
} = require('@qinglong/runtime-core/remote-worker-lease-control');
|
||||
const {
|
||||
createStepRunRecord,
|
||||
transitionStepRunRecord,
|
||||
} = require('@qinglong/runtime-core/step-run');
|
||||
const {
|
||||
PostgresRemoteWorkerLeaseControlRepository,
|
||||
} = require('../dist/entrypoints/runtime');
|
||||
|
||||
const SESSION_ID = '018f5c64-9b9d-7f1a-8c2d-1234567890ac';
|
||||
const LEASE_TOKEN = 'worker_generated_lease_capability_0000000000000001';
|
||||
const LEASE_DIGEST = createHash('sha256').update(LEASE_TOKEN).digest('hex');
|
||||
|
||||
function command() {
|
||||
return {
|
||||
workerId: 'worker-1',
|
||||
workerSessionId: SESSION_ID,
|
||||
workerGeneration: 2,
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
offerId: 'offer-1',
|
||||
leaseGeneration: 3,
|
||||
leaseToken: LEASE_TOKEN,
|
||||
expectedLeaseVersion: 4,
|
||||
leaseDurationMs: 30_000,
|
||||
timeoutEventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a1',
|
||||
};
|
||||
}
|
||||
|
||||
function aggregate(overrides = {}) {
|
||||
return {
|
||||
runId: 'run-1',
|
||||
projectId: 'project-1',
|
||||
runStatus: 'running',
|
||||
executionOwner: 'runtime',
|
||||
cancelRequestedAtMs: null,
|
||||
cancelReason: null,
|
||||
runVersion: 3,
|
||||
eventSequence: 7,
|
||||
attemptId: 'attempt-1',
|
||||
attemptRunId: 'run-1',
|
||||
attemptStatus: 'running',
|
||||
executorType: 'remote_worker',
|
||||
attemptWorkerId: 'worker-1',
|
||||
attemptWorkerSessionId: SESSION_ID,
|
||||
attemptWorkerGeneration: 2,
|
||||
attemptLeaseGeneration: 3,
|
||||
attemptLeaseVersion: 4,
|
||||
attemptLeaseTokenDigest: LEASE_DIGEST,
|
||||
attemptOfferId: 'offer-1',
|
||||
deadlineAtMs: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function lease(overrides = {}) {
|
||||
return {
|
||||
runId: 'run-1',
|
||||
leaseStatus: 'leased',
|
||||
leaseVersion: 4,
|
||||
leaseGeneration: 3,
|
||||
workerId: 'worker-1',
|
||||
workerSessionId: SESSION_ID,
|
||||
workerGeneration: 2,
|
||||
leaseTokenDigest: LEASE_DIGEST,
|
||||
offerId: 'offer-1',
|
||||
leaseExpiresAtMs: 2_000,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function workflowSteps() {
|
||||
const ready = createStepRunRecord({
|
||||
id: 'workflow-step-1',
|
||||
runId: 'run-1',
|
||||
stepKey: 'collect',
|
||||
kind: 'task',
|
||||
definitionRef: 'pkg:demo:alpha',
|
||||
definitionDigest: 'a'.repeat(64),
|
||||
required: true,
|
||||
initialStatus: 'ready',
|
||||
mutationId: 'workflow-step-created',
|
||||
createdAtMs: 100,
|
||||
});
|
||||
const running = transitionStepRunRecord(ready, {
|
||||
expectedVersion: ready.version,
|
||||
expectedDigest: ready.stepRunDigest,
|
||||
mutationId: 'workflow-step-running',
|
||||
to: 'running',
|
||||
atMs: 200,
|
||||
});
|
||||
return { ready, running };
|
||||
}
|
||||
|
||||
function fixture(options = {}) {
|
||||
const calls = [];
|
||||
const client = {
|
||||
async query(sql, params = []) {
|
||||
const normalized = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ sql: normalized, params });
|
||||
if (
|
||||
normalized === 'BEGIN' || normalized === 'COMMIT' ||
|
||||
normalized === 'ROLLBACK' || normalized.startsWith('SET LOCAL') ||
|
||||
normalized.startsWith('SELECT pg_advisory_xact_lock')
|
||||
) return { rows: [], rowCount: 0 };
|
||||
if (normalized.includes('FROM "ql3"."worker_sessions"')) {
|
||||
return {
|
||||
rows: [options.worker ?? {
|
||||
workerId: 'worker-1',
|
||||
sessionId: SESSION_ID,
|
||||
workerGeneration: 2,
|
||||
workerStatus: 'online',
|
||||
workerLeaseExpiresAtMs: 2_000,
|
||||
}],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('INNER JOIN "ql3"."run_attempts"')) {
|
||||
return { rows: [options.aggregate ?? aggregate()], rowCount: 1 };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."step_runs"')) {
|
||||
return {
|
||||
rows: options.stepRun
|
||||
? [{
|
||||
workflowStepVersion: options.stepRun.version,
|
||||
workflowStepDigest: options.stepRun.stepRunDigest,
|
||||
workflowStepJson: options.stepRun,
|
||||
}]
|
||||
: [],
|
||||
rowCount: options.stepRun ? 1 : 0,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."run_dispatch_leases"')) {
|
||||
return { rows: [options.lease ?? lease()], rowCount: 1 };
|
||||
}
|
||||
if (normalized.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: options.nowMs ?? 1_000 }], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
normalized.startsWith('SELECT created_at_ms') &&
|
||||
normalized.includes('FROM "ql3"."run_events"')
|
||||
) {
|
||||
return {
|
||||
rows: options.existingTimeoutAt === undefined
|
||||
? []
|
||||
: [{ createdAtMs: options.existingTimeoutAt }],
|
||||
rowCount: options.existingTimeoutAt === undefined ? 0 : 1,
|
||||
};
|
||||
}
|
||||
if (
|
||||
normalized.startsWith('UPDATE "ql3"."run_dispatch_leases"') &&
|
||||
normalized.includes('RETURNING')
|
||||
) {
|
||||
return {
|
||||
rows: [{ renewedAtMs: options.nowMs ?? 1_000, expiresAtMs: 31_000 }],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (normalized.startsWith('UPDATE "ql3"."runs"')) {
|
||||
return { rows: [], rowCount: options.runUpdateCount ?? 1 };
|
||||
}
|
||||
if (normalized.startsWith('UPDATE "ql3"."run_attempts"')) {
|
||||
return { rows: [], rowCount: options.attemptUpdateCount ?? 1 };
|
||||
}
|
||||
if (normalized.startsWith('INSERT INTO "ql3"."run_events"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`Unexpected SQL: ${normalized}`);
|
||||
},
|
||||
release() { calls.push({ sql: 'RELEASE', params: [] }); },
|
||||
};
|
||||
return {
|
||||
repository: new PostgresRemoteWorkerLeaseControlRepository({
|
||||
async connect() { return client; },
|
||||
}),
|
||||
calls,
|
||||
};
|
||||
}
|
||||
|
||||
test('renews one exact live Lease without creating control intent', async () => {
|
||||
const { repository, calls } = fixture();
|
||||
assert.deepEqual(await repository.control(command()), {
|
||||
status: 'renewed',
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
offerId: 'offer-1',
|
||||
leaseGeneration: 3,
|
||||
leaseVersion: 5,
|
||||
renewedAtMs: 1_000,
|
||||
expiresAtMs: 31_000,
|
||||
});
|
||||
assert.equal(calls.some(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"')), false);
|
||||
assert.equal(calls.some(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"')), false);
|
||||
assert.equal(JSON.stringify(calls).includes(LEASE_TOKEN), false);
|
||||
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), true);
|
||||
});
|
||||
|
||||
test('projects an existing durable cancellation while renewing', async () => {
|
||||
const { repository } = fixture({
|
||||
aggregate: aggregate({ cancelRequestedAtMs: 900, cancelReason: 'user' }),
|
||||
});
|
||||
const result = await repository.control(command());
|
||||
assert.equal(result.status, 'stop_requested');
|
||||
assert.deepEqual(result.stop, { reason: 'user', requestedAtMs: 900 });
|
||||
assert.equal(result.leaseVersion, 5);
|
||||
});
|
||||
|
||||
test('persists one due deadline as timeout intent before returning stop', async () => {
|
||||
const { repository, calls } = fixture({
|
||||
aggregate: aggregate({ deadlineAtMs: 950 }),
|
||||
});
|
||||
const result = await repository.control(command());
|
||||
assert.equal(result.status, 'stop_requested');
|
||||
assert.deepEqual(result.stop, { reason: 'timeout', requestedAtMs: 1_000 });
|
||||
const cancel = calls.find(({ sql }) => sql.startsWith('UPDATE "ql3"."runs"'));
|
||||
assert.deepEqual(cancel.params, ['run-1', 1_000, 4, 8, 3]);
|
||||
const event = calls.find(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"'));
|
||||
assert.equal(event.params[0], command().timeoutEventId);
|
||||
assert.equal(event.params[3], 'remote-timeout:attempt-1:3');
|
||||
});
|
||||
|
||||
test('stops one timed-out Workflow Task without cancelling its parent Run', async () => {
|
||||
const { ready, running } = workflowSteps();
|
||||
const { repository, calls } = fixture({
|
||||
stepRun: running,
|
||||
aggregate: aggregate({
|
||||
deadlineAtMs: 950,
|
||||
attemptStepRunId: running.id,
|
||||
workflowAttemptId: 'attempt-1',
|
||||
workflowStepRunId: running.id,
|
||||
admittedWorkflowStepVersion: ready.version,
|
||||
admittedWorkflowStepDigest: ready.stepRunDigest,
|
||||
}),
|
||||
});
|
||||
const result = await repository.control(command());
|
||||
assert.equal(result.status, 'stop_requested');
|
||||
assert.deepEqual(result.stop, {
|
||||
reason: 'timeout',
|
||||
requestedAtMs: 1_000,
|
||||
});
|
||||
|
||||
const runUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"'));
|
||||
assert.match(runUpdate.sql, /SET version = \$2, event_sequence = \$3/);
|
||||
assert.equal(runUpdate.sql.includes('cancel_requested_at_ms ='), false);
|
||||
assert.deepEqual(runUpdate.params, ['run-1', 4, 8, 3]);
|
||||
|
||||
const event = calls.find(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"'));
|
||||
assert.match(event.sql, /workflow\.task_timeout_requested/);
|
||||
assert.equal(event.params[5], running.id);
|
||||
assert.match(event.params[6], /"execution_scope":"workflow_task"/);
|
||||
});
|
||||
|
||||
test('returns an exact terminal projection without renewing', async () => {
|
||||
const { repository, calls } = fixture({
|
||||
aggregate: aggregate({
|
||||
runStatus: 'cancelled',
|
||||
attemptStatus: 'cancelled',
|
||||
}),
|
||||
lease: lease({ leaseStatus: 'completed', leaseVersion: 5 }),
|
||||
});
|
||||
assert.deepEqual(await repository.control(command()), {
|
||||
status: 'terminal',
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
offerId: 'offer-1',
|
||||
leaseGeneration: 3,
|
||||
terminalStatus: 'cancelled',
|
||||
});
|
||||
assert.equal(calls.some(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."run_dispatch_leases"')), false);
|
||||
});
|
||||
|
||||
test('fences stale versions and rolls the transaction back', async () => {
|
||||
const { repository, calls } = fixture({
|
||||
lease: lease({ leaseVersion: 5 }),
|
||||
});
|
||||
await assert.rejects(
|
||||
repository.control(command()),
|
||||
(error) =>
|
||||
error instanceof RemoteWorkerLeaseControlFenceRejectedError &&
|
||||
error.reason === 'version_mismatch',
|
||||
);
|
||||
assert.equal(calls.some(({ sql }) => sql === 'ROLLBACK'), true);
|
||||
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), false);
|
||||
});
|
||||
|
||||
test('preserves granular identity fence reasons without exposing capability data', async () => {
|
||||
const cases = [
|
||||
[{ worker: { workerId: 'worker-1', sessionId: SESSION_ID,
|
||||
workerGeneration: 3, workerStatus: 'online', workerLeaseExpiresAtMs: 2_000 } },
|
||||
'worker_generation_mismatch'],
|
||||
[{ aggregate: aggregate({ projectId: 'project-other' }) }, 'project_mismatch'],
|
||||
[{ aggregate: aggregate({ executionOwner: 'legacy' }) }, 'execution_owner_mismatch'],
|
||||
[{ lease: lease({ workerSessionId: '018f5c64-9b9d-7f1a-8c2d-1234567890ad' }) },
|
||||
'worker_session_mismatch'],
|
||||
[{ aggregate: aggregate({ attemptOfferId: 'offer-other' }) }, 'offer_mismatch'],
|
||||
[{ lease: lease({ leaseTokenDigest: 'f'.repeat(64) }) }, 'lease_token_mismatch'],
|
||||
];
|
||||
for (const [options, reason] of cases) {
|
||||
const { repository } = fixture(options);
|
||||
await assert.rejects(
|
||||
repository.control(command()),
|
||||
(error) =>
|
||||
error instanceof RemoteWorkerLeaseControlFenceRejectedError &&
|
||||
error.reason === reason &&
|
||||
!error.message.includes(LEASE_TOKEN),
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,142 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
createClusterTaskExecutionRevision,
|
||||
} = require('@qinglong/runtime-core/cluster-execution-revision');
|
||||
const {
|
||||
digestRunDispatchLeaseToken,
|
||||
} = require('@qinglong/runtime-core/run-dispatch-lease');
|
||||
const { createSecretRef } = require('@qinglong/runtime-core/secret-reference');
|
||||
const {
|
||||
PostgresRemoteWorkerSecretDeliveryAuthorityRepository,
|
||||
} = require('../dist/worker-credential/remoteWorkerSecretDeliveryRepository');
|
||||
|
||||
const SESSION_ID = '018f0000-0000-7000-8000-000000000001';
|
||||
const SOURCE_DIGEST = 'a'.repeat(64);
|
||||
const TASK_REVISION = `qltd:v1:1:${SOURCE_DIGEST}`;
|
||||
const LEASE_TOKEN = 'worker_generated_lease_capability_0000000000000001';
|
||||
const SECRET_REF = createSecretRef({ projectId: 'project-1', name: 'token' });
|
||||
|
||||
function revision() {
|
||||
return createClusterTaskExecutionRevision({
|
||||
projectId: 'project-1', taskId: 'task-1', taskRevision: TASK_REVISION,
|
||||
sourceRevision: 1, sourceContentDigest: SOURCE_DIGEST,
|
||||
executorType: 'remote_worker', planSchema: 'qinglong/command-execution@v1',
|
||||
command: { kind: 'argv', file: '/bin/true', args: [] },
|
||||
environment: [{ name: 'TOKEN', kind: 'secret', secretRef: SECRET_REF }],
|
||||
createdAtMs: 1,
|
||||
});
|
||||
}
|
||||
|
||||
function command(executionDigest, overrides = {}) {
|
||||
return {
|
||||
workerId: 'edge-1', workerSessionId: SESSION_ID, workerGeneration: 2,
|
||||
runId: 'run-1', attemptId: 'attempt-1', projectId: 'project-1',
|
||||
taskId: 'task-1', taskRevision: TASK_REVISION, executionDigest,
|
||||
offerId: 'offer-1', leaseGeneration: 3, leaseToken: LEASE_TOKEN,
|
||||
expectedLeaseVersion: 4, secretRefs: [SECRET_REF], ...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function authorityRow(plan, overrides = {}) {
|
||||
return {
|
||||
observedAtMs: '1000', runId: 'run-1', runProjectId: 'project-1',
|
||||
runTaskId: 'task-1', runTaskRevision: TASK_REVISION,
|
||||
runStatus: 'dispatching', executionOwner: 'runtime',
|
||||
cancelRequestedAtMs: null, attemptStatus: 'starting',
|
||||
attemptExecutorType: 'remote_worker', attemptWorkerId: 'edge-1',
|
||||
attemptWorkerSessionId: SESSION_ID, attemptWorkerGeneration: 2,
|
||||
attemptLeaseTokenDigest: digestRunDispatchLeaseToken(LEASE_TOKEN),
|
||||
attemptLeaseGeneration: 3, attemptLeaseVersion: 4,
|
||||
attemptOfferId: 'offer-1', sessionId: SESSION_ID, sessionGeneration: 2,
|
||||
sessionStatus: 'online', sessionExpiresAtMs: '5000',
|
||||
leaseRunId: 'run-1', leaseStatus: 'leased', leaseVersion: 4,
|
||||
leaseGeneration: 3, leaseWorkerId: 'edge-1',
|
||||
leaseWorkerSessionId: SESSION_ID, leaseWorkerGeneration: 2,
|
||||
leaseTokenDigest: digestRunDispatchLeaseToken(LEASE_TOKEN),
|
||||
leaseOfferId: 'offer-1', leaseExpiresAtMs: '5000',
|
||||
revisionProjectId: plan.projectId, revisionTaskId: plan.taskId,
|
||||
sourceRevision: plan.sourceRevision,
|
||||
revisionTaskRevision: plan.taskRevision,
|
||||
sourceContentDigest: plan.sourceContentDigest,
|
||||
revisionExecutorType: plan.executorType, planSchema: plan.planSchema,
|
||||
planJson: {
|
||||
command: plan.command,
|
||||
environment: plan.environment,
|
||||
...(plan.workingDirectory === undefined ? {} : { workingDirectory: plan.workingDirectory }),
|
||||
...(plan.timeoutMs === undefined ? {} : { timeoutMs: plan.timeoutMs }),
|
||||
...(plan.placement === undefined ? {} : { placement: plan.placement }),
|
||||
},
|
||||
revisionContentDigest: plan.contentDigest,
|
||||
revisionCreatedAtMs: String(plan.createdAtMs),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(row) {
|
||||
const queries = [];
|
||||
let released = 0;
|
||||
const client = {
|
||||
async query(sql, params) {
|
||||
queries.push({ sql, params });
|
||||
if (sql.includes('FROM observation')) return { rows: row ? [row] : [] };
|
||||
return { rows: [] };
|
||||
},
|
||||
release() { released += 1; },
|
||||
};
|
||||
return {
|
||||
repository: new PostgresRemoteWorkerSecretDeliveryAuthorityRepository({
|
||||
async connect() { return client; },
|
||||
}),
|
||||
queries,
|
||||
released: () => released,
|
||||
};
|
||||
}
|
||||
|
||||
test('authorizes exact Session, Lease and immutable execution revision fences', async () => {
|
||||
const plan = revision();
|
||||
const { repository, queries, released } = fixture(authorityRow(plan));
|
||||
const result = await repository.authorize(command(plan.contentDigest));
|
||||
assert.deepEqual(result.secretRefs, [SECRET_REF]);
|
||||
assert.equal(result.executionDigest, plan.contentDigest);
|
||||
assert.equal('leaseToken' in result, false);
|
||||
assert.equal(queries.some(({ sql }) => sql.includes('pg_advisory_xact_lock')), true);
|
||||
assert.equal(queries.at(-1).sql, 'COMMIT');
|
||||
assert.equal(released(), 1);
|
||||
});
|
||||
|
||||
test('rejects expired or replayed Lease authority before returning Secret scope', async () => {
|
||||
const plan = revision();
|
||||
for (const overrides of [
|
||||
{ leaseVersion: 5 },
|
||||
{ leaseExpiresAtMs: '1000' },
|
||||
{ sessionExpiresAtMs: '1000' },
|
||||
{ attemptStatus: 'running' },
|
||||
]) {
|
||||
const { repository, queries } = fixture(authorityRow(plan, overrides));
|
||||
await assert.rejects(
|
||||
repository.authorize(command(plan.contentDigest)),
|
||||
/authority_mismatch/,
|
||||
);
|
||||
assert.equal(queries.at(-1).sql, 'ROLLBACK');
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects partial Secret scope and execution digest drift', async () => {
|
||||
const plan = revision();
|
||||
const extra = createSecretRef({ projectId: 'project-1', name: 'other' });
|
||||
const partial = fixture(authorityRow(plan));
|
||||
await assert.rejects(
|
||||
partial.repository.authorize(command(plan.contentDigest, {
|
||||
secretRefs: [extra],
|
||||
})),
|
||||
/secret_scope_mismatch/,
|
||||
);
|
||||
const digestDrift = fixture(authorityRow(plan));
|
||||
await assert.rejects(
|
||||
digestDrift.repository.authorize(command('b'.repeat(64))),
|
||||
/secret_scope_mismatch/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,221 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
DuplicateIdempotencyKeyError,
|
||||
MAX_RUN_EVENT_PAYLOAD_BYTES,
|
||||
RunEventPayloadTooLargeError,
|
||||
RunRepositoryBusyError,
|
||||
RunRepositoryConstraintError,
|
||||
} = require('@qinglong/runtime-core');
|
||||
const { PostgresRunRepository, PostgresRunTransaction } = require('../dist');
|
||||
|
||||
const RUN = Object.freeze({
|
||||
id: '019f70b0-0000-7000-8000-000000000001',
|
||||
projectId: 'default',
|
||||
taskId: 'task-1',
|
||||
taskRevision: 'revision-1',
|
||||
taskName: 'test',
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
triggeredBy: 'user:1',
|
||||
status: 'created',
|
||||
version: 0,
|
||||
eventSequence: 0,
|
||||
priority: 0,
|
||||
idempotencyKey: 'request-1',
|
||||
createdAtMs: 1_750_000_000_000,
|
||||
});
|
||||
|
||||
const EVENT = Object.freeze({
|
||||
id: '019f70b0-0000-7000-8000-000000000003',
|
||||
runId: RUN.id,
|
||||
sequence: 1,
|
||||
type: 'run.created',
|
||||
dedupeKey: 'run.created',
|
||||
actorType: 'system',
|
||||
payload: Object.freeze({ source: 'postgres-test' }),
|
||||
createdAtMs: 1_750_000_000_002,
|
||||
});
|
||||
|
||||
function driverError(code, constraint) {
|
||||
return Object.assign(new Error('driver failure'), { code, constraint });
|
||||
}
|
||||
|
||||
function clientHarness(handler = async () => ({ rows: [], rowCount: 0 })) {
|
||||
const queries = [];
|
||||
let released = 0;
|
||||
const client = {
|
||||
async query(text, values) {
|
||||
queries.push({ text, values });
|
||||
return handler(text, values, queries.length);
|
||||
},
|
||||
release() {
|
||||
released += 1;
|
||||
},
|
||||
};
|
||||
return {
|
||||
queries,
|
||||
released: () => released,
|
||||
pool: {
|
||||
query: (...args) => client.query(...args),
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function databaseRunRow(overrides = {}) {
|
||||
return {
|
||||
...RUN,
|
||||
createdAtMs: String(RUN.createdAtMs),
|
||||
taskSnapshotRef: null,
|
||||
legacyCronId: null,
|
||||
parentRunId: null,
|
||||
retryOfRunId: null,
|
||||
triggerId: null,
|
||||
requestId: null,
|
||||
scheduledForMs: null,
|
||||
queuedAtMs: null,
|
||||
startedAtMs: null,
|
||||
finishedAtMs: null,
|
||||
cancelRequestedAtMs: null,
|
||||
cancelReason: null,
|
||||
inputRef: null,
|
||||
outputRef: null,
|
||||
errorCode: null,
|
||||
errorSummary: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('orders one bounded PostgreSQL transaction and releases its client', async () => {
|
||||
const harness = clientHarness();
|
||||
const result = await new PostgresRunRepository(harness.pool).transaction(
|
||||
async () => 'committed',
|
||||
);
|
||||
assert.equal(result, 'committed');
|
||||
assert.deepEqual(
|
||||
harness.queries.map(({ text }) => text),
|
||||
[
|
||||
'BEGIN',
|
||||
'SET TRANSACTION ISOLATION LEVEL READ COMMITTED',
|
||||
"SELECT set_config('statement_timeout', $1, true)",
|
||||
"SELECT set_config('lock_timeout', $1, true)",
|
||||
"SELECT set_config('idle_in_transaction_session_timeout', $1, true)",
|
||||
'COMMIT',
|
||||
],
|
||||
);
|
||||
assert.equal(harness.released(), 1);
|
||||
});
|
||||
|
||||
test('rolls back a serialization failure and exposes a stable domain error', async () => {
|
||||
const harness = clientHarness(async (text) => {
|
||||
if (text === 'COMMIT') throw driverError('40001');
|
||||
return { rows: [], rowCount: 0 };
|
||||
});
|
||||
await assert.rejects(
|
||||
new PostgresRunRepository(harness.pool).transaction(async () => 1),
|
||||
RunRepositoryBusyError,
|
||||
);
|
||||
assert.equal(harness.queries.at(-1).text, 'ROLLBACK');
|
||||
assert.equal(harness.released(), 1);
|
||||
});
|
||||
|
||||
test('normalizes bigint rows and treats missing rows as null', async () => {
|
||||
const row = databaseRunRow({ scheduledForMs: '1750000000100' });
|
||||
const harness = clientHarness(async (text, values) => {
|
||||
if (!text.includes('FROM "ql3"."runs"')) {
|
||||
throw new Error(`unexpected query: ${text}`);
|
||||
}
|
||||
return values[0] === RUN.id
|
||||
? { rows: [row], rowCount: 1 }
|
||||
: { rows: [], rowCount: 0 };
|
||||
});
|
||||
const repository = new PostgresRunRepository(harness.pool);
|
||||
assert.deepEqual(await repository.findRunById(RUN.id), {
|
||||
...RUN,
|
||||
scheduledForMs: 1_750_000_000_100,
|
||||
});
|
||||
assert.equal(await repository.findRunById('missing'), null);
|
||||
});
|
||||
|
||||
test('lists one Project with the indexed descending Run keyset', async () => {
|
||||
const harness = clientHarness(async () => ({
|
||||
rows: [
|
||||
databaseRunRow({ id: 'run-c', createdAtMs: '20' }),
|
||||
databaseRunRow({ id: 'run-b', createdAtMs: '20' }),
|
||||
],
|
||||
rowCount: 2,
|
||||
}));
|
||||
const repository = new PostgresRunRepository(harness.pool);
|
||||
const rows = await repository.listRunsByProject({
|
||||
projectId: 'default',
|
||||
limit: 2,
|
||||
after: { createdAtMs: 30, runId: 'run-z' },
|
||||
});
|
||||
assert.deepEqual(
|
||||
rows.map(({ id }) => id),
|
||||
['run-c', 'run-b'],
|
||||
);
|
||||
assert.deepEqual(harness.queries[0].values, ['default', 'run-z', 30, 2]);
|
||||
assert.match(
|
||||
harness.queries[0].text,
|
||||
/ORDER BY "created_at_ms" DESC, "id" DESC\s+LIMIT \$4/u,
|
||||
);
|
||||
await assert.rejects(
|
||||
repository.listRunsByProject({ projectId: 'default', limit: 66 }),
|
||||
TypeError,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects duplicate identity rows and corrupt enum data', async () => {
|
||||
const duplicate = clientHarness(async () => ({
|
||||
rows: [databaseRunRow(), databaseRunRow()],
|
||||
rowCount: 2,
|
||||
}));
|
||||
await assert.rejects(
|
||||
new PostgresRunRepository(duplicate.pool).findRunById(RUN.id),
|
||||
RunRepositoryConstraintError,
|
||||
);
|
||||
|
||||
const corrupt = clientHarness(async () => ({
|
||||
rows: [databaseRunRow({ status: 'invented' })],
|
||||
rowCount: 1,
|
||||
}));
|
||||
await assert.rejects(
|
||||
new PostgresRunRepository(corrupt.pool).findRunById(RUN.id),
|
||||
RunRepositoryConstraintError,
|
||||
);
|
||||
});
|
||||
|
||||
test('maps PostgreSQL constraint names without leaking driver errors', async () => {
|
||||
const transaction = new PostgresRunTransaction({
|
||||
async query() {
|
||||
throw driverError('23505', 'ql3_runs_project_idempotency_uidx');
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
transaction.insertRun(RUN),
|
||||
DuplicateIdempotencyKeyError,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects oversized event payloads before issuing SQL', async () => {
|
||||
let calls = 0;
|
||||
const transaction = new PostgresRunTransaction({
|
||||
async query() {
|
||||
calls += 1;
|
||||
return { rows: [], rowCount: 0 };
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
transaction.appendEvent({
|
||||
...EVENT,
|
||||
payload: { value: 'x'.repeat(MAX_RUN_EVENT_PAYLOAD_BYTES) },
|
||||
}),
|
||||
RunEventPayloadTooLargeError,
|
||||
);
|
||||
assert.equal(calls, 0);
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const { getTableConfig } = require('drizzle-orm/pg-core');
|
||||
const {
|
||||
postgresqlControlSchemaContract,
|
||||
ql3PostgresTables,
|
||||
} = require('../dist');
|
||||
|
||||
function describeTable(table) {
|
||||
const config = getTableConfig(table);
|
||||
const primaryIndexes = config.columns
|
||||
.filter((column) => column.primary)
|
||||
.map(() => `${config.name}_pkey`);
|
||||
return {
|
||||
name: config.name,
|
||||
schema: config.schema,
|
||||
columns: config.columns.map((column) => column.name),
|
||||
indexes: [
|
||||
...primaryIndexes,
|
||||
...config.primaryKeys.map((entry) => entry.getName()),
|
||||
...config.indexes.map((entry) => entry.config.name),
|
||||
],
|
||||
checks: config.checks.map((entry) => entry.name),
|
||||
foreignKeys: config.foreignKeys.map((entry) => entry.getName()),
|
||||
};
|
||||
}
|
||||
|
||||
test('Drizzle schema exactly matches the reviewed ql3 table and index contract', () => {
|
||||
const drizzleTables = ql3PostgresTables.map(describeTable);
|
||||
assert.deepEqual(
|
||||
drizzleTables.map(({ name, schema, columns }) => ({
|
||||
name,
|
||||
schema,
|
||||
columns,
|
||||
})),
|
||||
postgresqlControlSchemaContract.tables.map(({ name, columns }) => ({
|
||||
name,
|
||||
schema: postgresqlControlSchemaContract.schema,
|
||||
columns: [...columns],
|
||||
})),
|
||||
);
|
||||
assert.deepEqual(
|
||||
drizzleTables.flatMap(({ indexes }) => indexes).sort(),
|
||||
[...postgresqlControlSchemaContract.indexes].sort(),
|
||||
);
|
||||
});
|
||||
|
||||
test('Drizzle schema carries every named ql3 check and foreign-key boundary', () => {
|
||||
const drizzleTables = ql3PostgresTables.map(describeTable);
|
||||
const checks = drizzleTables.flatMap((table) => table.checks).sort();
|
||||
const foreignKeys = drizzleTables
|
||||
.flatMap((table) => table.foreignKeys)
|
||||
.sort();
|
||||
assert.deepEqual(checks, [...postgresqlControlSchemaContract.checks].sort());
|
||||
assert.deepEqual(
|
||||
foreignKeys,
|
||||
[...postgresqlControlSchemaContract.foreignKeys].sort(),
|
||||
);
|
||||
assert.equal(
|
||||
drizzleTables.some(({ name }) =>
|
||||
['crontabs', 'envs', 'subscriptions', 'runninginstances'].includes(name),
|
||||
),
|
||||
false,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
SecurityAuditUnavailableError,
|
||||
} = require('@qinglong/runtime-core/security-audit');
|
||||
const {
|
||||
PostgresSecurityAuditRepository,
|
||||
} = require('@qinglong/cluster-postgres/runtime');
|
||||
|
||||
function record(overrides = {}) {
|
||||
return {
|
||||
eventId: '123e4567-e89b-42d3-a456-426614174000',
|
||||
requestId: 'request-1',
|
||||
operationId: 'run.create',
|
||||
projectId: 'default',
|
||||
subject: { type: 'user', id: 'usr_primary' },
|
||||
authenticationId: 'api_credential:primary:1',
|
||||
outcome: 'allowed',
|
||||
reasons: ['role_grant'],
|
||||
fence: { projectVersion: 2, bindingVersion: 3 },
|
||||
occurredAtMs: 1000,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('inserts one normalized low-sensitive audit fact without reading it back', async () => {
|
||||
const calls = [];
|
||||
const repository = new PostgresSecurityAuditRepository({
|
||||
async query(sql, values) {
|
||||
calls.push({ sql, values });
|
||||
return { rows: [] };
|
||||
},
|
||||
});
|
||||
await repository.record(record());
|
||||
assert.equal(calls.length, 1);
|
||||
assert.match(calls[0].sql, /^INSERT INTO "ql3"\."security_audit_events"/);
|
||||
assert.deepEqual(calls[0].values, [
|
||||
'123e4567-e89b-42d3-a456-426614174000',
|
||||
'request-1',
|
||||
'run.create',
|
||||
'default',
|
||||
'user',
|
||||
'usr_primary',
|
||||
'api_credential:primary:1',
|
||||
'allowed',
|
||||
'["role_grant"]',
|
||||
2,
|
||||
3,
|
||||
1000,
|
||||
]);
|
||||
assert.equal(JSON.stringify(calls).includes('secret'), false);
|
||||
});
|
||||
|
||||
test('maps invalid facts and database failures to low-sensitive unavailable', async () => {
|
||||
let calls = 0;
|
||||
const invalid = new PostgresSecurityAuditRepository({
|
||||
async query() {
|
||||
calls += 1;
|
||||
return { rows: [] };
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
invalid.record(record({ reasons: ['driver password leaked'] })),
|
||||
SecurityAuditUnavailableError,
|
||||
);
|
||||
assert.equal(calls, 0);
|
||||
|
||||
const unavailable = new PostgresSecurityAuditRepository({
|
||||
async query() {
|
||||
throw new Error('driver detail');
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
unavailable.record(record()),
|
||||
SecurityAuditUnavailableError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,308 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
TaskStartFenceRejectedError,
|
||||
TaskStartNotFoundError,
|
||||
} = require('@qinglong/runtime-core/task-start');
|
||||
const {
|
||||
createTaskDefinitionRecord,
|
||||
} = require('@qinglong/runtime-core/task-definition');
|
||||
const {
|
||||
createBuiltInTaskSpecSemanticRegistry,
|
||||
} = require('@qinglong/runtime-core/task-spec-semantic');
|
||||
const {
|
||||
compileClusterCommandTaskDefinition,
|
||||
} = require('@qinglong/runtime-core/cluster-execution-revision');
|
||||
const {
|
||||
PostgresTaskStartRepository,
|
||||
} = require('@qinglong/cluster-postgres/task-start');
|
||||
|
||||
const IDS = [
|
||||
'019f7300-0000-7000-8000-000000000801',
|
||||
'019f7300-0000-7000-8000-000000000802',
|
||||
'019f7300-0000-7000-8000-000000000803',
|
||||
'019f7300-0000-7000-8000-000000000804',
|
||||
];
|
||||
const MUTATION_ID = '019f7300-0000-7000-8000-000000000800';
|
||||
const TASK_SEMANTICS = createBuiltInTaskSpecSemanticRegistry();
|
||||
const TASK_COMMAND = {
|
||||
projectId: 'project-1',
|
||||
taskId: 'task-1',
|
||||
expectedRevision: null,
|
||||
mutationId: '019f7300-0000-7000-8000-000000000899',
|
||||
name: 'Task 1',
|
||||
kind: 'command',
|
||||
spec: {
|
||||
schema: 'qinglong/command@v1',
|
||||
config: {
|
||||
command: { kind: 'argv', file: '/bin/echo', args: ['cluster'] },
|
||||
},
|
||||
},
|
||||
labels: {},
|
||||
enabled: true,
|
||||
occurredAtMs: 20,
|
||||
};
|
||||
const DEFINITION = createTaskDefinitionRecord({
|
||||
...TASK_COMMAND,
|
||||
spec: TASK_SEMANTICS.normalize({
|
||||
projectId: TASK_COMMAND.projectId,
|
||||
taskId: TASK_COMMAND.taskId,
|
||||
kind: TASK_COMMAND.kind,
|
||||
spec: TASK_COMMAND.spec,
|
||||
}),
|
||||
}, 10);
|
||||
const DISABLED_DEFINITION = createTaskDefinitionRecord({
|
||||
...TASK_COMMAND,
|
||||
enabled: false,
|
||||
spec: TASK_SEMANTICS.normalize({
|
||||
projectId: TASK_COMMAND.projectId,
|
||||
taskId: TASK_COMMAND.taskId,
|
||||
kind: TASK_COMMAND.kind,
|
||||
spec: TASK_COMMAND.spec,
|
||||
}),
|
||||
}, 10);
|
||||
const EXECUTION = compileClusterCommandTaskDefinition(
|
||||
DEFINITION,
|
||||
TASK_SEMANTICS,
|
||||
);
|
||||
|
||||
function command(overrides = {}) {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
taskId: 'task-1',
|
||||
mutationId: MUTATION_ID,
|
||||
expectedRevision: DEFINITION.revision,
|
||||
expectedContentDigest: DEFINITION.contentDigest,
|
||||
runId: IDS[0],
|
||||
attemptId: IDS[1],
|
||||
createdEventId: IDS[2],
|
||||
queuedEventId: IDS[3],
|
||||
subject: { type: 'user', id: 'user-1' },
|
||||
policyFence: { projectVersion: 2, bindingVersion: 3 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function taskRow(overrides = {}) {
|
||||
return {
|
||||
projectId: DEFINITION.projectId,
|
||||
taskId: DEFINITION.taskId,
|
||||
taskRevision: DEFINITION.revision,
|
||||
definitionMutationId: DEFINITION.mutationId,
|
||||
taskName: DEFINITION.name,
|
||||
description: null,
|
||||
taskKind: DEFINITION.kind,
|
||||
specJson: DEFINITION.spec,
|
||||
labelsJson: DEFINITION.labels,
|
||||
enabled: DEFINITION.enabled,
|
||||
taskContentDigest: DEFINITION.contentDigest,
|
||||
taskCreatedAtMs: DEFINITION.createdAtMs,
|
||||
taskUpdatedAtMs: DEFINITION.updatedAtMs,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function executionRow() {
|
||||
return {
|
||||
projectId: EXECUTION.projectId,
|
||||
taskId: EXECUTION.taskId,
|
||||
sourceRevision: EXECUTION.sourceRevision,
|
||||
taskRevision: EXECUTION.taskRevision,
|
||||
sourceContentDigest: EXECUTION.sourceContentDigest,
|
||||
executorType: EXECUTION.executorType,
|
||||
planSchema: EXECUTION.planSchema,
|
||||
planJson: {
|
||||
command: EXECUTION.command,
|
||||
environment: EXECUTION.environment,
|
||||
placement: EXECUTION.placement,
|
||||
},
|
||||
contentDigest: EXECUTION.contentDigest,
|
||||
createdAtMs: EXECUTION.createdAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(options = {}) {
|
||||
const calls = [];
|
||||
const client = {
|
||||
async query(sql, params = []) {
|
||||
const normalized = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ sql: normalized, params });
|
||||
if (
|
||||
normalized.startsWith('BEGIN') ||
|
||||
normalized === 'COMMIT' ||
|
||||
normalized === 'ROLLBACK' ||
|
||||
normalized.startsWith('SELECT set_config') ||
|
||||
normalized.startsWith('INSERT INTO')
|
||||
) return { rows: [], rowCount: normalized.startsWith('INSERT') ? 1 : 0 };
|
||||
if (normalized.includes('FROM "ql3"."projects"')) {
|
||||
const rows = options.projectRows ?? [{
|
||||
projectStatus: 'active',
|
||||
projectVersion: 2,
|
||||
}];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."project_role_bindings"')) {
|
||||
const rows = options.bindingRows ?? [{
|
||||
bindingVersion: 3,
|
||||
bindingState: 'active',
|
||||
bindingRole: 'operator',
|
||||
}];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."runs"')) {
|
||||
const rows = options.runRows ?? [];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."task_definitions"')) {
|
||||
const rows = options.taskRows ?? [taskRow()];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."task_execution_revisions"')) {
|
||||
const rows = options.executionRows ?? [executionRow()];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: 1_000 }], rowCount: 1 };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."run_attempts"')) {
|
||||
const rows = options.attemptRows ?? [];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."run_events"')) {
|
||||
const rows = options.eventRows ?? [];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
throw new Error(`Unexpected SQL: ${normalized}`);
|
||||
},
|
||||
release() { calls.push({ sql: 'RELEASE', params: [] }); },
|
||||
};
|
||||
return {
|
||||
repository: new PostgresTaskStartRepository({
|
||||
async connect() { return client; },
|
||||
}),
|
||||
calls,
|
||||
};
|
||||
}
|
||||
|
||||
test('revalidates Policy and Task/execution digests before one atomic Run aggregate', async () => {
|
||||
const { repository, calls } = fixture();
|
||||
assert.deepEqual(await repository.startTask(command()), {
|
||||
status: 'accepted',
|
||||
projectId: 'project-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: 1,
|
||||
taskContentDigest: DEFINITION.contentDigest,
|
||||
runId: IDS[0],
|
||||
attemptId: IDS[1],
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'remote_worker',
|
||||
executionRevisionDigest: EXECUTION.contentDigest,
|
||||
createdAtMs: 1_000,
|
||||
});
|
||||
const project = calls.findIndex(({ sql }) => sql.includes('FROM "ql3"."projects"'));
|
||||
const binding = calls.findIndex(({ sql }) => sql.includes('project_role_bindings'));
|
||||
const task = calls.findIndex(({ sql }) => sql.includes('task_definitions'));
|
||||
const execution = calls.findIndex(({ sql }) => sql.includes('task_execution_revisions'));
|
||||
assert.ok(project < binding && binding < task && task < execution);
|
||||
assert.equal(calls.filter(({ sql }) => sql.startsWith('INSERT INTO')).length, 4);
|
||||
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), true);
|
||||
});
|
||||
|
||||
test('returns the original durable identities for an exact replay', async () => {
|
||||
const createdPayload = {
|
||||
status: 'created',
|
||||
version: 1,
|
||||
execution_owner: 'runtime',
|
||||
executor_type: 'remote_worker',
|
||||
execution_revision_digest: EXECUTION.contentDigest,
|
||||
task_revision: 1,
|
||||
task_content_digest: DEFINITION.contentDigest,
|
||||
mutation_id: MUTATION_ID,
|
||||
policy_fence: { project_version: 2, binding_version: 3 },
|
||||
};
|
||||
const { repository, calls } = fixture({
|
||||
runRows: [{
|
||||
runId: IDS[0],
|
||||
projectId: 'project-1',
|
||||
taskId: 'task-1',
|
||||
taskRevisionRef: EXECUTION.taskRevision,
|
||||
triggerType: 'task_start',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
triggeredBy: 'user-1',
|
||||
requestId: MUTATION_ID,
|
||||
priority: 0,
|
||||
createdAtMs: 1_000,
|
||||
}],
|
||||
attemptRows: [{ attemptId: IDS[1], executorType: 'remote_worker' }],
|
||||
eventRows: [
|
||||
{
|
||||
sequence: 1,
|
||||
type: 'run.created',
|
||||
actorType: 'user',
|
||||
actorId: 'user-1',
|
||||
payload: createdPayload,
|
||||
createdAtMs: 1_000,
|
||||
},
|
||||
{
|
||||
sequence: 2,
|
||||
type: 'run.queued',
|
||||
actorType: 'user',
|
||||
actorId: 'user-1',
|
||||
payload: { from_status: 'created', to_status: 'queued', version: 2 },
|
||||
createdAtMs: 1_000,
|
||||
},
|
||||
],
|
||||
});
|
||||
const replay = await repository.startTask(command({
|
||||
runId: '019f7300-0000-7000-8000-000000000901',
|
||||
attemptId: '019f7300-0000-7000-8000-000000000902',
|
||||
createdEventId: '019f7300-0000-7000-8000-000000000903',
|
||||
queuedEventId: '019f7300-0000-7000-8000-000000000904',
|
||||
}));
|
||||
assert.equal(replay.status, 'existing');
|
||||
assert.equal(replay.runId, IDS[0]);
|
||||
assert.equal(replay.attemptId, IDS[1]);
|
||||
assert.equal(calls.some(({ sql }) => sql.includes('task_definitions')), false);
|
||||
assert.equal(calls.some(({ sql }) => sql.startsWith('INSERT INTO')), false);
|
||||
});
|
||||
|
||||
test('rejects missing, authorization, definition and disabled fences', async () => {
|
||||
await assert.rejects(
|
||||
fixture({ projectRows: [] }).repository.startTask(command()),
|
||||
TaskStartNotFoundError,
|
||||
);
|
||||
await assert.rejects(
|
||||
fixture({ bindingRows: [{
|
||||
bindingVersion: 4,
|
||||
bindingState: 'revoked',
|
||||
bindingRole: null,
|
||||
}] }).repository.startTask(command()),
|
||||
(error) =>
|
||||
error instanceof TaskStartFenceRejectedError &&
|
||||
error.reason === 'authorization_changed',
|
||||
);
|
||||
await assert.rejects(
|
||||
fixture().repository.startTask(command({ expectedRevision: 2 })),
|
||||
(error) =>
|
||||
error instanceof TaskStartFenceRejectedError &&
|
||||
error.reason === 'definition_changed',
|
||||
);
|
||||
await assert.rejects(
|
||||
fixture({
|
||||
taskRows: [taskRow({
|
||||
enabled: false,
|
||||
taskContentDigest: DISABLED_DEFINITION.contentDigest,
|
||||
})],
|
||||
}).repository.startTask(command({
|
||||
expectedContentDigest: DISABLED_DEFINITION.contentDigest,
|
||||
})),
|
||||
(error) =>
|
||||
error instanceof TaskStartFenceRejectedError &&
|
||||
error.reason === 'task_disabled',
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,311 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
createToolExecutionEvidenceBundle,
|
||||
ToolExecutionEvidenceConflictError,
|
||||
ToolExecutionEvidenceUnavailableError,
|
||||
} = require('@qinglong/runtime-core/tool-execution-evidence');
|
||||
const {
|
||||
PostgresToolExecutionEvidenceRepository,
|
||||
} = require('@qinglong/cluster-postgres/tool-execution-evidence');
|
||||
|
||||
function evidence(overrides = {}) {
|
||||
const createdAtMs = overrides.createdAtMs ?? 1700000000000;
|
||||
return createToolExecutionEvidenceBundle({
|
||||
traceId: overrides.traceId ?? '1'.repeat(32),
|
||||
spanId: overrides.spanId ?? '2'.repeat(16),
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
stepRunId: 'step-1',
|
||||
invocationPlanDigest: '3'.repeat(64),
|
||||
bindingDigest: '4'.repeat(64),
|
||||
adapterDigest: '5'.repeat(64),
|
||||
redactionContractDigest: '6'.repeat(64),
|
||||
auditContractDigest: '7'.repeat(64),
|
||||
audit: {
|
||||
eventId:
|
||||
overrides.eventId ?? '12345678-1234-4234-9234-123456789abc',
|
||||
requestId: 'request-1',
|
||||
operationId: 'tool.invoke.start',
|
||||
projectId: 'project-1',
|
||||
subject: { type: 'user', id: 'user-1' },
|
||||
authenticationId: 'authentication-1',
|
||||
outcome: 'allowed',
|
||||
reasons: ['policy_allowed'],
|
||||
fence: { projectVersion: 4, bindingVersion: 7 },
|
||||
occurredAtMs: createdAtMs,
|
||||
},
|
||||
createdAtMs,
|
||||
});
|
||||
}
|
||||
|
||||
function row(bundle, overrides = {}) {
|
||||
return {
|
||||
traceJson: bundle.trace,
|
||||
auditJson: bundle.audit,
|
||||
receiptJson: bundle.receipt,
|
||||
storedTraceId: bundle.trace.traceId,
|
||||
storedSpanId: bundle.trace.spanId,
|
||||
storedParentSpanId: bundle.trace.parentSpanId,
|
||||
traceProjectId: bundle.trace.projectId,
|
||||
traceRunId: bundle.trace.runId,
|
||||
traceStepRunId: bundle.trace.stepRunId,
|
||||
traceInvocationPlanDigest: bundle.trace.invocationPlanDigest,
|
||||
traceBindingDigest: bundle.trace.bindingDigest,
|
||||
storedAdapterDigest: bundle.trace.adapterDigest,
|
||||
storedRedactionContractDigest: bundle.trace.redactionContractDigest,
|
||||
storedAuditContractDigest: bundle.trace.auditContractDigest,
|
||||
traceCreatedAtMs: String(bundle.trace.createdAtMs),
|
||||
storedTraceDigest: bundle.trace.traceDigest,
|
||||
storedEventId: bundle.receipt.eventId,
|
||||
receiptProjectId: bundle.receipt.projectId,
|
||||
receiptRunId: bundle.receipt.runId,
|
||||
receiptStepRunId: bundle.receipt.stepRunId,
|
||||
receiptTraceId: bundle.receipt.traceId,
|
||||
receiptSpanId: bundle.receipt.spanId,
|
||||
receiptTraceDigest: bundle.receipt.traceDigest,
|
||||
receiptInvocationPlanDigest: bundle.receipt.invocationPlanDigest,
|
||||
receiptBindingDigest: bundle.receipt.bindingDigest,
|
||||
storedAuditRecordDigest: bundle.receipt.auditRecordDigest,
|
||||
receiptCreatedAtMs: String(bundle.receipt.createdAtMs),
|
||||
storedReceiptDigest: bundle.receipt.receiptDigest,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function clientWith(handler) {
|
||||
const calls = [];
|
||||
let released = false;
|
||||
return {
|
||||
calls,
|
||||
get released() {
|
||||
return released;
|
||||
},
|
||||
async query(text, values = []) {
|
||||
calls.push({ text, values });
|
||||
if (
|
||||
text.startsWith('BEGIN') ||
|
||||
text.includes("set_config('") ||
|
||||
text === 'COMMIT' ||
|
||||
text === 'ROLLBACK'
|
||||
) {
|
||||
return { rows: [] };
|
||||
}
|
||||
return handler(text, values, calls);
|
||||
},
|
||||
release() {
|
||||
released = true;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('prepares audit, trace and receipt in one serializable transaction', async () => {
|
||||
const bundle = evidence();
|
||||
const client = clientWith(async (sql) => {
|
||||
if (sql.includes('WHERE receipt.event_id')) return { rows: [] };
|
||||
if (sql.includes('FROM "ql3"."step_runs" AS step')) {
|
||||
return {
|
||||
rows: [{ kind: 'tool', status: 'ready', projectId: 'project-1' }],
|
||||
};
|
||||
}
|
||||
if (sql.startsWith('INSERT INTO')) return { rows: [], rowCount: 1 };
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
});
|
||||
const pool = {
|
||||
async query() {
|
||||
throw new Error('pool query is not used by prepare');
|
||||
},
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
};
|
||||
|
||||
const repository = new PostgresToolExecutionEvidenceRepository(pool);
|
||||
assert.deepEqual(await repository.prepare(bundle), {
|
||||
status: 'created',
|
||||
bundle,
|
||||
});
|
||||
assert.equal(client.released, true);
|
||||
assert.deepEqual(
|
||||
client.calls
|
||||
.map(({ text }) => text)
|
||||
.filter((sql) => sql.startsWith('INSERT INTO'))
|
||||
.map((sql) => sql.match(/"ql3"\."([^"]+)"/)[1]),
|
||||
[
|
||||
'security_audit_events',
|
||||
'tool_execution_trace_anchors',
|
||||
'tool_execution_audit_receipts',
|
||||
],
|
||||
);
|
||||
assert.equal(
|
||||
client.calls.some(({ text }) => text === 'COMMIT'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
client.calls.some(({ text }) => text === 'ROLLBACK'),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('replays exact evidence and rejects identity reuse', async () => {
|
||||
const bundle = evidence();
|
||||
const stored = row(bundle);
|
||||
const exactClient = clientWith(async (sql) => {
|
||||
if (sql.includes('WHERE receipt.event_id')) return { rows: [stored] };
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
});
|
||||
const exact = new PostgresToolExecutionEvidenceRepository({
|
||||
async query() {
|
||||
throw new Error('unused');
|
||||
},
|
||||
async connect() {
|
||||
return exactClient;
|
||||
},
|
||||
});
|
||||
assert.deepEqual(await exact.prepare(bundle), {
|
||||
status: 'existing',
|
||||
bundle,
|
||||
});
|
||||
|
||||
const changed = evidence({
|
||||
createdAtMs: bundle.trace.createdAtMs + 1,
|
||||
traceId: bundle.trace.traceId,
|
||||
spanId: bundle.trace.spanId,
|
||||
eventId: bundle.audit.eventId,
|
||||
});
|
||||
const conflictClient = clientWith(async (sql) => {
|
||||
if (sql.includes('WHERE receipt.event_id')) return { rows: [stored] };
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
});
|
||||
const conflict = new PostgresToolExecutionEvidenceRepository({
|
||||
async query() {
|
||||
throw new Error('unused');
|
||||
},
|
||||
async connect() {
|
||||
return conflictClient;
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
conflict.prepare(changed),
|
||||
ToolExecutionEvidenceConflictError,
|
||||
);
|
||||
assert.equal(
|
||||
conflictClient.calls.some(({ text }) => text === 'ROLLBACK'),
|
||||
true,
|
||||
);
|
||||
assert.equal(conflictClient.released, true);
|
||||
});
|
||||
|
||||
test('reads bounded evidence without SELECT authority on the audit table', async () => {
|
||||
const first = evidence();
|
||||
const second = evidence({
|
||||
createdAtMs: first.trace.createdAtMs + 1,
|
||||
traceId: '8'.repeat(32),
|
||||
spanId: '9'.repeat(16),
|
||||
eventId: '87654321-4321-4321-8321-cba987654321',
|
||||
});
|
||||
const calls = [];
|
||||
const repository = new PostgresToolExecutionEvidenceRepository({
|
||||
async query(text, values) {
|
||||
calls.push({ text, values });
|
||||
if (text.includes('WHERE trace.trace_id')) {
|
||||
return { rows: [row(first)] };
|
||||
}
|
||||
if (text.includes('WHERE receipt.event_id')) {
|
||||
return { rows: [row(first)] };
|
||||
}
|
||||
if (text.includes('WHERE trace.run_id')) {
|
||||
return { rows: [row(first), row(second)] };
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${text}`);
|
||||
},
|
||||
async connect() {
|
||||
throw new Error('unused');
|
||||
},
|
||||
});
|
||||
|
||||
assert.deepEqual(
|
||||
await repository.findByTrace(first.trace.traceId, first.trace.spanId),
|
||||
first,
|
||||
);
|
||||
assert.deepEqual(
|
||||
await repository.findByAuditEventId(first.audit.eventId),
|
||||
first,
|
||||
);
|
||||
assert.deepEqual(
|
||||
await repository.listByRun({ runId: 'run-1', limit: 1 }),
|
||||
{
|
||||
bundles: [first],
|
||||
truncated: true,
|
||||
next: {
|
||||
createdAtMs: first.trace.createdAtMs,
|
||||
traceId: first.trace.traceId,
|
||||
spanId: first.trace.spanId,
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(
|
||||
calls.some(({ text }) =>
|
||||
text.includes('JOIN "ql3"."security_audit_events"'),
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.deepEqual(calls.at(-1).values, ['run-1', null, 0, '', 2]);
|
||||
});
|
||||
|
||||
test('rejects invalid lookups before SQL and fails closed on corrupt rows', async () => {
|
||||
let queries = 0;
|
||||
const repository = new PostgresToolExecutionEvidenceRepository({
|
||||
async query() {
|
||||
queries += 1;
|
||||
return {
|
||||
rows: [row(evidence(), { storedTraceDigest: '0'.repeat(64) })],
|
||||
};
|
||||
},
|
||||
async connect() {
|
||||
throw new Error('unused');
|
||||
},
|
||||
});
|
||||
await assert.rejects(repository.findByTrace('bad', 'also-bad'), {
|
||||
code: 'TOOL_EXECUTION_EVIDENCE_INVALID',
|
||||
});
|
||||
assert.equal(queries, 0);
|
||||
await assert.rejects(
|
||||
repository.findByTrace('1'.repeat(32), '2'.repeat(16)),
|
||||
ToolExecutionEvidenceUnavailableError,
|
||||
);
|
||||
});
|
||||
|
||||
test('rolls back when the StepRun is not an admitted same-Project Tool step', async () => {
|
||||
const bundle = evidence();
|
||||
const client = clientWith(async (sql) => {
|
||||
if (sql.includes('WHERE receipt.event_id')) return { rows: [] };
|
||||
if (sql.includes('FROM "ql3"."step_runs" AS step')) {
|
||||
return {
|
||||
rows: [{ kind: 'task', status: 'ready', projectId: 'project-1' }],
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
});
|
||||
const repository = new PostgresToolExecutionEvidenceRepository({
|
||||
async query() {
|
||||
throw new Error('unused');
|
||||
},
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
repository.prepare(bundle),
|
||||
ToolExecutionEvidenceConflictError,
|
||||
);
|
||||
assert.equal(
|
||||
client.calls.some(({ text }) => text.startsWith('INSERT INTO')),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
client.calls.some(({ text }) => text === 'ROLLBACK'),
|
||||
true,
|
||||
);
|
||||
assert.equal(client.released, true);
|
||||
});
|
||||
@@ -0,0 +1,844 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
createPluginPackageResourceGenerationFromReferences,
|
||||
} = require('@qinglong/runtime-core/plugin-package-resource-generation');
|
||||
const {
|
||||
createProjectToolDefinitionSnapshot,
|
||||
projectToolDefinitionRegistry,
|
||||
} = require('@qinglong/runtime-core/project-tool-definition-snapshot');
|
||||
const {
|
||||
createStepRunMutation,
|
||||
transitionStepRunMutation,
|
||||
} = require('@qinglong/runtime-core/step-run');
|
||||
const {
|
||||
ToolExecutionStartBarrierConflictError,
|
||||
ToolExecutionStartBarrierUnavailableError,
|
||||
createToolExecutionStartCommand,
|
||||
toolExecutionStartBarrierRecord,
|
||||
} = require('@qinglong/runtime-core/tool-execution-start-barrier');
|
||||
const {
|
||||
TOOL_EXECUTION_START_AUDIT_OPERATION,
|
||||
createToolExecutionEvidenceBundle,
|
||||
toolExecutionAdmissionEvidence,
|
||||
} = require('@qinglong/runtime-core/tool-execution-evidence');
|
||||
const {
|
||||
TrustedToolHandlerBindingRegistry,
|
||||
admitTrustedToolExecution,
|
||||
createTrustedToolHandlerBinding,
|
||||
createTrustedToolInvocationPlan,
|
||||
trustedToolContractIdentityDigest,
|
||||
} = require('@qinglong/runtime-core/trusted-tool-invocation');
|
||||
const {
|
||||
prepareToolInvocation,
|
||||
} = require('@qinglong/runtime-core/tool-registry');
|
||||
const {
|
||||
createToolExecutionCompletionCommand,
|
||||
createToolExecutionResultArtifact,
|
||||
} = require('@qinglong/runtime-core/tool-execution-completion');
|
||||
const {
|
||||
TOOL_EXECUTION_FAILURE_FACTS,
|
||||
ToolExecutionFailureCompletionConflictError,
|
||||
createToolExecutionFailureCompletionCommand,
|
||||
createToolExecutionFailureResult,
|
||||
} = require('@qinglong/runtime-core/tool-execution-failure-completion');
|
||||
const {
|
||||
TRUSTED_TOOL_EXECUTION_RESULT_SCHEMA,
|
||||
} = require('@qinglong/runtime-core/trusted-tool-execution');
|
||||
const {
|
||||
createToolResultKeyCatalogBootstrapCommand,
|
||||
normalizeToolResultKeyCatalogRecord,
|
||||
requireActiveToolResultKey,
|
||||
toolResultKeyCatalogFence,
|
||||
toolResultKeyMaterialProof,
|
||||
} = require('@qinglong/runtime-core/tool-result-key-catalog');
|
||||
const {
|
||||
PostgresToolExecutionStartBarrierRepository,
|
||||
} = require('@qinglong/cluster-postgres/tool-execution-start-barrier');
|
||||
const {
|
||||
PostgresToolExecutionCompletionRepository,
|
||||
} = require('@qinglong/cluster-postgres/tool-execution-completion');
|
||||
const {
|
||||
PostgresToolExecutionFailureCompletionRepository,
|
||||
} = require('@qinglong/cluster-postgres/tool-execution-failure-completion');
|
||||
|
||||
const DIGEST_A = 'a'.repeat(64);
|
||||
const DIGEST_B = 'b'.repeat(64);
|
||||
const DIGEST_C = 'c'.repeat(64);
|
||||
const REQUESTER = Object.freeze({ type: 'user', id: 'usr-tool-owner' });
|
||||
const FENCE = Object.freeze({ projectVersion: 3, bindingVersion: 7 });
|
||||
const NOW_MS = 1_400;
|
||||
const OUTPUT_DIGEST_DOMAIN = Buffer.from(
|
||||
'qinglong/trusted-tool-execution-output-digest@v1\0',
|
||||
'utf8',
|
||||
);
|
||||
const RESULT_DIGEST_DOMAIN = Buffer.from(
|
||||
'qinglong/trusted-tool-execution-result-digest@v1\0',
|
||||
'utf8',
|
||||
);
|
||||
|
||||
function resultKeyCatalog() {
|
||||
const command = createToolResultKeyCatalogBootstrapCommand({
|
||||
keyId: 'tool-result-key-test',
|
||||
materialProof: toolResultKeyMaterialProof(
|
||||
'tool-result-key-test',
|
||||
Buffer.alloc(32, 5),
|
||||
),
|
||||
mutationId: 'tool-result-key-bootstrap-test',
|
||||
});
|
||||
return normalizeToolResultKeyCatalogRecord({
|
||||
...command.next,
|
||||
committedAtMs: 1_200,
|
||||
});
|
||||
}
|
||||
|
||||
function hash(domain, value) {
|
||||
return createHash('sha256')
|
||||
.update(domain)
|
||||
.update(JSON.stringify(value))
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
function authorizer() {
|
||||
return {
|
||||
async authorize() {
|
||||
return { effect: 'allow', reasons: ['role_grant'], fence: FENCE };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function principal() {
|
||||
return {
|
||||
subject: REQUESTER,
|
||||
authenticationId: 'auth-tool-1',
|
||||
authenticatedAtMs: 800,
|
||||
expiresAtMs: 10_000,
|
||||
assurance: 'local_console',
|
||||
};
|
||||
}
|
||||
|
||||
function projectToolSnapshot() {
|
||||
const generation = createPluginPackageResourceGenerationFromReferences({
|
||||
installationId: 'install-demo',
|
||||
projectId: 'project-001',
|
||||
packageName: 'demo',
|
||||
lockDigest: DIGEST_A,
|
||||
generation: 1,
|
||||
previousActiveLockDigest: null,
|
||||
contentDigest: DIGEST_B,
|
||||
resources: [],
|
||||
});
|
||||
return createProjectToolDefinitionSnapshot({
|
||||
projectId: 'project-001',
|
||||
contributions: [
|
||||
{
|
||||
generation,
|
||||
revisionDigest: DIGEST_C,
|
||||
definitions: [
|
||||
{
|
||||
name: 'demo.compare',
|
||||
version: '1.0.0',
|
||||
description: 'Compare one bounded Run projection',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
runId: { type: 'string', minLength: 1, maxLength: 64 },
|
||||
},
|
||||
required: ['runId'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
outputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
summary: { type: 'string', maxLength: 1024 },
|
||||
},
|
||||
required: ['summary'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
effect: 'read',
|
||||
risk: 'low',
|
||||
requiredPermissions: ['run.read'],
|
||||
timeoutSeconds: 30,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
async function command(overrides = {}) {
|
||||
const snapshot = projectToolSnapshot();
|
||||
const binding = createTrustedToolHandlerBinding(snapshot, {
|
||||
tool: { name: 'demo.compare', version: '1.0.0' },
|
||||
adapter: { id: 'builtin.demo-compare', version: '1.0.0' },
|
||||
executionClass: 'builtin_in_process',
|
||||
profiles: ['edge', 'standalone'],
|
||||
authorities: ['database.read'],
|
||||
timeoutSeconds: 20,
|
||||
redactionContract: {
|
||||
id: 'redaction.demo-compare',
|
||||
version: '1.0.0',
|
||||
},
|
||||
auditContract: { id: 'audit.tool-call', version: '1.0.0' },
|
||||
});
|
||||
const bindings = new TrustedToolHandlerBindingRegistry(snapshot, [binding]);
|
||||
const invocation = await prepareToolInvocation(
|
||||
projectToolDefinitionRegistry(snapshot),
|
||||
{
|
||||
projectId: 'project-001',
|
||||
principal: principal(),
|
||||
nowMs: 900,
|
||||
tool: { name: 'demo.compare', version: '1.0.0' },
|
||||
input: { runId: 'run-001' },
|
||||
},
|
||||
authorizer(),
|
||||
);
|
||||
const plan = (
|
||||
await createTrustedToolInvocationPlan(bindings, invocation, {
|
||||
actionRef: 'tool-plan:run-001',
|
||||
inputArtifactId: 'artifact-input-001',
|
||||
previewArtifactId: 'artifact-preview-001',
|
||||
artifactKeyId: 'tool-key-test',
|
||||
artifactKey: Buffer.alloc(32, 7),
|
||||
artifactNonce: Buffer.alloc(12, 9),
|
||||
profile: 'edge',
|
||||
preview: {
|
||||
title: 'Compare Run',
|
||||
summary: 'Reads one Run projection',
|
||||
fields: [{ kind: 'identifier', label: 'Run', value: 'run-001' }],
|
||||
warnings: [],
|
||||
},
|
||||
sealedAtMs: 1_000,
|
||||
})
|
||||
).plan;
|
||||
const creation = createStepRunMutation(
|
||||
{
|
||||
id: 'step-run-001',
|
||||
runId: 'run-001',
|
||||
stepKey: 'workflow.compare',
|
||||
kind: 'tool',
|
||||
definitionRef: 'tool:demo.compare@1.0.0',
|
||||
definitionDigest: snapshot.definitions[0].definitionDigest,
|
||||
required: true,
|
||||
initialStatus: 'ready',
|
||||
inputRef: 'artifact:step-input-001',
|
||||
mutationId: 'step-create-001',
|
||||
createdAtMs: 1_000,
|
||||
},
|
||||
{
|
||||
expectedRunVersion: 0,
|
||||
expectedRunEventSequence: 0,
|
||||
eventId: '50000000-0000-4000-8000-000000000001',
|
||||
dedupeKey: 'step-create:step-run-001',
|
||||
actor: REQUESTER,
|
||||
},
|
||||
);
|
||||
const evidence = createToolExecutionEvidenceBundle({
|
||||
traceId: '1'.repeat(32),
|
||||
spanId: '2'.repeat(16),
|
||||
projectId: 'project-001',
|
||||
runId: 'run-001',
|
||||
stepRunId: creation.stepRun.id,
|
||||
invocationPlanDigest: plan.planDigest,
|
||||
bindingDigest: binding.bindingDigest,
|
||||
adapterDigest: trustedToolContractIdentityDigest(binding.adapter),
|
||||
redactionContractDigest: trustedToolContractIdentityDigest(
|
||||
binding.redactionContract,
|
||||
),
|
||||
auditContractDigest: trustedToolContractIdentityDigest(
|
||||
binding.auditContract,
|
||||
),
|
||||
audit: {
|
||||
eventId: '40000000-0000-4000-8000-000000000001',
|
||||
requestId: 'tool-request-001',
|
||||
operationId: TOOL_EXECUTION_START_AUDIT_OPERATION,
|
||||
projectId: 'project-001',
|
||||
subject: REQUESTER,
|
||||
authenticationId: 'auth-tool-1',
|
||||
outcome: 'allowed',
|
||||
reasons: ['tool_execution_start'],
|
||||
fence: FENCE,
|
||||
occurredAtMs: NOW_MS,
|
||||
},
|
||||
createdAtMs: NOW_MS,
|
||||
});
|
||||
const admission = await admitTrustedToolExecution(bindings, plan, {
|
||||
principal: principal(),
|
||||
profile: 'edge',
|
||||
nowMs: NOW_MS,
|
||||
authorizer: authorizer(),
|
||||
evidence: {
|
||||
stepRun: {
|
||||
id: creation.stepRun.id,
|
||||
version: creation.stepRun.version,
|
||||
digest: creation.stepRun.stepRunDigest,
|
||||
},
|
||||
...toolExecutionAdmissionEvidence(evidence),
|
||||
},
|
||||
});
|
||||
const mutation = transitionStepRunMutation(
|
||||
creation.stepRun,
|
||||
{
|
||||
expectedVersion: creation.stepRun.version,
|
||||
expectedDigest: creation.stepRun.stepRunDigest,
|
||||
mutationId: 'step-running-002',
|
||||
to: 'running',
|
||||
atMs: NOW_MS,
|
||||
},
|
||||
{
|
||||
expectedRunVersion: 1,
|
||||
expectedRunEventSequence: 1,
|
||||
eventId: '50000000-0000-4000-8000-000000000002',
|
||||
dedupeKey: 'step-running:step-run-001',
|
||||
actor: REQUESTER,
|
||||
},
|
||||
);
|
||||
return createToolExecutionStartCommand({
|
||||
startId: overrides.startId ?? 'tool-start-001',
|
||||
admission,
|
||||
evidence,
|
||||
stepRunMutation: mutation,
|
||||
});
|
||||
}
|
||||
|
||||
function barrierRow(barrier, overrides = {}) {
|
||||
return {
|
||||
barrierJson: barrier,
|
||||
storedStartId: barrier.startId,
|
||||
storedProjectId: barrier.projectId,
|
||||
storedRunId: barrier.runId,
|
||||
storedStepRunId: barrier.stepRunId,
|
||||
storedStepRunVersion: String(barrier.startedStepRunVersion),
|
||||
storedMutationId: barrier.stepRunMutationId,
|
||||
storedRunEventId: barrier.runEventId,
|
||||
storedTraceId: barrier.traceId,
|
||||
storedSpanId: barrier.spanId,
|
||||
storedAuditEventId: barrier.auditEventId,
|
||||
storedCommandDigest: barrier.commandDigest,
|
||||
storedBarrierDigest: barrier.barrierDigest,
|
||||
storedStartedAtMs: String(barrier.startedAtMs),
|
||||
storedMutationDigest: barrier.stepRunMutationDigest,
|
||||
storedStartedStepRunDigest: barrier.startedStepRunDigest,
|
||||
storedTraceDigest: barrier.traceDigest,
|
||||
storedAuditReceiptDigest: barrier.auditReceiptDigest,
|
||||
storedArtifactProjectId: barrier.projectId,
|
||||
storedArtifactActionRef: barrier.actionRef,
|
||||
storedInputArtifactId: barrier.invocationArtifact.artifactId,
|
||||
storedInputArtifactDigest: barrier.invocationArtifact.artifactDigest,
|
||||
storedInputDigest: barrier.invocationArtifact.inputDigest,
|
||||
storedPreviewArtifactId: barrier.previewArtifact.artifactId,
|
||||
storedPreviewArtifactDigest: barrier.previewArtifact.artifactDigest,
|
||||
storedArtifactActionDigest: barrier.previewArtifact.actionDigest,
|
||||
storedPreviewDigest: barrier.previewArtifact.previewDigest,
|
||||
storedArtifactRedactionContractDigest:
|
||||
barrier.previewArtifact.redactionContractDigest,
|
||||
storedArtifactBoundAtMs: String(barrier.startedAtMs),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function clientWith(handler) {
|
||||
const calls = [];
|
||||
let released = false;
|
||||
return {
|
||||
calls,
|
||||
get released() {
|
||||
return released;
|
||||
},
|
||||
async query(text, values = []) {
|
||||
calls.push({ text, values });
|
||||
if (
|
||||
text.startsWith('BEGIN') ||
|
||||
text.includes("set_config('") ||
|
||||
text.includes('pg_advisory_xact_lock') ||
|
||||
text === 'COMMIT' ||
|
||||
text === 'ROLLBACK'
|
||||
) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text.includes('plugin_package_tool_start_allowed')) {
|
||||
return { rows: [{ allowed: true }] };
|
||||
}
|
||||
return handler(text, values, calls);
|
||||
},
|
||||
release() {
|
||||
released = true;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function repositoryFor(client) {
|
||||
return new PostgresToolExecutionStartBarrierRepository({
|
||||
async query() {
|
||||
throw new Error('pool query is not used by prepare');
|
||||
},
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function completionRepositoryFor(client) {
|
||||
return new PostgresToolExecutionCompletionRepository({
|
||||
async query() {
|
||||
throw new Error('pool query is not used by commit');
|
||||
},
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function failureCompletionRepositoryFor(client) {
|
||||
return new PostgresToolExecutionFailureCompletionRepository({
|
||||
async query() {
|
||||
throw new Error('pool query is not used by commit');
|
||||
},
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function executionResult(barrier, output, completedAtMs) {
|
||||
const outputDigest = hash(OUTPUT_DIGEST_DOMAIN, output);
|
||||
const unsigned = Object.freeze({
|
||||
schema: TRUSTED_TOOL_EXECUTION_RESULT_SCHEMA,
|
||||
startId: barrier.startId,
|
||||
barrierDigest: barrier.barrierDigest,
|
||||
adapterDigest: barrier.adapterDigest,
|
||||
output,
|
||||
outputDigest,
|
||||
completedAtMs,
|
||||
});
|
||||
return Object.freeze({
|
||||
...unsigned,
|
||||
resultDigest: hash(RESULT_DIGEST_DOMAIN, unsigned),
|
||||
});
|
||||
}
|
||||
|
||||
test('atomically commits evidence, StepRun start and the barrier', async () => {
|
||||
const start = await command();
|
||||
const mutation = start.stepRunMutation;
|
||||
const client = clientWith(async (sql) => {
|
||||
if (sql.includes('FROM "ql3"."tool_execution_start_barriers"')) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (sql.includes('FROM "ql3"."step_runs" AS step')) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
stepKind: 'tool',
|
||||
stepStatus: mutation.previousStatus,
|
||||
stepVersion: String(mutation.expectedStepRunVersion),
|
||||
stepDigest: mutation.expectedStepRunDigest,
|
||||
definitionRef: mutation.stepRun.definitionRef,
|
||||
definitionDigest: mutation.stepRun.definitionDigest,
|
||||
projectId: 'project-001',
|
||||
runStatus: 'running',
|
||||
runVersion: String(mutation.expectedRunVersion),
|
||||
runEventSequence: String(mutation.expectedRunEventSequence),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (sql.startsWith('INSERT INTO') || sql.startsWith('UPDATE "ql3"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
});
|
||||
|
||||
const result = await repositoryFor(client).prepare(start);
|
||||
assert.deepEqual(result, {
|
||||
status: 'created',
|
||||
barrier: toolExecutionStartBarrierRecord(start),
|
||||
});
|
||||
assert.equal(client.released, true);
|
||||
assert.deepEqual(
|
||||
client.calls
|
||||
.map(({ text }) => text)
|
||||
.filter(
|
||||
(sql) =>
|
||||
sql.startsWith('INSERT INTO') || sql.startsWith('UPDATE "ql3"'),
|
||||
)
|
||||
.map((sql) => {
|
||||
const match = sql.match(/"ql3"\."([^"]+)"/);
|
||||
return `${sql.startsWith('UPDATE') ? 'update' : 'insert'}:${match[1]}`;
|
||||
}),
|
||||
[
|
||||
'insert:security_audit_events',
|
||||
'insert:tool_execution_trace_anchors',
|
||||
'insert:tool_execution_audit_receipts',
|
||||
'update:step_runs',
|
||||
'update:runs',
|
||||
'insert:run_events',
|
||||
'insert:step_run_mutations',
|
||||
'insert:tool_execution_start_barriers',
|
||||
'insert:tool_execution_start_artifact_bindings',
|
||||
],
|
||||
);
|
||||
const mutationInsert = client.calls.find(({ text }) =>
|
||||
text.includes('INSERT INTO "ql3"."step_run_mutations"'),
|
||||
);
|
||||
assert.match(mutationInsert.text, /transaction_timestamp\(\)/);
|
||||
assert.equal(mutationInsert.values.length, 9);
|
||||
assert.equal(
|
||||
client.calls.some(({ text }) => text === 'COMMIT'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
client.calls.some(({ text }) => text === 'ROLLBACK'),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('atomically persists encrypted result and the succeeded StepRun fence', async () => {
|
||||
const start = await command();
|
||||
const barrier = toolExecutionStartBarrierRecord(start);
|
||||
const result = executionResult(barrier, { summary: 'Run is healthy' }, 1_500);
|
||||
const registry = projectToolDefinitionRegistry(projectToolSnapshot());
|
||||
const artifact = createToolExecutionResultArtifact(
|
||||
{
|
||||
artifactId: 'artifact-result-001',
|
||||
projectId: barrier.projectId,
|
||||
runId: barrier.runId,
|
||||
stepRunId: barrier.stepRunId,
|
||||
tool: { name: 'demo.compare', version: '1.0.0' },
|
||||
executionResult: result,
|
||||
keyId: 'tool-result-key-test',
|
||||
key: Buffer.alloc(32, 5),
|
||||
},
|
||||
registry,
|
||||
() => Buffer.alloc(12, 4),
|
||||
);
|
||||
const running = start.stepRunMutation.stepRun;
|
||||
const mutation = transitionStepRunMutation(
|
||||
running,
|
||||
{
|
||||
expectedVersion: running.version,
|
||||
expectedDigest: running.stepRunDigest,
|
||||
mutationId: 'step-succeeded-003',
|
||||
to: 'succeeded',
|
||||
atMs: result.completedAtMs,
|
||||
outputRef: artifact.artifactId,
|
||||
},
|
||||
{
|
||||
expectedRunVersion: 2,
|
||||
expectedRunEventSequence: 2,
|
||||
eventId: '50000000-0000-4000-8000-000000000003',
|
||||
dedupeKey: 'step-succeeded:step-run-001',
|
||||
actor: REQUESTER,
|
||||
},
|
||||
);
|
||||
const completionCommand = createToolExecutionCompletionCommand({
|
||||
barrier,
|
||||
executionResult: result,
|
||||
resultArtifact: artifact,
|
||||
resultKeyCatalogFence: toolResultKeyCatalogFence(
|
||||
resultKeyCatalog(),
|
||||
requireActiveToolResultKey(resultKeyCatalog()),
|
||||
),
|
||||
stepRunMutation: mutation,
|
||||
});
|
||||
const client = clientWith(async (sql) => {
|
||||
if (sql.includes('FROM "ql3"."tool_execution_completions"')) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (sql.includes('FROM "ql3"."tool_execution_failure_completions"')) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (sql.includes('FROM "ql3"."tool_result_key_catalog_generations"')) {
|
||||
return { rows: [{ catalogJson: resultKeyCatalog() }] };
|
||||
}
|
||||
if (sql.includes('FROM "ql3"."tool_execution_start_barriers" AS barrier')) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
barrierJson: barrier,
|
||||
startedRunVersion: '2',
|
||||
startedEventSequence: '2',
|
||||
stepKind: 'tool',
|
||||
stepStatus: 'running',
|
||||
stepVersion: String(mutation.expectedStepRunVersion),
|
||||
stepDigest: mutation.expectedStepRunDigest,
|
||||
projectId: barrier.projectId,
|
||||
runStatus: 'running',
|
||||
runVersion: '2',
|
||||
runEventSequence: '2',
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (sql.startsWith('INSERT INTO') || sql.startsWith('UPDATE "ql3"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
});
|
||||
|
||||
const committed = await completionRepositoryFor(client).commit(
|
||||
completionCommand,
|
||||
);
|
||||
assert.equal(committed.status, 'created');
|
||||
assert.equal(committed.completion.startId, barrier.startId);
|
||||
assert.equal(client.released, true);
|
||||
assert.deepEqual(
|
||||
client.calls
|
||||
.map(({ text }) => text)
|
||||
.filter(
|
||||
(sql) =>
|
||||
sql.startsWith('INSERT INTO') || sql.startsWith('UPDATE "ql3"'),
|
||||
)
|
||||
.map((sql) => {
|
||||
const match = sql.match(/"ql3"\."([^"]+)"/);
|
||||
return `${sql.startsWith('UPDATE') ? 'update' : 'insert'}:${match[1]}`;
|
||||
}),
|
||||
[
|
||||
'update:step_runs',
|
||||
'update:runs',
|
||||
'insert:run_events',
|
||||
'insert:step_run_mutations',
|
||||
'insert:tool_execution_completions',
|
||||
'insert:tool_execution_result_key_bindings',
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('atomically persists fixed Tool failure facts and excludes success', async () => {
|
||||
const start = await command();
|
||||
const barrier = toolExecutionStartBarrierRecord(start);
|
||||
const failure = createToolExecutionFailureResult(barrier, 'timed_out', 1_500);
|
||||
const running = start.stepRunMutation.stepRun;
|
||||
const mutation = transitionStepRunMutation(
|
||||
running,
|
||||
{
|
||||
expectedVersion: running.version,
|
||||
expectedDigest: running.stepRunDigest,
|
||||
mutationId: 'step-timed-out-003',
|
||||
to: 'timed_out',
|
||||
atMs: failure.completedAtMs,
|
||||
...TOOL_EXECUTION_FAILURE_FACTS.timed_out,
|
||||
},
|
||||
{
|
||||
expectedRunVersion: 2,
|
||||
expectedRunEventSequence: 2,
|
||||
eventId: '50000000-0000-4000-8000-000000000003',
|
||||
dedupeKey: 'step-timed-out:step-run-001',
|
||||
actor: REQUESTER,
|
||||
},
|
||||
);
|
||||
const completionCommand = createToolExecutionFailureCompletionCommand({
|
||||
barrier,
|
||||
failure,
|
||||
stepRunMutation: mutation,
|
||||
});
|
||||
const client = clientWith(async (sql) => {
|
||||
if (sql.includes('FROM "ql3"."tool_execution_failure_completions"')) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (sql.includes('FROM "ql3"."tool_execution_completions"')) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (sql.includes('FROM "ql3"."tool_execution_start_barriers" AS barrier')) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
barrierJson: barrier,
|
||||
startedRunVersion: '2',
|
||||
startedEventSequence: '2',
|
||||
stepKind: 'tool',
|
||||
stepStatus: 'running',
|
||||
stepVersion: String(mutation.expectedStepRunVersion),
|
||||
stepDigest: mutation.expectedStepRunDigest,
|
||||
projectId: barrier.projectId,
|
||||
runStatus: 'running',
|
||||
runVersion: '2',
|
||||
runEventSequence: '2',
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (sql.startsWith('INSERT INTO') || sql.startsWith('UPDATE "ql3"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
});
|
||||
|
||||
const committed = await failureCompletionRepositoryFor(client).commit(
|
||||
completionCommand,
|
||||
);
|
||||
assert.equal(committed.status, 'created');
|
||||
assert.equal(committed.completion.outcome, 'timed_out');
|
||||
assert.equal(
|
||||
committed.completion.errorSummary,
|
||||
'Trusted Tool execution deadline exceeded',
|
||||
);
|
||||
assert.deepEqual(
|
||||
client.calls
|
||||
.map(({ text }) => text)
|
||||
.filter(
|
||||
(sql) =>
|
||||
sql.startsWith('INSERT INTO') || sql.startsWith('UPDATE "ql3"'),
|
||||
)
|
||||
.map((sql) => {
|
||||
const match = sql.match(/"ql3"\."([^"]+)"/);
|
||||
return `${sql.startsWith('UPDATE') ? 'update' : 'insert'}:${match[1]}`;
|
||||
}),
|
||||
[
|
||||
'update:step_runs',
|
||||
'update:runs',
|
||||
'insert:run_events',
|
||||
'insert:step_run_mutations',
|
||||
'insert:tool_execution_failure_completions',
|
||||
],
|
||||
);
|
||||
|
||||
const conflictClient = clientWith(async (sql) => {
|
||||
if (sql.includes('FROM "ql3"."tool_execution_failure_completions"')) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (sql.includes('FROM "ql3"."tool_execution_completions"')) {
|
||||
return { rows: [{ exists: 1 }] };
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
});
|
||||
await assert.rejects(
|
||||
failureCompletionRepositoryFor(conflictClient).commit(completionCommand),
|
||||
ToolExecutionFailureCompletionConflictError,
|
||||
);
|
||||
assert.equal(
|
||||
conflictClient.calls.some(({ text }) => text === 'ROLLBACK'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('exactly replays a committed start and rejects identity drift', async () => {
|
||||
const start = await command();
|
||||
const stored = toolExecutionStartBarrierRecord(start);
|
||||
const exactClient = clientWith(async (sql) => {
|
||||
if (sql.includes('FROM "ql3"."tool_execution_start_barriers"')) {
|
||||
return { rows: [barrierRow(stored)] };
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
});
|
||||
assert.deepEqual(await repositoryFor(exactClient).prepare(start), {
|
||||
status: 'existing',
|
||||
barrier: stored,
|
||||
});
|
||||
assert.equal(
|
||||
exactClient.calls.some(({ text }) => text.startsWith('INSERT INTO')),
|
||||
false,
|
||||
);
|
||||
|
||||
const drift = await command({ startId: 'tool-start-other' });
|
||||
const conflictClient = clientWith(async (sql) => {
|
||||
if (sql.includes('FROM "ql3"."tool_execution_start_barriers"')) {
|
||||
return { rows: [barrierRow(stored)] };
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
});
|
||||
await assert.rejects(
|
||||
repositoryFor(conflictClient).prepare(drift),
|
||||
ToolExecutionStartBarrierConflictError,
|
||||
);
|
||||
assert.equal(
|
||||
conflictClient.calls.some(({ text }) => text === 'ROLLBACK'),
|
||||
true,
|
||||
);
|
||||
assert.equal(conflictClient.released, true);
|
||||
});
|
||||
|
||||
test('rolls back before evidence writes when the durable fence changed', async () => {
|
||||
const start = await command();
|
||||
const mutation = start.stepRunMutation;
|
||||
const client = clientWith(async (sql) => {
|
||||
if (sql.includes('FROM "ql3"."tool_execution_start_barriers"')) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (sql.includes('FROM "ql3"."step_runs" AS step')) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
stepKind: 'tool',
|
||||
stepStatus: 'cancelled',
|
||||
stepVersion: String(mutation.expectedStepRunVersion),
|
||||
stepDigest: mutation.expectedStepRunDigest,
|
||||
definitionRef: mutation.stepRun.definitionRef,
|
||||
definitionDigest: mutation.stepRun.definitionDigest,
|
||||
projectId: 'project-001',
|
||||
runStatus: 'running',
|
||||
runVersion: String(mutation.expectedRunVersion),
|
||||
runEventSequence: String(mutation.expectedRunEventSequence),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
});
|
||||
await assert.rejects(
|
||||
repositoryFor(client).prepare(start),
|
||||
ToolExecutionStartBarrierConflictError,
|
||||
);
|
||||
assert.equal(
|
||||
client.calls.some(({ text }) => text.startsWith('INSERT INTO')),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
client.calls.some(({ text }) => text === 'ROLLBACK'),
|
||||
true,
|
||||
);
|
||||
assert.equal(client.released, true);
|
||||
});
|
||||
|
||||
test('fails closed when a stored barrier projection is corrupted', async () => {
|
||||
const start = await command();
|
||||
const barrier = toolExecutionStartBarrierRecord(start);
|
||||
const repository = new PostgresToolExecutionStartBarrierRepository({
|
||||
async query(sql) {
|
||||
if (sql.includes('FROM "ql3"."tool_execution_start_barriers"')) {
|
||||
return {
|
||||
rows: [
|
||||
barrierRow(barrier, {
|
||||
storedBarrierDigest: '0'.repeat(64),
|
||||
}),
|
||||
],
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
},
|
||||
async connect() {
|
||||
throw new Error('unused');
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
repository.findByStartId(barrier.startId),
|
||||
ToolExecutionStartBarrierUnavailableError,
|
||||
);
|
||||
});
|
||||
|
||||
test('fails closed when a historical barrier has no Artifact binding', async () => {
|
||||
const start = await command();
|
||||
const barrier = toolExecutionStartBarrierRecord(start);
|
||||
const repository = new PostgresToolExecutionStartBarrierRepository({
|
||||
async query(sql) {
|
||||
if (sql.includes('FROM "ql3"."tool_execution_start_barriers"')) {
|
||||
return {
|
||||
rows: [
|
||||
barrierRow(barrier, {
|
||||
storedInputArtifactId: null,
|
||||
}),
|
||||
],
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
},
|
||||
async connect() {
|
||||
throw new Error('unused');
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
repository.findByStartId(barrier.startId),
|
||||
ToolExecutionStartBarrierUnavailableError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,329 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ToolInvocationArtifactConflictError,
|
||||
ToolInvocationArtifactUnavailableError,
|
||||
createToolInvocationInputArtifact,
|
||||
createToolInvocationPreviewArtifact,
|
||||
} = require('@qinglong/runtime-core/tool-invocation-artifact');
|
||||
const {
|
||||
PostgresToolInvocationArtifactRepository,
|
||||
} = require('@qinglong/cluster-postgres/tool-invocation-artifact');
|
||||
|
||||
const KEY = Buffer.alloc(32, 7);
|
||||
const NONCE = Buffer.alloc(12, 9);
|
||||
const INVOCATION_ACTION_DIGEST = 'a'.repeat(64);
|
||||
const ACTION_DIGEST = 'b'.repeat(64);
|
||||
const REDACTION_DIGEST = 'c'.repeat(64);
|
||||
|
||||
function digest(value) {
|
||||
return createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
||||
}
|
||||
|
||||
function artifacts(index = 1, overrides = {}) {
|
||||
const input = {
|
||||
runId: `run-${String(index).padStart(3, '0')}`,
|
||||
token: 'secret-value',
|
||||
};
|
||||
const common = {
|
||||
projectId: 'project-001',
|
||||
actionRef: `tool-plan:${input.runId}`,
|
||||
sealedAtMs: 1_000 + index,
|
||||
...overrides.common,
|
||||
};
|
||||
return {
|
||||
inputArtifact: createToolInvocationInputArtifact(
|
||||
{
|
||||
artifactId: `artifact-input-${index}`,
|
||||
requestedBy: { type: 'user', id: 'usr-owner' },
|
||||
tool: { name: 'demo.compare', version: '1.0.0' },
|
||||
input,
|
||||
inputDigest: digest(input),
|
||||
invocationActionDigest: INVOCATION_ACTION_DIGEST,
|
||||
keyId: 'tool-key-test',
|
||||
key: KEY,
|
||||
...common,
|
||||
...overrides.input,
|
||||
},
|
||||
() => NONCE,
|
||||
),
|
||||
previewArtifact: createToolInvocationPreviewArtifact({
|
||||
artifactId: `artifact-preview-${index}`,
|
||||
actionDigest: ACTION_DIGEST,
|
||||
redactionContractDigest: REDACTION_DIGEST,
|
||||
preview: {
|
||||
title: 'Compare Run',
|
||||
summary: 'Reads one bounded Run projection',
|
||||
fields: [
|
||||
{ kind: 'identifier', label: 'Run', value: input.runId },
|
||||
{ kind: 'redacted', label: 'Credential', value: null },
|
||||
],
|
||||
warnings: [],
|
||||
},
|
||||
...common,
|
||||
...overrides.preview,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function inputRow(artifact, overrides = {}) {
|
||||
return {
|
||||
artifactId: artifact.artifactId,
|
||||
projectId: artifact.projectId,
|
||||
actionRef: artifact.actionRef,
|
||||
inputDigest: artifact.inputDigest,
|
||||
invocationActionDigest: artifact.invocationActionDigest,
|
||||
artifactDigest: artifact.artifactDigest,
|
||||
keyId: artifact.keyId,
|
||||
algorithm: artifact.algorithm,
|
||||
plaintextBytes: String(artifact.plaintextBytes),
|
||||
sealedAtMs: String(artifact.sealedAtMs),
|
||||
artifactJson: artifact,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function previewRow(artifact, overrides = {}) {
|
||||
return {
|
||||
artifactId: artifact.artifactId,
|
||||
projectId: artifact.projectId,
|
||||
actionRef: artifact.actionRef,
|
||||
actionDigest: artifact.actionDigest,
|
||||
previewDigest: artifact.previewDigest,
|
||||
redactionContractDigest: artifact.redactionContractDigest,
|
||||
artifactDigest: artifact.artifactDigest,
|
||||
byteLength: String(artifact.byteLength),
|
||||
sealedAtMs: String(artifact.sealedAtMs),
|
||||
artifactJson: artifact,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function clientWith(handler) {
|
||||
const calls = [];
|
||||
let released = false;
|
||||
return {
|
||||
calls,
|
||||
get released() {
|
||||
return released;
|
||||
},
|
||||
async query(text, values = []) {
|
||||
calls.push({ text, values });
|
||||
if (
|
||||
text.startsWith('BEGIN') ||
|
||||
text.includes("set_config('") ||
|
||||
text === 'COMMIT' ||
|
||||
text === 'ROLLBACK'
|
||||
) {
|
||||
return { rows: [] };
|
||||
}
|
||||
return handler(text, values, calls);
|
||||
},
|
||||
release() {
|
||||
released = true;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function repositoryFor(client, query = client.query.bind(client)) {
|
||||
return new PostgresToolInvocationArtifactRepository({
|
||||
query,
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
test('atomically inserts one Artifact pair without storing plaintext JSON', async () => {
|
||||
const pair = artifacts();
|
||||
const client = clientWith(async (sql) => {
|
||||
if (sql.startsWith('SELECT')) return { rows: [] };
|
||||
if (sql.startsWith('INSERT INTO')) return { rows: [], rowCount: 1 };
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
});
|
||||
assert.deepEqual(
|
||||
await repositoryFor(client).put(
|
||||
pair.inputArtifact,
|
||||
pair.previewArtifact,
|
||||
),
|
||||
{ status: 'inserted' },
|
||||
);
|
||||
assert.equal(client.released, true);
|
||||
assert.equal(
|
||||
client.calls.filter(({ text }) => text.startsWith('INSERT INTO'))
|
||||
.length,
|
||||
2,
|
||||
);
|
||||
assert.equal(
|
||||
client.calls
|
||||
.filter(({ text }) => text.startsWith('INSERT INTO'))
|
||||
.some(({ values }) => values.some((value) =>
|
||||
String(value).includes('secret-value'),
|
||||
)),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
client.calls.some(({ text }) => text === 'COMMIT'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
client.calls.some(({ text }) => text.includes('FOR SHARE')),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('exactly replays a complete pair and rejects partial durable state', async () => {
|
||||
const pair = artifacts();
|
||||
const exact = clientWith(async (sql) => {
|
||||
if (sql.includes('tool_invocation_input_artifacts')) {
|
||||
return { rows: [inputRow(pair.inputArtifact)] };
|
||||
}
|
||||
if (sql.includes('tool_invocation_preview_artifacts')) {
|
||||
return { rows: [previewRow(pair.previewArtifact)] };
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
});
|
||||
assert.deepEqual(
|
||||
await repositoryFor(exact).put(
|
||||
pair.inputArtifact,
|
||||
pair.previewArtifact,
|
||||
),
|
||||
{ status: 'existing' },
|
||||
);
|
||||
assert.equal(
|
||||
exact.calls.some(({ text }) => text.startsWith('INSERT INTO')),
|
||||
false,
|
||||
);
|
||||
|
||||
const partial = clientWith(async (sql) => {
|
||||
if (sql.includes('tool_invocation_input_artifacts')) {
|
||||
return { rows: [inputRow(pair.inputArtifact)] };
|
||||
}
|
||||
if (sql.includes('tool_invocation_preview_artifacts')) {
|
||||
return { rows: [] };
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
});
|
||||
await assert.rejects(
|
||||
repositoryFor(partial).put(pair.inputArtifact, pair.previewArtifact),
|
||||
ToolInvocationArtifactConflictError,
|
||||
);
|
||||
assert.equal(
|
||||
partial.calls.some(({ text }) => text === 'ROLLBACK'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects detached pairs before opening a transaction', async () => {
|
||||
const first = artifacts();
|
||||
const detached = artifacts(2, {
|
||||
common: {
|
||||
projectId: 'project-001',
|
||||
actionRef: 'tool-plan:detached',
|
||||
sealedAtMs: 1_002,
|
||||
},
|
||||
});
|
||||
let connected = false;
|
||||
const repository = new PostgresToolInvocationArtifactRepository({
|
||||
async query() {
|
||||
throw new Error('unused');
|
||||
},
|
||||
async connect() {
|
||||
connected = true;
|
||||
throw new Error('unused');
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
repository.put(first.inputArtifact, detached.previewArtifact),
|
||||
ToolInvocationArtifactConflictError,
|
||||
);
|
||||
assert.equal(connected, false);
|
||||
});
|
||||
|
||||
test('converges a concurrent unique-key winner through exact replay', async () => {
|
||||
const pair = artifacts();
|
||||
const uniqueViolation = clientWith(async (sql) => {
|
||||
if (sql.startsWith('SELECT')) return { rows: [] };
|
||||
if (sql.startsWith('INSERT INTO')) {
|
||||
const error = new Error('duplicate key');
|
||||
error.code = '23505';
|
||||
throw error;
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
});
|
||||
const winner = clientWith(async (sql) => {
|
||||
if (sql.includes('tool_invocation_input_artifacts')) {
|
||||
return { rows: [inputRow(pair.inputArtifact)] };
|
||||
}
|
||||
if (sql.includes('tool_invocation_preview_artifacts')) {
|
||||
return { rows: [previewRow(pair.previewArtifact)] };
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
});
|
||||
const clients = [uniqueViolation, winner];
|
||||
let connections = 0;
|
||||
const repository = new PostgresToolInvocationArtifactRepository({
|
||||
async query() {
|
||||
throw new Error('unused');
|
||||
},
|
||||
async connect() {
|
||||
const client = clients[connections];
|
||||
connections += 1;
|
||||
return client;
|
||||
},
|
||||
});
|
||||
assert.deepEqual(
|
||||
await repository.put(pair.inputArtifact, pair.previewArtifact),
|
||||
{ status: 'existing' },
|
||||
);
|
||||
assert.equal(connections, 2);
|
||||
assert.equal(
|
||||
uniqueViolation.calls.some(({ text }) => text === 'ROLLBACK'),
|
||||
true,
|
||||
);
|
||||
assert.equal(uniqueViolation.released, true);
|
||||
assert.equal(winner.released, true);
|
||||
});
|
||||
|
||||
test('fails closed when a stored projection is corrupted', async () => {
|
||||
const pair = artifacts();
|
||||
const client = clientWith(async () => {
|
||||
throw new Error('unused');
|
||||
});
|
||||
const repository = repositoryFor(client, async (sql) => {
|
||||
if (sql.includes('tool_invocation_input_artifacts')) {
|
||||
return {
|
||||
rows: [
|
||||
inputRow(pair.inputArtifact, {
|
||||
inputDigest: 'd'.repeat(64),
|
||||
}),
|
||||
],
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
});
|
||||
await assert.rejects(
|
||||
repository.findInput(pair.inputArtifact.artifactId),
|
||||
ToolInvocationArtifactUnavailableError,
|
||||
);
|
||||
});
|
||||
|
||||
test('publishes the explicit runtime adapter subpath', () => {
|
||||
const root = require('../dist');
|
||||
const runtime = require('../dist/entrypoints/runtime');
|
||||
const subpath = require('@qinglong/cluster-postgres/tool-invocation-artifact');
|
||||
assert.equal(
|
||||
root.PostgresToolInvocationArtifactRepository,
|
||||
PostgresToolInvocationArtifactRepository,
|
||||
);
|
||||
assert.equal(
|
||||
runtime.PostgresToolInvocationArtifactRepository,
|
||||
PostgresToolInvocationArtifactRepository,
|
||||
);
|
||||
assert.equal(
|
||||
subpath.PostgresToolInvocationArtifactRepository,
|
||||
PostgresToolInvocationArtifactRepository,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,238 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ToolResultKeyCatalogConflictError,
|
||||
createToolResultKeyCatalogBootstrapCommand,
|
||||
createToolResultKeyRetirementCommand,
|
||||
createToolResultKeyRotationCommand,
|
||||
toolResultKeyMaterialProof,
|
||||
} = require('@qinglong/runtime-core/tool-result-key-catalog');
|
||||
const {
|
||||
createToolResultKeyRetirementReceipt,
|
||||
} = require('@qinglong/runtime-core/tool-result-rekey');
|
||||
const {
|
||||
PostgresToolResultKeyCatalogReader,
|
||||
} = require('@qinglong/cluster-postgres/runtime');
|
||||
const {
|
||||
PostgresToolResultKeyCatalogRepository,
|
||||
} = require('@qinglong/cluster-postgres/tool-result-key-catalog');
|
||||
|
||||
function jsonb(value) {
|
||||
if (Array.isArray(value)) return value.map(jsonb);
|
||||
if (!value || typeof value !== 'object') return value;
|
||||
return Object.fromEntries(
|
||||
Object.keys(value)
|
||||
.sort()
|
||||
.map((key) => [key, jsonb(value[key])]),
|
||||
);
|
||||
}
|
||||
|
||||
function row(catalog, commandDigest) {
|
||||
return {
|
||||
generation: String(catalog.generation),
|
||||
previousCatalogDigest: catalog.previousCatalogDigest,
|
||||
activeKeyId: catalog.activeKeyId,
|
||||
mutationKind: catalog.mutationKind,
|
||||
mutationId: catalog.mutationId,
|
||||
catalogDigest: catalog.catalogDigest,
|
||||
commandDigest,
|
||||
committedAtMs: String(catalog.committedAtMs),
|
||||
catalogJson: jsonb(catalog),
|
||||
};
|
||||
}
|
||||
|
||||
function harness() {
|
||||
const calls = [];
|
||||
const history = [];
|
||||
const receipts = [];
|
||||
let released = 0;
|
||||
const client = {
|
||||
async query(text, values = []) {
|
||||
calls.push({ text, values });
|
||||
if (
|
||||
text.startsWith('BEGIN') ||
|
||||
text === 'COMMIT' ||
|
||||
text === 'ROLLBACK' ||
|
||||
text.includes('pg_advisory_xact_lock')
|
||||
) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text.includes('AND (mutation_id =')) {
|
||||
const found = history.filter(
|
||||
({ catalog }) =>
|
||||
catalog.mutationId === values[1] ||
|
||||
catalog.generation === values[2] ||
|
||||
catalog.catalogDigest === values[3],
|
||||
);
|
||||
return {
|
||||
rows: found.map(({ catalog, commandDigest }) =>
|
||||
row(catalog, commandDigest),
|
||||
),
|
||||
};
|
||||
}
|
||||
if (text.includes('tool_result_key_retirement_receipts')) {
|
||||
const receipt = receipts.find(
|
||||
(candidate) => candidate.receiptDigest === values[0],
|
||||
);
|
||||
return { rows: receipt ? [{ receiptJson: receipt }] : [] };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."tool_result_key_catalog_generations"')) {
|
||||
return {
|
||||
rows: history
|
||||
.slice()
|
||||
.reverse()
|
||||
.slice(0, 2)
|
||||
.map(({ catalog, commandDigest }) => row(catalog, commandDigest)),
|
||||
};
|
||||
}
|
||||
if (text.includes('clock_timestamp()')) {
|
||||
return { rows: [{ now: String(1_000 + history.length) }] };
|
||||
}
|
||||
if (
|
||||
text.startsWith(
|
||||
'INSERT INTO "ql3"."tool_result_key_catalog_generations"',
|
||||
)
|
||||
) {
|
||||
history.push({
|
||||
catalog: JSON.parse(values[10]),
|
||||
commandDigest: values[8],
|
||||
});
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${text}`);
|
||||
},
|
||||
release() {
|
||||
released += 1;
|
||||
},
|
||||
};
|
||||
const pool = {
|
||||
async query(text, values) {
|
||||
return client.query(text, values);
|
||||
},
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
};
|
||||
return {
|
||||
calls,
|
||||
history,
|
||||
receipts,
|
||||
pool,
|
||||
repository: new PostgresToolResultKeyCatalogRepository(pool),
|
||||
reader: new PostgresToolResultKeyCatalogReader(pool),
|
||||
released: () => released,
|
||||
};
|
||||
}
|
||||
|
||||
function bootstrapCommand() {
|
||||
return createToolResultKeyCatalogBootstrapCommand({
|
||||
keyId: 'tool-result-key-001',
|
||||
materialProof: toolResultKeyMaterialProof(
|
||||
'tool-result-key-001',
|
||||
Buffer.alloc(32, 1),
|
||||
),
|
||||
mutationId: 'tool-result-key-bootstrap-001',
|
||||
});
|
||||
}
|
||||
|
||||
test('serializes PostgreSQL key generations behind the catalog lock', async () => {
|
||||
const current = harness();
|
||||
assert.equal(await current.reader.findCurrent(), null);
|
||||
const bootstrap = bootstrapCommand();
|
||||
const first = await current.repository.append(bootstrap);
|
||||
assert.equal(first.status, 'created');
|
||||
assert.deepEqual(await current.repository.append(bootstrap), {
|
||||
status: 'existing',
|
||||
catalog: first.catalog,
|
||||
});
|
||||
|
||||
const second = await current.repository.append(
|
||||
createToolResultKeyRotationCommand(first.catalog, {
|
||||
keyId: 'tool-result-key-002',
|
||||
materialProof: toolResultKeyMaterialProof(
|
||||
'tool-result-key-002',
|
||||
Buffer.alloc(32, 2),
|
||||
),
|
||||
mutationId: 'tool-result-key-rotate-002',
|
||||
}),
|
||||
);
|
||||
assert.equal(second.catalog.generation, 2);
|
||||
assert.deepEqual(await current.reader.findCurrent(), second.catalog);
|
||||
assert.equal(current.history.length, 2);
|
||||
assert.equal(current.released(), 3);
|
||||
|
||||
await assert.rejects(
|
||||
current.repository.append(
|
||||
createToolResultKeyRetirementCommand(second.catalog, {
|
||||
keyId: 'tool-result-key-001',
|
||||
retirementReceiptDigest: 'f'.repeat(64),
|
||||
mutationId: 'tool-result-key-retire-forged-001',
|
||||
}),
|
||||
),
|
||||
ToolResultKeyCatalogConflictError,
|
||||
);
|
||||
const retiring = second.catalog.keys.find(
|
||||
(entry) => entry.keyId === 'tool-result-key-001',
|
||||
);
|
||||
const receipt = createToolResultKeyRetirementReceipt({
|
||||
catalogGeneration: second.catalog.generation,
|
||||
catalogDigest: second.catalog.catalogDigest,
|
||||
keyId: retiring.keyId,
|
||||
materialProof: retiring.materialProof,
|
||||
mutationId: 'tool-result-key-retirement-receipt-001',
|
||||
bindingCount: 0,
|
||||
overlayHeadCount: 0,
|
||||
coverageDigest: 'c'.repeat(64),
|
||||
createdAtMs: 1_500,
|
||||
});
|
||||
current.receipts.push(receipt);
|
||||
const retired = await current.repository.append(
|
||||
createToolResultKeyRetirementCommand(second.catalog, {
|
||||
keyId: 'tool-result-key-001',
|
||||
retirementReceiptDigest: receipt.receiptDigest,
|
||||
mutationId: 'tool-result-key-retire-001',
|
||||
}),
|
||||
);
|
||||
assert.equal(
|
||||
retired.catalog.keys.find(
|
||||
(entry) => entry.keyId === 'tool-result-key-001',
|
||||
).state,
|
||||
'retired',
|
||||
);
|
||||
|
||||
const transactionCalls = current.calls
|
||||
.map(({ text }) => text)
|
||||
.filter(
|
||||
(text) =>
|
||||
text.startsWith('BEGIN') ||
|
||||
text.includes('pg_advisory_xact_lock') ||
|
||||
text.startsWith(
|
||||
'INSERT INTO "ql3"."tool_result_key_catalog_generations"',
|
||||
) ||
|
||||
text === 'COMMIT',
|
||||
);
|
||||
assert.equal(
|
||||
transactionCalls.indexOf('SELECT pg_advisory_xact_lock(190397473, 3)') <
|
||||
transactionCalls.findIndex((text) =>
|
||||
text.startsWith(
|
||||
'INSERT INTO "ql3"."tool_result_key_catalog_generations"',
|
||||
),
|
||||
),
|
||||
true,
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
current.repository.append(
|
||||
createToolResultKeyRotationCommand(first.catalog, {
|
||||
keyId: 'tool-result-key-stale',
|
||||
materialProof: toolResultKeyMaterialProof(
|
||||
'tool-result-key-stale',
|
||||
Buffer.alloc(32, 3),
|
||||
),
|
||||
mutationId: 'tool-result-key-stale-003',
|
||||
}),
|
||||
),
|
||||
ToolResultKeyCatalogConflictError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,476 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
BUILTIN_RUN_READ_TOOL,
|
||||
BUILTIN_RUN_READ_TOOL_DEFINITION,
|
||||
} = require('@qinglong/runtime-core/builtin-run-read-tool');
|
||||
const {
|
||||
TRUSTED_TOOL_EXECUTION_RESULT_SCHEMA,
|
||||
} = require('@qinglong/runtime-core/trusted-tool-execution');
|
||||
const {
|
||||
TOOL_EXECUTION_RESULT_KEY_BINDING_SCHEMA,
|
||||
createToolExecutionResultArtifact,
|
||||
normalizeToolExecutionResultKeyBinding,
|
||||
} = require('@qinglong/runtime-core/tool-execution-completion');
|
||||
const {
|
||||
createToolResultKeyCatalogBootstrapCommand,
|
||||
createToolResultKeyRotationCommand,
|
||||
normalizeToolResultKeyCatalogRecord,
|
||||
requireActiveToolResultKey,
|
||||
toolResultKeyCatalogFence,
|
||||
toolResultKeyMaterialProof,
|
||||
} = require('@qinglong/runtime-core/tool-result-key-catalog');
|
||||
const {
|
||||
ToolExecutionResultRekeyConflictError,
|
||||
createToolExecutionResultRekeyCommand,
|
||||
createToolResultKeyRetirementReceiptCommand,
|
||||
} = require('@qinglong/runtime-core/tool-result-rekey');
|
||||
const {
|
||||
ToolDefinitionRegistry,
|
||||
} = require('@qinglong/runtime-core/tool-registry');
|
||||
const {
|
||||
PostgresToolResultRekeyReader,
|
||||
} = require('@qinglong/cluster-postgres/runtime');
|
||||
const {
|
||||
PostgresToolResultRekeyRepository,
|
||||
} = require('@qinglong/cluster-postgres/tool-result-rekey');
|
||||
|
||||
const KEY_A = Buffer.alloc(32, 1);
|
||||
const KEY_B = Buffer.alloc(32, 2);
|
||||
const OUTPUT_DIGEST_DOMAIN = Buffer.from(
|
||||
'qinglong/trusted-tool-execution-output-digest@v1\0',
|
||||
);
|
||||
const RESULT_DIGEST_DOMAIN = Buffer.from(
|
||||
'qinglong/trusted-tool-execution-result-digest@v1\0',
|
||||
);
|
||||
const BINDING_DIGEST_DOMAIN = Buffer.from(
|
||||
'qinglong/tool-execution-result-key-binding-digest@v1\0',
|
||||
);
|
||||
|
||||
function hash(domain, value) {
|
||||
return createHash('sha256')
|
||||
.update(domain)
|
||||
.update(JSON.stringify(value))
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
function jsonb(value) {
|
||||
if (Array.isArray(value)) return value.map(jsonb);
|
||||
if (!value || typeof value !== 'object') return value;
|
||||
return Object.fromEntries(
|
||||
Object.keys(value)
|
||||
.sort()
|
||||
.map((key) => [key, jsonb(value[key])]),
|
||||
);
|
||||
}
|
||||
|
||||
function output() {
|
||||
return {
|
||||
createdAtMs: 1_000,
|
||||
eventSequence: 3,
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
found: true,
|
||||
id: 'run-rekey-postgres-001',
|
||||
priority: 10,
|
||||
queuedAtMs: 1_100,
|
||||
startedAtMs: 1_200,
|
||||
status: 'succeeded',
|
||||
taskId: 'task-rekey-postgres-001',
|
||||
taskRevision: 'task-rekey-postgres-001@1',
|
||||
version: 2,
|
||||
};
|
||||
}
|
||||
|
||||
function registry() {
|
||||
return new ToolDefinitionRegistry([BUILTIN_RUN_READ_TOOL_DEFINITION]);
|
||||
}
|
||||
|
||||
function sourceArtifact() {
|
||||
const value = output();
|
||||
const unsigned = {
|
||||
schema: TRUSTED_TOOL_EXECUTION_RESULT_SCHEMA,
|
||||
startId: 'tool-start-rekey-postgres-001',
|
||||
barrierDigest: 'a'.repeat(64),
|
||||
adapterDigest: 'b'.repeat(64),
|
||||
output: value,
|
||||
outputDigest: hash(OUTPUT_DIGEST_DOMAIN, value),
|
||||
completedAtMs: 1_500,
|
||||
};
|
||||
const result = {
|
||||
...unsigned,
|
||||
resultDigest: hash(RESULT_DIGEST_DOMAIN, unsigned),
|
||||
};
|
||||
return createToolExecutionResultArtifact(
|
||||
{
|
||||
artifactId: 'artifact-result-rekey-postgres-001',
|
||||
projectId: 'project-rekey-postgres-001',
|
||||
runId: 'run-host-rekey-postgres-001',
|
||||
stepRunId: 'step-run-rekey-postgres-001',
|
||||
tool: BUILTIN_RUN_READ_TOOL,
|
||||
executionResult: result,
|
||||
keyId: 'result-key-a',
|
||||
key: KEY_A,
|
||||
},
|
||||
registry(),
|
||||
() => Buffer.alloc(12, 4),
|
||||
);
|
||||
}
|
||||
|
||||
function sourceBinding(artifact, catalog) {
|
||||
const unsigned = {
|
||||
schema: TOOL_EXECUTION_RESULT_KEY_BINDING_SCHEMA,
|
||||
startId: artifact.startId,
|
||||
artifactId: artifact.artifactId,
|
||||
artifactDigest: artifact.artifactDigest,
|
||||
catalogGeneration: catalog.generation,
|
||||
catalogDigest: catalog.catalogDigest,
|
||||
keyId: artifact.keyId,
|
||||
materialProof: toolResultKeyMaterialProof(artifact.keyId, KEY_A),
|
||||
};
|
||||
return normalizeToolExecutionResultKeyBinding({
|
||||
...unsigned,
|
||||
bindingDigest: hash(BINDING_DIGEST_DOMAIN, unsigned),
|
||||
});
|
||||
}
|
||||
|
||||
function catalogs() {
|
||||
const bootstrap = createToolResultKeyCatalogBootstrapCommand({
|
||||
keyId: 'result-key-a',
|
||||
materialProof: toolResultKeyMaterialProof('result-key-a', KEY_A),
|
||||
mutationId: 'result-key-bootstrap-postgres-a',
|
||||
});
|
||||
const first = normalizeToolResultKeyCatalogRecord({
|
||||
...bootstrap.next,
|
||||
committedAtMs: 1_000,
|
||||
});
|
||||
const rotation = createToolResultKeyRotationCommand(first, {
|
||||
keyId: 'result-key-b',
|
||||
materialProof: toolResultKeyMaterialProof('result-key-b', KEY_B),
|
||||
mutationId: 'result-key-rotate-postgres-b',
|
||||
});
|
||||
return {
|
||||
first,
|
||||
second: normalizeToolResultKeyCatalogRecord({
|
||||
...rotation.next,
|
||||
committedAtMs: 1_001,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function overlayRow(overlay, commandDigest) {
|
||||
return {
|
||||
overlayId: overlay.overlayId,
|
||||
artifactId: overlay.sourceArtifact.artifactId,
|
||||
sourceBindingDigest: overlay.sourceBindingDigest,
|
||||
revision: String(overlay.revision),
|
||||
previousOverlayDigest: overlay.previousOverlayDigest,
|
||||
fromKeyId: overlay.fromKeyId,
|
||||
targetCatalogGeneration: String(overlay.targetCatalogFence.generation),
|
||||
targetCatalogDigest: overlay.targetCatalogFence.catalogDigest,
|
||||
targetKeyId: overlay.targetCatalogFence.keyId,
|
||||
targetMaterialProof: overlay.targetCatalogFence.materialProof,
|
||||
mutationId: overlay.mutationId,
|
||||
commandDigest,
|
||||
overlayDigest: overlay.overlayDigest,
|
||||
rekeyedAtMs: String(overlay.rekeyedAtMs),
|
||||
overlayJson: jsonb(overlay),
|
||||
};
|
||||
}
|
||||
|
||||
function receiptRow(receipt, commandDigest) {
|
||||
return {
|
||||
receiptDigest: receipt.receiptDigest,
|
||||
catalogGeneration: String(receipt.catalogGeneration),
|
||||
catalogDigest: receipt.catalogDigest,
|
||||
keyId: receipt.keyId,
|
||||
materialProof: receipt.materialProof,
|
||||
mutationId: receipt.mutationId,
|
||||
commandDigest,
|
||||
bindingCount: String(receipt.bindingCount),
|
||||
overlayHeadCount: String(receipt.overlayHeadCount),
|
||||
uncoveredBindingCount: '0',
|
||||
uncoveredOverlayHeadCount: '0',
|
||||
coverageDigest: receipt.coverageDigest,
|
||||
createdAtMs: String(receipt.createdAtMs),
|
||||
receiptJson: jsonb(receipt),
|
||||
};
|
||||
}
|
||||
|
||||
function harness(artifact, binding, catalog) {
|
||||
const calls = [];
|
||||
let overlayState = null;
|
||||
let overlayCommandDigest = null;
|
||||
let head = null;
|
||||
let receiptState = null;
|
||||
let receiptCommandDigest = null;
|
||||
let released = 0;
|
||||
const client = {
|
||||
async query(text, values = []) {
|
||||
calls.push({ text, values });
|
||||
if (
|
||||
text.startsWith('BEGIN') ||
|
||||
text === 'COMMIT' ||
|
||||
text === 'ROLLBACK' ||
|
||||
text.includes("set_config('") ||
|
||||
text.includes('pg_advisory_xact_lock')
|
||||
) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (
|
||||
text.startsWith(
|
||||
'INSERT INTO "ql3"."tool_execution_result_rekey_overlays"',
|
||||
)
|
||||
) {
|
||||
overlayState = JSON.parse(values[15]);
|
||||
overlayCommandDigest = values[12];
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('tool_execution_result_rekey_overlays')) {
|
||||
if (!overlayState) return { rows: [] };
|
||||
if (text.includes('JOIN "ql3"."tool_execution_result_rekey_heads"')) {
|
||||
return {
|
||||
rows:
|
||||
head && values[0] === overlayState.sourceArtifact.artifactId
|
||||
? [overlayRow(overlayState, overlayCommandDigest)]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
const matches =
|
||||
values.length === 0 ||
|
||||
values.includes(overlayState.overlayId) ||
|
||||
values.includes(overlayState.overlayDigest) ||
|
||||
values.includes(overlayState.mutationId);
|
||||
return {
|
||||
rows: matches
|
||||
? [overlayRow(overlayState, overlayCommandDigest)]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes('tool_execution_result_key_bindings') &&
|
||||
text.includes('tool_execution_completions')
|
||||
) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
bindingArtifactDigest: binding.artifactDigest,
|
||||
bindingKeyId: binding.keyId,
|
||||
bindingDigest: binding.bindingDigest,
|
||||
artifactDigest: artifact.artifactDigest,
|
||||
outputDigest: artifact.outputDigest,
|
||||
executionResultDigest: artifact.executionResultDigest,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes('tool_execution_result_rekey_heads') &&
|
||||
text.includes('FOR UPDATE')
|
||||
) {
|
||||
return {
|
||||
rows: head
|
||||
? [
|
||||
{
|
||||
revision: String(head.revision),
|
||||
overlayDigest: head.overlayDigest,
|
||||
targetKeyId: head.targetKeyId,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
if (text.includes('tool_result_key_catalog_generations')) {
|
||||
return { rows: [{ catalogJson: catalog }] };
|
||||
}
|
||||
if (
|
||||
text.startsWith(
|
||||
'INSERT INTO "ql3"."tool_execution_result_rekey_heads"',
|
||||
)
|
||||
) {
|
||||
head = {
|
||||
revision: values[1],
|
||||
overlayDigest: values[3],
|
||||
targetKeyId: values[6],
|
||||
};
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('tool_result_key_retirement_receipts')) {
|
||||
if (text.startsWith('INSERT')) {
|
||||
receiptState = JSON.parse(values[12]);
|
||||
receiptCommandDigest = values[7];
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
const matches =
|
||||
receiptState &&
|
||||
(values.includes(receiptState.mutationId) ||
|
||||
values.includes(receiptState.receiptDigest));
|
||||
return {
|
||||
rows: matches
|
||||
? [receiptRow(receiptState, receiptCommandDigest)]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes('LEFT JOIN "ql3"."tool_execution_result_rekey_heads"')
|
||||
) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
artifactId: artifact.artifactId,
|
||||
bindingDigest: binding.bindingDigest,
|
||||
bindingKeyId: binding.keyId,
|
||||
headOverlayDigest: head.overlayDigest,
|
||||
headTargetKeyId: head.targetKeyId,
|
||||
headTargetCatalogGeneration: String(catalog.generation),
|
||||
headTargetCatalogDigest: catalog.catalogDigest,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('clock_timestamp()')) {
|
||||
return { rows: [{ now: '1700000000000' }] };
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${text}`);
|
||||
},
|
||||
release() {
|
||||
released += 1;
|
||||
},
|
||||
};
|
||||
const pool = {
|
||||
query: (text, values) => client.query(text, values),
|
||||
connect: async () => client,
|
||||
};
|
||||
return {
|
||||
calls,
|
||||
reader: new PostgresToolResultRekeyReader(pool),
|
||||
repository: new PostgresToolResultRekeyRepository(pool),
|
||||
released: () => released,
|
||||
};
|
||||
}
|
||||
|
||||
test('serializes PostgreSQL rekey heads and coverage receipts behind the catalog lock', async () => {
|
||||
const { first, second } = catalogs();
|
||||
const artifact = sourceArtifact();
|
||||
const binding = sourceBinding(artifact, first);
|
||||
const current = harness(artifact, binding, second);
|
||||
assert.equal(
|
||||
await current.reader.findHeadByArtifactId(artifact.artifactId),
|
||||
null,
|
||||
);
|
||||
|
||||
const overlayCommand = createToolExecutionResultRekeyCommand({
|
||||
artifact,
|
||||
binding,
|
||||
previousOverlay: null,
|
||||
overlayId: 'result-rekey-overlay-postgres-001',
|
||||
mutationId: 'result-rekey-mutation-postgres-001',
|
||||
targetCatalogFence: toolResultKeyCatalogFence(
|
||||
second,
|
||||
requireActiveToolResultKey(second),
|
||||
),
|
||||
targetKey: KEY_B,
|
||||
output: output(),
|
||||
rekeyedAtMs: 1_700,
|
||||
registry: registry(),
|
||||
nonceFactory: () => Buffer.alloc(12, 5),
|
||||
});
|
||||
const appended = await current.repository.append(overlayCommand);
|
||||
assert.equal(appended.status, 'created');
|
||||
assert.deepEqual(await current.repository.append(overlayCommand), {
|
||||
status: 'existing',
|
||||
overlay: appended.overlay,
|
||||
});
|
||||
assert.deepEqual(
|
||||
await current.reader.findHeadByArtifactId(artifact.artifactId),
|
||||
appended.overlay,
|
||||
);
|
||||
|
||||
const receiptCommand = createToolResultKeyRetirementReceiptCommand({
|
||||
expectedCatalogGeneration: second.generation,
|
||||
expectedCatalogDigest: second.catalogDigest,
|
||||
keyId: 'result-key-a',
|
||||
mutationId: 'result-key-retirement-postgres-a',
|
||||
});
|
||||
const receipt = await current.repository.create(receiptCommand);
|
||||
assert.equal(receipt.status, 'created');
|
||||
assert.equal(receipt.receipt.bindingCount, 1);
|
||||
assert.equal(receipt.receipt.overlayHeadCount, 1);
|
||||
assert.deepEqual(await current.repository.create(receiptCommand), {
|
||||
status: 'existing',
|
||||
receipt: receipt.receipt,
|
||||
});
|
||||
assert.deepEqual(
|
||||
await current.repository.findByDigest(receipt.receipt.receiptDigest),
|
||||
receipt.receipt,
|
||||
);
|
||||
assert.equal(current.released(), 4);
|
||||
|
||||
const statements = current.calls.map(({ text }) => text);
|
||||
const advisory = statements.indexOf(
|
||||
'SELECT pg_advisory_xact_lock(190397473, 3)',
|
||||
);
|
||||
const overlayInsert = statements.findIndex((text) =>
|
||||
text.startsWith(
|
||||
'INSERT INTO "ql3"."tool_execution_result_rekey_overlays"',
|
||||
),
|
||||
);
|
||||
assert.equal(advisory >= 0 && advisory < overlayInsert, true);
|
||||
const coverageCall = current.calls.find(({ text }) =>
|
||||
text.includes(
|
||||
'LEFT JOIN "ql3"."tool_execution_result_rekey_heads" AS head',
|
||||
),
|
||||
);
|
||||
assert.deepEqual(coverageCall.values, ['result-key-a', '', 64]);
|
||||
const currentCatalogCalls = current.calls.filter(({ text }) =>
|
||||
text.includes('tool_result_key_catalog_generations'),
|
||||
);
|
||||
assert.equal(
|
||||
currentCatalogCalls.every(({ text }) => text.includes('LIMIT 1')),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
current.calls.some(({ text }) => text.includes('FOR UPDATE OF binding')),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a stale PostgreSQL rekey head before writing an overlay', async () => {
|
||||
const { first, second } = catalogs();
|
||||
const artifact = sourceArtifact();
|
||||
const binding = sourceBinding(artifact, first);
|
||||
const current = harness(artifact, binding, second);
|
||||
const command = createToolExecutionResultRekeyCommand({
|
||||
artifact,
|
||||
binding,
|
||||
previousOverlay: null,
|
||||
overlayId: 'result-rekey-overlay-postgres-stale',
|
||||
mutationId: 'result-rekey-mutation-postgres-stale',
|
||||
targetCatalogFence: toolResultKeyCatalogFence(
|
||||
second,
|
||||
requireActiveToolResultKey(second),
|
||||
),
|
||||
targetKey: KEY_B,
|
||||
output: output(),
|
||||
rekeyedAtMs: 1_700,
|
||||
registry: registry(),
|
||||
nonceFactory: () => Buffer.alloc(12, 6),
|
||||
});
|
||||
const drifted = {
|
||||
...command,
|
||||
overlay: {
|
||||
...command.overlay,
|
||||
fromKeyId: 'result-key-drift',
|
||||
},
|
||||
};
|
||||
assert.throws(
|
||||
() => current.repository.append(drifted),
|
||||
/rekey overlay digest does not match/,
|
||||
);
|
||||
await assert.rejects(
|
||||
current.reader.findHeadByArtifactId('../escape'),
|
||||
/source Artifact id is invalid/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,149 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
createToolResultKeyCatalogBootstrapCommand,
|
||||
createToolResultKeyRotationCommand,
|
||||
normalizeToolResultKeyCatalogRecord,
|
||||
toolResultKeyMaterialProof,
|
||||
} = require('@qinglong/runtime-core/tool-result-key-catalog');
|
||||
const {
|
||||
createToolResultKeyRetirementReceiptCommand,
|
||||
} = require('@qinglong/runtime-core/tool-result-rekey');
|
||||
const {
|
||||
PostgresToolResultRekeyRepository,
|
||||
} = require('@qinglong/cluster-postgres/tool-result-rekey');
|
||||
|
||||
const COVERAGE_COUNT = 129;
|
||||
const PAGE_SIZE = 64;
|
||||
const KEY_A = Buffer.alloc(32, 1);
|
||||
const KEY_B = Buffer.alloc(32, 2);
|
||||
|
||||
function digest(value) {
|
||||
return createHash('sha256').update(value).digest('hex');
|
||||
}
|
||||
|
||||
function rotatedCatalog() {
|
||||
const bootstrap = createToolResultKeyCatalogBootstrapCommand({
|
||||
keyId: 'result-key-a',
|
||||
materialProof: toolResultKeyMaterialProof('result-key-a', KEY_A),
|
||||
mutationId: 'coverage-pressure-bootstrap-a',
|
||||
});
|
||||
const first = normalizeToolResultKeyCatalogRecord({
|
||||
...bootstrap.next,
|
||||
committedAtMs: 1_000,
|
||||
});
|
||||
const rotation = createToolResultKeyRotationCommand(first, {
|
||||
keyId: 'result-key-b',
|
||||
materialProof: toolResultKeyMaterialProof('result-key-b', KEY_B),
|
||||
mutationId: 'coverage-pressure-rotate-b',
|
||||
});
|
||||
return normalizeToolResultKeyCatalogRecord({
|
||||
...rotation.next,
|
||||
committedAtMs: 1_001,
|
||||
});
|
||||
}
|
||||
|
||||
function coverageRows(catalog) {
|
||||
return Array.from({ length: COVERAGE_COUNT }, (_, index) => {
|
||||
const suffix = String(index).padStart(3, '0');
|
||||
return {
|
||||
artifactId: `artifact-coverage-pressure-${suffix}`,
|
||||
bindingDigest: digest(`binding-${suffix}`),
|
||||
bindingKeyId: 'result-key-a',
|
||||
headOverlayDigest: digest(`overlay-${suffix}`),
|
||||
headTargetKeyId: 'result-key-b',
|
||||
headTargetCatalogGeneration: String(catalog.generation),
|
||||
headTargetCatalogDigest: catalog.catalogDigest,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function harness(catalog) {
|
||||
const rows = coverageRows(catalog);
|
||||
const coverageCalls = [];
|
||||
let released = 0;
|
||||
const client = {
|
||||
async query(statement, values = []) {
|
||||
if (
|
||||
statement.startsWith('BEGIN') ||
|
||||
statement === 'COMMIT' ||
|
||||
statement === 'ROLLBACK' ||
|
||||
statement.includes("set_config('") ||
|
||||
statement.includes('pg_advisory_xact_lock')
|
||||
) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (statement.includes('tool_result_key_retirement_receipts')) {
|
||||
if (statement.startsWith('INSERT')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
return { rows: [] };
|
||||
}
|
||||
if (statement.includes('tool_result_key_catalog_generations')) {
|
||||
return { rows: [{ catalogJson: catalog }] };
|
||||
}
|
||||
if (
|
||||
statement.includes(
|
||||
'LEFT JOIN "ql3"."tool_execution_result_rekey_heads" AS head',
|
||||
)
|
||||
) {
|
||||
const cursor = values[1];
|
||||
const limit = values[2];
|
||||
coverageCalls.push({ cursor, limit });
|
||||
const start = rows.findIndex((row) => row.artifactId > cursor);
|
||||
return {
|
||||
rows: start < 0 ? [] : rows.slice(start, start + limit),
|
||||
};
|
||||
}
|
||||
if (statement.includes('clock_timestamp()')) {
|
||||
return { rows: [{ now: '1700000000000' }] };
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${statement}`);
|
||||
},
|
||||
release() {
|
||||
released += 1;
|
||||
},
|
||||
};
|
||||
const pool = {
|
||||
connect: async () => client,
|
||||
query: (statement, values) => client.query(statement, values),
|
||||
};
|
||||
return {
|
||||
coverageCalls,
|
||||
released: () => released,
|
||||
repository: new PostgresToolResultRekeyRepository(pool),
|
||||
};
|
||||
}
|
||||
|
||||
test('streams 129 covered PostgreSQL bindings through three 64-row keyset pages', async () => {
|
||||
const catalog = rotatedCatalog();
|
||||
const current = harness(catalog);
|
||||
const result = await current.repository.create(
|
||||
createToolResultKeyRetirementReceiptCommand({
|
||||
expectedCatalogGeneration: catalog.generation,
|
||||
expectedCatalogDigest: catalog.catalogDigest,
|
||||
keyId: 'result-key-a',
|
||||
mutationId: 'coverage-pressure-retirement-receipt',
|
||||
}),
|
||||
);
|
||||
|
||||
assert.equal(result.status, 'created');
|
||||
assert.equal(result.receipt.bindingCount, COVERAGE_COUNT);
|
||||
assert.equal(result.receipt.overlayHeadCount, COVERAGE_COUNT);
|
||||
assert.equal(result.receipt.uncoveredBindingCount, 0);
|
||||
assert.equal(result.receipt.uncoveredOverlayHeadCount, 0);
|
||||
assert.deepEqual(current.coverageCalls, [
|
||||
{ cursor: '', limit: PAGE_SIZE },
|
||||
{
|
||||
cursor: 'artifact-coverage-pressure-063',
|
||||
limit: PAGE_SIZE,
|
||||
},
|
||||
{
|
||||
cursor: 'artifact-coverage-pressure-127',
|
||||
limit: PAGE_SIZE,
|
||||
},
|
||||
]);
|
||||
assert.equal(current.released(), 1);
|
||||
});
|
||||
+399
@@ -0,0 +1,399 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
WorkerCredentialDeliveryConflictError,
|
||||
} = require('@qinglong/runtime-core/worker-credential-delivery');
|
||||
const {
|
||||
PostgresWorkerCredentialAdministrationRepository,
|
||||
} = require('../dist/entrypoints/admin');
|
||||
|
||||
const MUTATION_ID = '123e4567-e89b-42d3-a456-426614174801';
|
||||
|
||||
function credentialCommand() {
|
||||
return {
|
||||
expectedCurrentVersion: 0,
|
||||
credential: {
|
||||
credentialId: 'worker_generation_2',
|
||||
version: 1,
|
||||
state: 'active',
|
||||
workerId: 'edge-router-1',
|
||||
secretDigest: 'a'.repeat(64),
|
||||
createdAtMs: 1_000,
|
||||
notBeforeAtMs: 1_000,
|
||||
expiresAtMs: 2_000,
|
||||
},
|
||||
mutation: {
|
||||
mutationId: MUTATION_ID,
|
||||
operation: 'issue',
|
||||
credentialId: 'worker_generation_2',
|
||||
credentialVersion: 1,
|
||||
expectedPreviousVersion: 0,
|
||||
changedBy: { type: 'user', id: 'usr_admin' },
|
||||
createdAtMs: 1_000,
|
||||
},
|
||||
audit: {
|
||||
eventId: MUTATION_ID,
|
||||
requestId: 'request-worker-delivery-1',
|
||||
operationId: 'worker_credential.issue',
|
||||
projectId: null,
|
||||
subject: { type: 'user', id: 'usr_admin' },
|
||||
authenticationId: 'session:admin:1',
|
||||
outcome: 'allowed',
|
||||
reasons: ['worker_credential_admin'],
|
||||
fence: null,
|
||||
occurredAtMs: 1_000,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function delivery(overrides = {}) {
|
||||
return {
|
||||
deliveryId: MUTATION_ID,
|
||||
version: 1,
|
||||
state: 'credential_committed',
|
||||
workerId: 'edge-router-1',
|
||||
credentialId: 'worker_generation_2',
|
||||
credentialVersion: 1,
|
||||
previousCredentialId: 'worker_generation_1',
|
||||
secretDigest: 'a'.repeat(64),
|
||||
tokenDigest: 'b'.repeat(64),
|
||||
deploymentTargetDigest: 'c'.repeat(64),
|
||||
deploymentGeneration: 'secret-generation-2',
|
||||
stagedAtMs: 1_000,
|
||||
credentialCommittedAtMs: 1_000,
|
||||
publishedAtMs: null,
|
||||
publicationDigest: null,
|
||||
observedAtMs: null,
|
||||
observedSessionId: null,
|
||||
observedSessionVersion: null,
|
||||
previousRevokedAtMs: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function stageIntent(overrides = {}) {
|
||||
const { version, state, credentialCommittedAtMs, publishedAtMs,
|
||||
publicationDigest, observedAtMs, observedSessionId,
|
||||
observedSessionVersion, previousRevokedAtMs, ...intent } = delivery();
|
||||
return { ...intent, ...overrides };
|
||||
}
|
||||
|
||||
function fakePool() {
|
||||
const state = {
|
||||
audit: null,
|
||||
credential: null,
|
||||
mutation: null,
|
||||
deliveries: [],
|
||||
discards: [],
|
||||
calls: [],
|
||||
releases: 0,
|
||||
loseFirstCommitResponse: false,
|
||||
};
|
||||
|
||||
function mutationRow() {
|
||||
if (!state.mutation) return [];
|
||||
return [{
|
||||
mutationId: state.mutation.mutationId,
|
||||
operation: state.mutation.operation,
|
||||
credentialId: state.mutation.credentialId,
|
||||
credentialVersion: state.mutation.credentialVersion,
|
||||
expectedPreviousVersion: state.mutation.expectedPreviousVersion,
|
||||
changedByType: state.mutation.changedByType,
|
||||
changedById: state.mutation.changedById,
|
||||
createdAtMs: state.mutation.createdAtMs,
|
||||
state: state.credential.state,
|
||||
workerId: state.credential.workerId,
|
||||
secretDigest: state.credential.secretDigest,
|
||||
notBeforeAtMs: state.credential.notBeforeAtMs,
|
||||
expiresAtMs: state.credential.expiresAtMs,
|
||||
auditEventId: state.audit.eventId,
|
||||
auditRequestId: state.audit.requestId,
|
||||
auditOperationId: state.audit.operationId,
|
||||
auditProjectId: state.audit.projectId,
|
||||
auditSubjectType: state.audit.subjectType,
|
||||
auditSubjectId: state.audit.subjectId,
|
||||
auditAuthenticationId: state.audit.authenticationId,
|
||||
auditOutcome: state.audit.outcome,
|
||||
auditReasons: state.audit.reasons,
|
||||
auditProjectVersion: state.audit.projectVersion,
|
||||
auditBindingVersion: state.audit.bindingVersion,
|
||||
auditOccurredAtMs: state.audit.occurredAtMs,
|
||||
}];
|
||||
}
|
||||
|
||||
function deliveryRows() {
|
||||
return state.deliveries.map((value) => ({ ...value }));
|
||||
}
|
||||
|
||||
async function query(text, params = []) {
|
||||
state.calls.push(text);
|
||||
if (text === 'COMMIT') {
|
||||
if (state.loseFirstCommitResponse) {
|
||||
state.loseFirstCommitResponse = false;
|
||||
const error = new Error('commit response lost');
|
||||
error.code = '40001';
|
||||
throw error;
|
||||
}
|
||||
return { rows: [] };
|
||||
}
|
||||
if (
|
||||
text === 'ROLLBACK' ||
|
||||
text.startsWith('BEGIN') ||
|
||||
text.includes('set_config') ||
|
||||
text.includes('pg_advisory_xact_lock')
|
||||
) return { rows: [] };
|
||||
if (text.includes('FROM "ql3"."worker_credential_mutations" AS mutation')) {
|
||||
return { rows: mutationRow() };
|
||||
}
|
||||
if (text.startsWith('WITH observation AS (') &&
|
||||
text.includes('worker_credential_stage_discards')) {
|
||||
const latest = new Map();
|
||||
for (const record of state.discards) latest.set(record.deliveryId, record);
|
||||
const rows = [...latest.values()]
|
||||
.filter((record) => record.state === 'discard_authorized')
|
||||
.sort((a, b) => a.deliveryId.localeCompare(b.deliveryId))
|
||||
.slice(0, params[1])
|
||||
.map((record) => ({ ...record, observedAtMs: 1_300 }));
|
||||
return { rows: rows.length > 0 ? rows : [{ deliveryId: null, observedAtMs: 1_300 }] };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."worker_credential_stage_discards"')) {
|
||||
return { rows: state.discards.map((value) => ({ ...value })) };
|
||||
}
|
||||
if (text.includes('AS "authorizedAtMs"') && text.includes('mutationExists')) {
|
||||
return { rows: [{
|
||||
authorizedAtMs: 1_100,
|
||||
mutationExists: state.mutation !== null,
|
||||
deliveryExists: state.deliveries.length > 0,
|
||||
}] };
|
||||
}
|
||||
if (text.includes('AS "discardedAtMs"')) {
|
||||
return { rows: [{ discardedAtMs: 1_200 }] };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."worker_credential_deliveries"')) {
|
||||
return { rows: deliveryRows() };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."worker_credentials"')) {
|
||||
return {
|
||||
rows: state.credential
|
||||
? [{ version: state.credential.version, workerId: state.credential.workerId }]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
if (text.includes('INSERT INTO "ql3"."security_audit_events"')) {
|
||||
state.audit = {
|
||||
eventId: params[0], requestId: params[1], operationId: params[2],
|
||||
projectId: params[3], subjectType: params[4], subjectId: params[5],
|
||||
authenticationId: params[6], outcome: params[7],
|
||||
reasons: JSON.parse(params[8]), projectVersion: params[9],
|
||||
bindingVersion: params[10], occurredAtMs: params[11],
|
||||
};
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text.includes('INSERT INTO "ql3"."worker_credentials"')) {
|
||||
state.credential = {
|
||||
credentialId: params[0], version: params[1], state: params[2],
|
||||
workerId: params[3], secretDigest: params[4], createdAtMs: params[5],
|
||||
notBeforeAtMs: params[6], expiresAtMs: params[7],
|
||||
};
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text.includes('INSERT INTO "ql3"."worker_credential_mutations"')) {
|
||||
state.mutation = {
|
||||
mutationId: params[0], operation: params[1], credentialId: params[2],
|
||||
credentialVersion: params[3], expectedPreviousVersion: params[4],
|
||||
changedByType: params[5], changedById: params[6], createdAtMs: params[8],
|
||||
};
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text.includes('INSERT INTO "ql3"."worker_credential_deliveries"')) {
|
||||
state.deliveries.push({
|
||||
deliveryId: params[0], version: params[1], state: params[2],
|
||||
workerId: params[3], credentialId: params[4], credentialVersion: params[5],
|
||||
previousCredentialId: params[6], secretDigest: params[7],
|
||||
tokenDigest: params[8], deploymentTargetDigest: params[9],
|
||||
deploymentGeneration: params[10], stagedAtMs: params[11],
|
||||
credentialCommittedAtMs: params[12], publishedAtMs: params[13],
|
||||
publicationDigest: params[14], observedAtMs: params[15],
|
||||
observedSessionId: params[16], observedSessionVersion: params[17],
|
||||
previousRevokedAtMs: params[18],
|
||||
});
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text.includes('INSERT INTO "ql3"."worker_credential_stage_discards"')) {
|
||||
state.discards.push({
|
||||
deliveryId: params[0], version: params[1], state: params[2],
|
||||
workerId: params[3], credentialId: params[4], credentialVersion: params[5],
|
||||
previousCredentialId: params[6], secretDigest: params[7],
|
||||
tokenDigest: params[8], deploymentTargetDigest: params[9],
|
||||
deploymentGeneration: params[10], stagedAtMs: params[11],
|
||||
authorizedAtMs: params[12], discardedAtMs: params[13],
|
||||
});
|
||||
return { rows: [] };
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${text}`);
|
||||
}
|
||||
|
||||
return {
|
||||
state,
|
||||
port: {
|
||||
query,
|
||||
async connect() {
|
||||
return {
|
||||
query,
|
||||
release() { state.releases += 1; },
|
||||
};
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('commits credential, mutation and delivery v1 atomically then appends publication v2', async () => {
|
||||
const database = fakePool();
|
||||
const repository = new PostgresWorkerCredentialAdministrationRepository(database.port);
|
||||
const committed = await repository.commitDelivered({
|
||||
credential: credentialCommand(),
|
||||
delivery: delivery(),
|
||||
});
|
||||
assert.equal(committed.status, 'created');
|
||||
assert.equal(database.state.deliveries.length, 1);
|
||||
const mutationInsert = database.state.calls.findIndex((sql) =>
|
||||
sql.includes('INSERT INTO "ql3"."worker_credential_mutations"'));
|
||||
const deliveryInsert = database.state.calls.findIndex((sql) =>
|
||||
sql.includes('INSERT INTO "ql3"."worker_credential_deliveries"'));
|
||||
const commit = database.state.calls.indexOf('COMMIT');
|
||||
assert.ok(mutationInsert < deliveryInsert && deliveryInsert < commit);
|
||||
|
||||
const published = await repository.markPublished({
|
||||
deliveryId: MUTATION_ID,
|
||||
expectedVersion: 1,
|
||||
publicationDigest: 'd'.repeat(64),
|
||||
publishedAtMs: 1_100,
|
||||
});
|
||||
assert.equal(published.state, 'published');
|
||||
assert.equal(database.state.deliveries.length, 2);
|
||||
const resolved = await repository.resolveDelivered(MUTATION_ID);
|
||||
assert.equal(resolved.delivery.version, 2);
|
||||
assert.equal(resolved.delivery.publicationDigest, 'd'.repeat(64));
|
||||
|
||||
const replay = await repository.commitDelivered({
|
||||
credential: credentialCommand(),
|
||||
delivery: delivery(),
|
||||
});
|
||||
assert.equal(replay.status, 'existing');
|
||||
assert.equal(database.state.deliveries.length, 2);
|
||||
assert.ok(database.state.releases >= 3);
|
||||
});
|
||||
|
||||
test('converges a lost commit response and rejects delivery semantic drift', async () => {
|
||||
const database = fakePool();
|
||||
database.state.loseFirstCommitResponse = true;
|
||||
const repository = new PostgresWorkerCredentialAdministrationRepository(database.port);
|
||||
const replay = await repository.commitDelivered({
|
||||
credential: credentialCommand(),
|
||||
delivery: delivery(),
|
||||
});
|
||||
assert.equal(replay.status, 'existing');
|
||||
assert.equal(database.state.deliveries.length, 1);
|
||||
await assert.rejects(
|
||||
repository.commitDelivered({
|
||||
credential: credentialCommand(),
|
||||
delivery: delivery({ deploymentGeneration: 'other-generation' }),
|
||||
}),
|
||||
WorkerCredentialDeliveryConflictError,
|
||||
);
|
||||
await repository.markPublished({
|
||||
deliveryId: MUTATION_ID,
|
||||
expectedVersion: 1,
|
||||
publicationDigest: 'd'.repeat(64),
|
||||
publishedAtMs: 1_100,
|
||||
});
|
||||
const publicationReplay = await repository.markPublished({
|
||||
deliveryId: MUTATION_ID,
|
||||
expectedVersion: 1,
|
||||
publicationDigest: 'd'.repeat(64),
|
||||
publishedAtMs: 1_200,
|
||||
});
|
||||
assert.equal(publicationReplay.version, 2);
|
||||
assert.equal(database.state.deliveries.length, 2);
|
||||
});
|
||||
|
||||
test('rejects a gapped or rewritten append-only delivery history', async () => {
|
||||
const database = fakePool();
|
||||
const repository = new PostgresWorkerCredentialAdministrationRepository(database.port);
|
||||
database.state.deliveries.push(delivery({
|
||||
version: 2,
|
||||
state: 'published',
|
||||
publishedAtMs: 1_100,
|
||||
publicationDigest: 'd'.repeat(64),
|
||||
}));
|
||||
await assert.rejects(
|
||||
repository.resolveDelivery(MUTATION_ID),
|
||||
WorkerCredentialDeliveryConflictError,
|
||||
);
|
||||
database.state.deliveries.splice(
|
||||
0,
|
||||
1,
|
||||
delivery(),
|
||||
delivery({
|
||||
version: 2,
|
||||
state: 'published',
|
||||
deploymentGeneration: 'rewritten-generation',
|
||||
publishedAtMs: 1_100,
|
||||
publicationDigest: 'd'.repeat(64),
|
||||
}),
|
||||
);
|
||||
await assert.rejects(
|
||||
repository.resolveDelivery(MUTATION_ID),
|
||||
WorkerCredentialDeliveryConflictError,
|
||||
);
|
||||
});
|
||||
|
||||
test('authorizes one exact orphan discard and permanently fences delivery commit', async () => {
|
||||
const database = fakePool();
|
||||
const repository = new PostgresWorkerCredentialAdministrationRepository(database.port);
|
||||
const authorized = await repository.authorizeStageDiscard(stageIntent());
|
||||
assert.equal(authorized.state, 'discard_authorized');
|
||||
assert.equal(authorized.authorizedAtMs, 1_100);
|
||||
assert.equal(
|
||||
(await repository.authorizeStageDiscard(stageIntent())).version,
|
||||
1,
|
||||
);
|
||||
assert.equal(database.state.discards.length, 1);
|
||||
const page = await repository.listStageDiscardRecoveryPage({ limit: 1 });
|
||||
assert.equal(page.discards[0].deliveryId, MUTATION_ID);
|
||||
assert.equal(page.truncated, false);
|
||||
const discarded = await repository.markStageDiscarded({
|
||||
deliveryId: MUTATION_ID,
|
||||
expectedVersion: 1,
|
||||
});
|
||||
assert.equal(discarded.state, 'discarded');
|
||||
assert.equal(discarded.discardedAtMs, 1_200);
|
||||
assert.equal(
|
||||
(await repository.markStageDiscarded({
|
||||
deliveryId: MUTATION_ID,
|
||||
expectedVersion: 1,
|
||||
})).version,
|
||||
2,
|
||||
);
|
||||
await assert.rejects(
|
||||
repository.commitDelivered({
|
||||
credential: credentialCommand(),
|
||||
delivery: delivery(),
|
||||
}),
|
||||
WorkerCredentialDeliveryConflictError,
|
||||
);
|
||||
});
|
||||
|
||||
test('refuses orphan authorization after credential delivery wins', async () => {
|
||||
const database = fakePool();
|
||||
const repository = new PostgresWorkerCredentialAdministrationRepository(database.port);
|
||||
await repository.commitDelivered({
|
||||
credential: credentialCommand(),
|
||||
delivery: delivery(),
|
||||
});
|
||||
await assert.rejects(
|
||||
repository.authorizeStageDiscard(stageIntent()),
|
||||
WorkerCredentialDeliveryConflictError,
|
||||
);
|
||||
assert.equal(database.state.discards.length, 0);
|
||||
});
|
||||
+204
@@ -0,0 +1,204 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PostgresWorkerCredentialAdministrationRepository,
|
||||
} = require('../dist/entrypoints/admin');
|
||||
|
||||
const DELIVERY_ID = '123e4567-e89b-42d3-a456-426614174801';
|
||||
const REVOKE_ID = '123e4567-e89b-42d3-a456-426614174802';
|
||||
const SESSION_ID = '019f7094-a853-72f3-82ab-dfa08e6bd1c1';
|
||||
|
||||
function delivery(version, overrides = {}) {
|
||||
return {
|
||||
deliveryId: DELIVERY_ID,
|
||||
version,
|
||||
state: version === 1
|
||||
? 'credential_committed'
|
||||
: version === 2
|
||||
? 'published'
|
||||
: version === 3
|
||||
? 'observed'
|
||||
: 'previous_revoked',
|
||||
workerId: 'edge-router-1',
|
||||
credentialId: 'worker_generation_2',
|
||||
credentialVersion: 1,
|
||||
previousCredentialId: 'worker_generation_1',
|
||||
secretDigest: 'a'.repeat(64),
|
||||
tokenDigest: 'b'.repeat(64),
|
||||
deploymentTargetDigest: 'c'.repeat(64),
|
||||
deploymentGeneration: 'secret-generation-2',
|
||||
stagedAtMs: 1_000,
|
||||
credentialCommittedAtMs: 1_000,
|
||||
publishedAtMs: version >= 2 ? 1_100 : null,
|
||||
publicationDigest: version >= 2 ? 'd'.repeat(64) : null,
|
||||
observedAtMs: version >= 3 ? 1_200 : null,
|
||||
observedSessionId: version >= 3 ? SESSION_ID : null,
|
||||
observedSessionVersion: version >= 3 ? 4 : null,
|
||||
previousRevokedAtMs: version >= 4 ? 1_300 : null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function revokeCommand() {
|
||||
return {
|
||||
expectedCurrentVersion: 1,
|
||||
credential: {
|
||||
credentialId: 'worker_generation_1',
|
||||
version: 2,
|
||||
state: 'revoked',
|
||||
workerId: 'edge-router-1',
|
||||
secretDigest: '0'.repeat(64),
|
||||
createdAtMs: 1_300,
|
||||
notBeforeAtMs: 1_300,
|
||||
expiresAtMs: 2_300,
|
||||
},
|
||||
mutation: {
|
||||
mutationId: REVOKE_ID,
|
||||
operation: 'revoke',
|
||||
credentialId: 'worker_generation_1',
|
||||
credentialVersion: 2,
|
||||
expectedPreviousVersion: 1,
|
||||
changedBy: { type: 'system', id: 'credential-recovery' },
|
||||
createdAtMs: 1_300,
|
||||
},
|
||||
audit: {
|
||||
eventId: REVOKE_ID,
|
||||
requestId: `worker-delivery-revoke:${DELIVERY_ID}`,
|
||||
operationId: 'worker_credential.revoke',
|
||||
projectId: null,
|
||||
subject: { type: 'system', id: 'credential-recovery' },
|
||||
authenticationId: 'service:credential-recovery',
|
||||
outcome: 'allowed',
|
||||
reasons: ['worker_credential_admin'],
|
||||
fence: null,
|
||||
occurredAtMs: 1_300,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function database() {
|
||||
const deliveries = [delivery(1), delivery(2), delivery(3)];
|
||||
const calls = [];
|
||||
let revoke = null;
|
||||
let audit = null;
|
||||
const query = async (text, values = []) => {
|
||||
const sql = String(text);
|
||||
calls.push(sql);
|
||||
if (
|
||||
sql.startsWith('BEGIN') || sql === 'COMMIT' || sql === 'ROLLBACK' ||
|
||||
sql.includes('set_config') || sql.includes('pg_advisory_xact_lock')
|
||||
) return { rows: [] };
|
||||
if (sql.startsWith('WITH observation AS')) {
|
||||
const latest = deliveries.at(-1);
|
||||
if (latest.state === 'previous_revoked') {
|
||||
return { rows: [{ observedAtMs: '1400', deliveryId: null }] };
|
||||
}
|
||||
return { rows: [{
|
||||
...latest,
|
||||
observedAtMs: '1400',
|
||||
observedAtMsDelivery: latest.observedAtMs,
|
||||
}] };
|
||||
}
|
||||
if (sql.includes('FROM "ql3"."worker_credential_mutations" AS mutation')) {
|
||||
if (!revoke || values[0] !== REVOKE_ID) return { rows: [] };
|
||||
return { rows: [{
|
||||
mutationId: revoke.mutation.mutationId,
|
||||
operation: revoke.mutation.operation,
|
||||
credentialId: revoke.mutation.credentialId,
|
||||
credentialVersion: revoke.mutation.credentialVersion,
|
||||
expectedPreviousVersion: revoke.mutation.expectedPreviousVersion,
|
||||
changedByType: revoke.mutation.changedBy.type,
|
||||
changedById: revoke.mutation.changedBy.id,
|
||||
createdAtMs: revoke.mutation.createdAtMs,
|
||||
state: revoke.credential.state,
|
||||
workerId: revoke.credential.workerId,
|
||||
secretDigest: revoke.credential.secretDigest,
|
||||
notBeforeAtMs: revoke.credential.notBeforeAtMs,
|
||||
expiresAtMs: revoke.credential.expiresAtMs,
|
||||
auditEventId: audit.eventId,
|
||||
auditRequestId: audit.requestId,
|
||||
auditOperationId: audit.operationId,
|
||||
auditProjectId: null,
|
||||
auditSubjectType: audit.subjectType,
|
||||
auditSubjectId: audit.subjectId,
|
||||
auditAuthenticationId: audit.authenticationId,
|
||||
auditOutcome: audit.outcome,
|
||||
auditReasons: audit.reasons,
|
||||
auditProjectVersion: null,
|
||||
auditBindingVersion: null,
|
||||
auditOccurredAtMs: audit.occurredAtMs,
|
||||
}] };
|
||||
}
|
||||
if (sql.includes('FROM "ql3"."worker_credential_deliveries"')) {
|
||||
return { rows: deliveries.map((value) => ({ ...value })) };
|
||||
}
|
||||
if (sql.includes('FROM "ql3"."worker_credentials"')) {
|
||||
return { rows: [{ version: 1, state: 'active', workerId: 'edge-router-1' }] };
|
||||
}
|
||||
if (sql.includes('INSERT INTO "ql3"."security_audit_events"')) {
|
||||
audit = {
|
||||
eventId: values[0], requestId: values[1], operationId: values[2],
|
||||
subjectType: values[4], subjectId: values[5],
|
||||
authenticationId: values[6], outcome: values[7],
|
||||
reasons: JSON.parse(values[8]), occurredAtMs: values[11],
|
||||
};
|
||||
return { rows: [] };
|
||||
}
|
||||
if (sql.includes('INSERT INTO "ql3"."worker_credentials"')) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (sql.includes('INSERT INTO "ql3"."worker_credential_mutations"')) {
|
||||
revoke = { credential: revokeCommand().credential, mutation: revokeCommand().mutation };
|
||||
return { rows: [] };
|
||||
}
|
||||
if (sql.includes('INSERT INTO "ql3"."worker_credential_deliveries"')) {
|
||||
deliveries.push(delivery(4));
|
||||
return { rows: [] };
|
||||
}
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
};
|
||||
return {
|
||||
calls,
|
||||
deliveries,
|
||||
port: {
|
||||
query,
|
||||
async connect() { return { query, release() {} }; },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('pages observed delivery and atomically appends revoke plus v4', async () => {
|
||||
const db = database();
|
||||
const repository = new PostgresWorkerCredentialAdministrationRepository(db.port);
|
||||
const page = await repository.listRecoveryPage({ limit: 1 });
|
||||
assert.equal(page.deliveries[0].state, 'observed');
|
||||
assert.equal(page.observedAtMs, 1_400);
|
||||
const result = await repository.revokePreviousDelivered({
|
||||
credential: revokeCommand(),
|
||||
delivery: delivery(4),
|
||||
});
|
||||
assert.equal(result.status, 'created');
|
||||
assert.equal(db.deliveries.at(-1).state, 'previous_revoked');
|
||||
const auditInsert = db.calls.findIndex((sql) =>
|
||||
sql.includes('INSERT INTO "ql3"."security_audit_events"'));
|
||||
const credentialInsert = db.calls.findIndex((sql) =>
|
||||
sql.includes('INSERT INTO "ql3"."worker_credentials"'));
|
||||
const mutationInsert = db.calls.findIndex((sql) =>
|
||||
sql.includes('INSERT INTO "ql3"."worker_credential_mutations"'));
|
||||
const deliveryInsert = db.calls.findIndex((sql) =>
|
||||
sql.includes('INSERT INTO "ql3"."worker_credential_deliveries"'));
|
||||
const commit = db.calls.lastIndexOf('COMMIT');
|
||||
assert.ok(
|
||||
auditInsert < credentialInsert &&
|
||||
credentialInsert < mutationInsert &&
|
||||
mutationInsert < deliveryInsert &&
|
||||
deliveryInsert < commit,
|
||||
);
|
||||
const replay = await repository.revokePreviousDelivered({
|
||||
credential: revokeCommand(),
|
||||
delivery: delivery(4),
|
||||
});
|
||||
assert.equal(replay.status, 'existing');
|
||||
assert.equal(db.deliveries.length, 4);
|
||||
assert.deepEqual((await repository.listRecoveryPage()).deliveries, []);
|
||||
});
|
||||
@@ -0,0 +1,140 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
WorkerCredentialManagementPlanConflictError,
|
||||
WorkerCredentialManagementPlanUnavailableError,
|
||||
createWorkerCredentialManagementPlan,
|
||||
} = require('@qinglong/runtime-core/worker-credential-management-plan');
|
||||
const {
|
||||
PostgresWorkerCredentialManagementPlanRepository,
|
||||
} = require('@qinglong/cluster-postgres/worker-credential-management-plan');
|
||||
|
||||
function plan(actionRef = 'worker-credential:worker-a:generation-2') {
|
||||
return createWorkerCredentialManagementPlan({
|
||||
actionRef,
|
||||
authorityProjectId: 'cluster-instance-authority',
|
||||
action: 'rotate',
|
||||
target: {
|
||||
deliveryId: '123e4567-e89b-42d3-a456-426614174702',
|
||||
workerId: 'worker-a',
|
||||
credentialId: 'credential-b',
|
||||
previousCredentialId: 'credential-a',
|
||||
credentialNotBeforeAtMs: 11_000,
|
||||
credentialExpiresAtMs: 21_000,
|
||||
deploymentTargetDigest: '1'.repeat(64),
|
||||
deploymentGeneration: 'generation-2',
|
||||
},
|
||||
requestedBy: { type: 'user', id: 'operator-a' },
|
||||
plannedAtMs: 10_000,
|
||||
expiresAtMs: 20_000,
|
||||
});
|
||||
}
|
||||
|
||||
function fixture() {
|
||||
const stored = new Map();
|
||||
const queries = [];
|
||||
const pool = {
|
||||
async query(text, values) {
|
||||
queries.push({ text, values });
|
||||
if (text.includes('INSERT INTO')) {
|
||||
const actionRef = values[0];
|
||||
if (stored.has(actionRef)) return { rows: [], rowCount: 0 };
|
||||
stored.set(actionRef, JSON.parse(values[17]));
|
||||
return { rows: [{ actionRef }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('SELECT plan_json')) {
|
||||
const value = stored.get(values[0]);
|
||||
return { rows: value ? [{ planJson: value }] : [], rowCount: value ? 1 : 0 };
|
||||
}
|
||||
throw new Error('unexpected query');
|
||||
},
|
||||
};
|
||||
return {
|
||||
repository: new PostgresWorkerCredentialManagementPlanRepository(pool),
|
||||
stored,
|
||||
queries,
|
||||
};
|
||||
}
|
||||
|
||||
test('creates and exactly replays one immutable Worker credential plan', async () => {
|
||||
const state = fixture();
|
||||
const expected = plan();
|
||||
const created = await state.repository.create(expected);
|
||||
const replay = await state.repository.create(expected);
|
||||
assert.equal(created.status, 'created');
|
||||
assert.equal(replay.status, 'existing');
|
||||
assert.deepEqual(created.plan, expected);
|
||||
assert.deepEqual(replay.plan, expected);
|
||||
assert.equal(state.stored.size, 1);
|
||||
const insert = state.queries.find(({ text }) => text.includes('INSERT INTO'));
|
||||
assert.equal(insert.values.includes('credential-a'), true);
|
||||
assert.equal(insert.values.some((value) => /ql3w|token/i.test(String(value))), false);
|
||||
});
|
||||
|
||||
test('replays the stored plan when only server-authored time fields differ', async () => {
|
||||
const state = fixture();
|
||||
const first = plan();
|
||||
const replayedRequest = createWorkerCredentialManagementPlan({
|
||||
actionRef: first.actionRef,
|
||||
authorityProjectId: first.authorityProjectId,
|
||||
action: first.action,
|
||||
target: first.target,
|
||||
requestedBy: first.requestedBy,
|
||||
plannedAtMs: first.plannedAtMs + 500,
|
||||
expiresAtMs: first.expiresAtMs + 500,
|
||||
});
|
||||
assert.notEqual(replayedRequest.planDigest, first.planDigest);
|
||||
await state.repository.create(first);
|
||||
const replay = await state.repository.create(replayedRequest);
|
||||
assert.equal(replay.status, 'existing');
|
||||
assert.deepEqual(replay.plan, first);
|
||||
assert.equal(state.stored.size, 1);
|
||||
});
|
||||
|
||||
test('rejects an actionRef replay bound to different plan content', async () => {
|
||||
const state = fixture();
|
||||
const first = plan();
|
||||
await state.repository.create(first);
|
||||
const changed = createWorkerCredentialManagementPlan({
|
||||
actionRef: first.actionRef,
|
||||
authorityProjectId: first.authorityProjectId,
|
||||
action: first.action,
|
||||
target: {
|
||||
...first.target,
|
||||
deploymentGeneration: 'generation-3',
|
||||
},
|
||||
requestedBy: first.requestedBy,
|
||||
plannedAtMs: first.plannedAtMs,
|
||||
expiresAtMs: first.expiresAtMs,
|
||||
});
|
||||
await assert.rejects(
|
||||
state.repository.create(changed),
|
||||
WorkerCredentialManagementPlanConflictError,
|
||||
);
|
||||
});
|
||||
|
||||
test('maps malformed durable JSON and storage errors to unavailable', async () => {
|
||||
const malformed = {
|
||||
async query() {
|
||||
return { rows: [{ planJson: { schema: 'wrong' } }], rowCount: 1 };
|
||||
},
|
||||
};
|
||||
await assert.rejects(
|
||||
new PostgresWorkerCredentialManagementPlanRepository(malformed)
|
||||
.findByActionRef('worker-credential:worker-a:generation-2'),
|
||||
WorkerCredentialManagementPlanUnavailableError,
|
||||
);
|
||||
const failed = {
|
||||
async query() { throw new Error('sensitive database failure'); },
|
||||
};
|
||||
await assert.rejects(
|
||||
new PostgresWorkerCredentialManagementPlanRepository(failed)
|
||||
.findByActionRef('worker-credential:worker-a:generation-2'),
|
||||
(error) => {
|
||||
assert.ok(error instanceof WorkerCredentialManagementPlanUnavailableError);
|
||||
assert.doesNotMatch(error.message, /sensitive|database failure/i);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,85 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
PostgresWorkerCredentialManagementQuotaRepository,
|
||||
} = require('@qinglong/cluster-postgres/worker-credential-manager');
|
||||
|
||||
const command = Object.freeze({
|
||||
projectId: 'cluster-authority',
|
||||
subject: Object.freeze({ type: 'user', id: 'operator-a' }),
|
||||
operation: 'worker-credential.plan',
|
||||
idempotencyKey: 'worker-credential:worker-a:generation-2',
|
||||
});
|
||||
|
||||
test('uses one database-clock UPSERT and exact replay receipt', async () => {
|
||||
const calls = [];
|
||||
const repository = new PostgresWorkerCredentialManagementQuotaRepository(
|
||||
{
|
||||
async query(text, values) {
|
||||
calls.push({ text, values });
|
||||
return {
|
||||
rows: [{
|
||||
admitted: true,
|
||||
consumedCount: 1,
|
||||
resetAtMs: 60_000,
|
||||
observedAtMs: 1_000,
|
||||
}],
|
||||
};
|
||||
},
|
||||
},
|
||||
{ windowMs: 60_000, limits: { 'worker-credential.plan': 1 } },
|
||||
);
|
||||
assert.deepEqual(await repository.consume(command), {
|
||||
admitted: true,
|
||||
retryAfterMs: null,
|
||||
});
|
||||
assert.equal(calls.length, 1);
|
||||
assert.match(calls[0].text, /clock_timestamp\(\)/);
|
||||
assert.match(calls[0].text, /receipt_ids \? \$5::text/);
|
||||
assert.deepEqual(calls[0].values.slice(0, 5), [
|
||||
command.projectId,
|
||||
'user',
|
||||
'operator-a',
|
||||
command.operation,
|
||||
command.idempotencyKey,
|
||||
]);
|
||||
});
|
||||
|
||||
test('returns one bounded durable rejection without application-clock decisions', async () => {
|
||||
let calls = 0;
|
||||
const repository = new PostgresWorkerCredentialManagementQuotaRepository({
|
||||
async query(text) {
|
||||
calls += 1;
|
||||
if (text.includes('INSERT INTO')) return { rows: [] };
|
||||
return {
|
||||
rows: [{
|
||||
admitted: false,
|
||||
consumedCount: 30,
|
||||
resetAtMs: 60_000,
|
||||
observedAtMs: 59_000,
|
||||
}],
|
||||
};
|
||||
},
|
||||
});
|
||||
assert.deepEqual(await repository.consume(command), {
|
||||
admitted: false,
|
||||
retryAfterMs: 1_000,
|
||||
});
|
||||
assert.equal(calls, 2);
|
||||
});
|
||||
|
||||
test('rejects widened operations and configuration before PostgreSQL', async () => {
|
||||
let queries = 0;
|
||||
const pool = { async query() { queries += 1; return { rows: [] }; } };
|
||||
const repository = new PostgresWorkerCredentialManagementQuotaRepository(pool);
|
||||
await assert.rejects(
|
||||
repository.consume({ ...command, operation: 'worker-credential.execute' }),
|
||||
TypeError,
|
||||
);
|
||||
assert.equal(queries, 0);
|
||||
assert.throws(
|
||||
() => new PostgresWorkerCredentialManagementQuotaRepository(pool, { windowMs: 999 }),
|
||||
TypeError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
WorkerCredentialUnavailableError,
|
||||
} = require('@qinglong/runtime-core/worker-credential');
|
||||
const {
|
||||
PostgresWorkerCredentialRepository,
|
||||
} = require('../dist/entrypoints/workerIngress');
|
||||
|
||||
function row(overrides = {}) {
|
||||
return {
|
||||
credentialId: 'worker_primary',
|
||||
version: '2',
|
||||
state: 'active',
|
||||
workerId: 'edge-1',
|
||||
secretDigest: 'a'.repeat(64),
|
||||
createdAtMs: '100',
|
||||
notBeforeAtMs: '100',
|
||||
expiresAtMs: '1000',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('resolves only the latest normalized Worker credential', async () => {
|
||||
const calls = [];
|
||||
const repository = new PostgresWorkerCredentialRepository({
|
||||
async query(text, values) {
|
||||
calls.push({ text, values });
|
||||
return { rows: [row(), row({ version: '1' })] };
|
||||
},
|
||||
});
|
||||
assert.deepEqual(await repository.resolve('worker_primary'), {
|
||||
credentialId: 'worker_primary',
|
||||
version: 2,
|
||||
state: 'active',
|
||||
workerId: 'edge-1',
|
||||
secretDigest: 'a'.repeat(64),
|
||||
createdAtMs: 100,
|
||||
notBeforeAtMs: 100,
|
||||
expiresAtMs: 1000,
|
||||
});
|
||||
assert.deepEqual(calls[0].values, ['worker_primary']);
|
||||
assert.match(calls[0].text, /ORDER BY version DESC/);
|
||||
assert.match(calls[0].text, /LIMIT 2/);
|
||||
});
|
||||
|
||||
test('returns null for absence and fails closed on corrupt or unordered rows', async () => {
|
||||
const empty = new PostgresWorkerCredentialRepository({
|
||||
async query() { return { rows: [] }; },
|
||||
});
|
||||
assert.equal(await empty.resolve('worker_primary'), null);
|
||||
for (const rows of [
|
||||
[row({ secretDigest: 'corrupt' })],
|
||||
[row({ version: '1' }), row({ version: '2' })],
|
||||
]) {
|
||||
const repository = new PostgresWorkerCredentialRepository({
|
||||
async query() { return { rows }; },
|
||||
});
|
||||
await assert.rejects(
|
||||
repository.resolve('worker_primary'),
|
||||
WorkerCredentialUnavailableError,
|
||||
);
|
||||
}
|
||||
await assert.rejects(empty.resolve('../escape'), TypeError);
|
||||
});
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PostgresWorkerSessionRepository,
|
||||
} = require('@qinglong/cluster-postgres/worker-ingress');
|
||||
const {
|
||||
WorkerCredentialDeliveryUnavailableError,
|
||||
} = require('@qinglong/runtime-core/worker-credential-delivery');
|
||||
|
||||
const SESSION_ID = '019f7094-a853-72f3-82ab-dfa08e6bd1c1';
|
||||
const DELIVERY_ID = '123e4567-e89b-42d3-a456-426614174601';
|
||||
const CAPABILITIES_HASH =
|
||||
'44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a';
|
||||
|
||||
function session(version, overrides = {}) {
|
||||
return {
|
||||
workerId: 'edge-router-1',
|
||||
sessionId: SESSION_ID,
|
||||
generation: 2,
|
||||
status: 'online',
|
||||
version,
|
||||
capabilitiesJson: '{}',
|
||||
capabilitiesHash: CAPABILITIES_HASH,
|
||||
maxConcurrentRuns: 2,
|
||||
availableSlots: 1,
|
||||
registeredAtMs: '900',
|
||||
lastHeartbeatAtMs: '1000',
|
||||
leaseExpiresAtMs: '5000',
|
||||
updatedAtMs: '1000',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function delivery(version, overrides = {}) {
|
||||
const published = version >= 2;
|
||||
const observed = version >= 3;
|
||||
return {
|
||||
deliveryId: DELIVERY_ID,
|
||||
version,
|
||||
state: version === 1
|
||||
? 'credential_committed'
|
||||
: version === 2
|
||||
? 'published'
|
||||
: 'observed',
|
||||
workerId: 'edge-router-1',
|
||||
credentialId: 'worker_generation_2',
|
||||
credentialVersion: 1,
|
||||
previousCredentialId: 'worker_generation_1',
|
||||
secretDigest: 'a'.repeat(64),
|
||||
tokenDigest: 'b'.repeat(64),
|
||||
deploymentTargetDigest: 'c'.repeat(64),
|
||||
deploymentGeneration: 'secret-generation-2',
|
||||
stagedAtMs: '800',
|
||||
credentialCommittedAtMs: '850',
|
||||
publishedAtMs: published ? '900' : null,
|
||||
publicationDigest: published ? 'd'.repeat(64) : null,
|
||||
observedAtMs: observed ? '1100' : null,
|
||||
observedSessionId: observed ? SESSION_ID : null,
|
||||
observedSessionVersion: observed ? 4 : null,
|
||||
previousRevokedAtMs: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function database(deliveryRows) {
|
||||
const events = [];
|
||||
let released = 0;
|
||||
const client = {
|
||||
async query(text, values) {
|
||||
const sql = String(text);
|
||||
if (sql === 'BEGIN' || sql === 'COMMIT' || sql === 'ROLLBACK') {
|
||||
events.push(sql);
|
||||
return { rows: [] };
|
||||
}
|
||||
if (sql.startsWith('SET LOCAL')) return { rows: [] };
|
||||
if (sql.includes('FROM "ql3"."worker_sessions"') && sql.includes('FOR UPDATE')) {
|
||||
events.push('lock-session');
|
||||
return { rows: [session(3)] };
|
||||
}
|
||||
if (sql.includes('statement_timestamp()')) {
|
||||
events.push('database-time');
|
||||
return { rows: [{ observedAtMs: '1200' }] };
|
||||
}
|
||||
if (sql.includes('UPDATE "ql3"."worker_sessions"')) {
|
||||
events.push('update-session');
|
||||
return { rows: [session(4, {
|
||||
lastHeartbeatAtMs: '1200',
|
||||
leaseExpiresAtMs: '31200',
|
||||
updatedAtMs: '1200',
|
||||
})] };
|
||||
}
|
||||
if (sql.includes('FROM "ql3"."worker_credential_deliveries"')) {
|
||||
assert.equal(sql.includes('FOR UPDATE'), false);
|
||||
events.push(`read-delivery:${values.join(':')}`);
|
||||
return { rows: deliveryRows };
|
||||
}
|
||||
if (sql.includes('INSERT INTO "ql3"."worker_credential_deliveries"')) {
|
||||
events.push(`insert-observation:${values[1]}:${values[2]}`);
|
||||
assert.equal(values[15], 1200);
|
||||
assert.equal(values[16], SESSION_ID);
|
||||
assert.equal(values[17], 4);
|
||||
return { rows: [] };
|
||||
}
|
||||
throw new Error(`unexpected query: ${sql}`);
|
||||
},
|
||||
release() { released += 1; },
|
||||
};
|
||||
return {
|
||||
pool: {
|
||||
async connect() { return client; },
|
||||
async query() { throw new Error('not used'); },
|
||||
},
|
||||
events,
|
||||
released: () => released,
|
||||
};
|
||||
}
|
||||
|
||||
function heartbeat(repository) {
|
||||
return repository.heartbeatAuthenticated(
|
||||
{
|
||||
workerId: 'edge-router-1',
|
||||
sessionId: SESSION_ID,
|
||||
generation: 2,
|
||||
expectedVersion: 3,
|
||||
availableSlots: 1,
|
||||
leaseDurationMs: 30_000,
|
||||
},
|
||||
{
|
||||
workerId: 'edge-router-1',
|
||||
credentialId: 'worker_generation_2',
|
||||
credentialVersion: 1,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
test('appends authenticated observation in the Session heartbeat transaction', async () => {
|
||||
const db = database([delivery(1), delivery(2)]);
|
||||
const worker = await heartbeat(new PostgresWorkerSessionRepository(db.pool));
|
||||
assert.equal(worker.version, 4);
|
||||
assert.deepEqual(db.events, [
|
||||
'BEGIN',
|
||||
'lock-session',
|
||||
'database-time',
|
||||
'update-session',
|
||||
'read-delivery:edge-router-1:worker_generation_2:1',
|
||||
'insert-observation:3:observed',
|
||||
'COMMIT',
|
||||
]);
|
||||
assert.equal(db.released(), 1);
|
||||
});
|
||||
|
||||
test('rolls Session mutation back until the exact delivery is published', async () => {
|
||||
const db = database([delivery(1)]);
|
||||
await assert.rejects(
|
||||
heartbeat(new PostgresWorkerSessionRepository(db.pool)),
|
||||
WorkerCredentialDeliveryUnavailableError,
|
||||
);
|
||||
assert.equal(db.events.includes('COMMIT'), false);
|
||||
assert.equal(db.events.at(-1), 'ROLLBACK');
|
||||
assert.equal(db.released(), 1);
|
||||
});
|
||||
|
||||
test('replays an existing observation and ignores credentials outside delivery', async () => {
|
||||
for (const rows of [[delivery(1), delivery(2), delivery(3)], []]) {
|
||||
const db = database(rows);
|
||||
const worker = await heartbeat(new PostgresWorkerSessionRepository(db.pool));
|
||||
assert.equal(worker.version, 4);
|
||||
assert.equal(
|
||||
db.events.some((event) => event.startsWith('insert-observation:')),
|
||||
false,
|
||||
);
|
||||
assert.equal(db.events.at(-1), 'COMMIT');
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
PostgresWorkerSessionRepository,
|
||||
} = require('@qinglong/cluster-postgres/worker-ingress');
|
||||
|
||||
const SESSION_ID = '019f7094-a853-72f3-82ab-dfa08e6bd1c1';
|
||||
const CAPABILITIES_HASH =
|
||||
'44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a';
|
||||
|
||||
test('pins the shared transition status parameter to PostgreSQL varchar', async () => {
|
||||
const events = [];
|
||||
const session = {
|
||||
workerId: 'edge-router-1',
|
||||
sessionId: SESSION_ID,
|
||||
generation: 2,
|
||||
status: 'online',
|
||||
version: 3,
|
||||
capabilitiesJson: '{}',
|
||||
capabilitiesHash: CAPABILITIES_HASH,
|
||||
maxConcurrentRuns: 2,
|
||||
availableSlots: 1,
|
||||
registeredAtMs: '900',
|
||||
lastHeartbeatAtMs: '1000',
|
||||
leaseExpiresAtMs: '5000',
|
||||
updatedAtMs: '1000',
|
||||
};
|
||||
const client = {
|
||||
async query(text) {
|
||||
const sql = String(text);
|
||||
if (sql === 'BEGIN' || sql === 'COMMIT' || sql === 'ROLLBACK') {
|
||||
events.push(sql);
|
||||
return { rows: [] };
|
||||
}
|
||||
if (sql.startsWith('SET LOCAL')) return { rows: [] };
|
||||
if (
|
||||
sql.includes('FROM "ql3"."worker_sessions"') &&
|
||||
sql.includes('FOR UPDATE')
|
||||
) {
|
||||
return { rows: [session] };
|
||||
}
|
||||
if (sql.includes('statement_timestamp()')) {
|
||||
return { rows: [{ observedAtMs: '1200' }] };
|
||||
}
|
||||
if (sql.includes('UPDATE "ql3"."worker_sessions"')) {
|
||||
assert.match(sql, /status = \$5::varchar/);
|
||||
assert.match(sql, /WHEN \$5::varchar = 'offline'/);
|
||||
return {
|
||||
rows: [{
|
||||
...session,
|
||||
status: 'draining',
|
||||
version: 4,
|
||||
availableSlots: 0,
|
||||
updatedAtMs: '1200',
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (sql.includes('FROM "ql3"."worker_credential_deliveries"')) {
|
||||
return { rows: [] };
|
||||
}
|
||||
throw new Error(`unexpected query: ${sql}`);
|
||||
},
|
||||
release() {},
|
||||
};
|
||||
const repository = new PostgresWorkerSessionRepository({
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
async query() {
|
||||
throw new Error('not used');
|
||||
},
|
||||
});
|
||||
const transitioned = await repository.transitionAuthenticated(
|
||||
{
|
||||
workerId: 'edge-router-1',
|
||||
sessionId: SESSION_ID,
|
||||
generation: 2,
|
||||
expectedVersion: 3,
|
||||
status: 'draining',
|
||||
},
|
||||
{
|
||||
workerId: 'edge-router-1',
|
||||
credentialId: 'worker_generation_2',
|
||||
credentialVersion: 1,
|
||||
},
|
||||
);
|
||||
assert.equal(transitioned.status, 'draining');
|
||||
assert.equal(transitioned.version, 4);
|
||||
assert.deepEqual(events, ['BEGIN', 'COMMIT']);
|
||||
});
|
||||
@@ -0,0 +1,163 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
PostgresRunDispatchLeaseRepository,
|
||||
} = require('../dist/remote-execution/runDispatchLeaseRepository');
|
||||
|
||||
const SESSION_ID = '018f5c64-9b9d-7f1a-8c2d-1234567890ac';
|
||||
const LEASE_TOKEN = 'workflow_task_lease_capability_0000000000000001';
|
||||
const LEASE_DIGEST = createHash('sha256')
|
||||
.update(LEASE_TOKEN)
|
||||
.digest('hex');
|
||||
|
||||
test('leases a Workflow Task Attempt without changing the aggregate Run status', async () => {
|
||||
const calls = [];
|
||||
const client = {
|
||||
async query(sql, params = []) {
|
||||
const normalized = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ sql: normalized, params });
|
||||
if (
|
||||
normalized === 'BEGIN' ||
|
||||
normalized.startsWith('SET LOCAL') ||
|
||||
normalized === 'COMMIT' ||
|
||||
normalized === 'ROLLBACK' ||
|
||||
normalized.startsWith('SELECT pg_advisory_xact_lock')
|
||||
) {
|
||||
return { rows: [], rowCount: 0 };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."worker_sessions"')) {
|
||||
return {
|
||||
rows: [{
|
||||
workerId: 'edge-1',
|
||||
sessionId: SESSION_ID,
|
||||
generation: 2,
|
||||
status: 'online',
|
||||
maxConcurrentRuns: 4,
|
||||
availableSlots: 2,
|
||||
leaseExpiresAtMs: 20_000,
|
||||
}],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (
|
||||
normalized.includes('INNER JOIN "ql3"."run_attempts"') &&
|
||||
normalized.includes('workflow_task')
|
||||
) {
|
||||
return {
|
||||
rows: [{
|
||||
runId: 'workflow-run-1',
|
||||
runStatus: 'running',
|
||||
executionOwner: 'runtime',
|
||||
cancelRequestedAtMs: null,
|
||||
runVersion: 7,
|
||||
eventSequence: 7,
|
||||
attemptId: 'workflow-attempt-1',
|
||||
attemptStatus: 'claimed',
|
||||
attemptRunId: 'workflow-run-1',
|
||||
attemptStepRunId: 'workflow-step-1',
|
||||
workflowAttemptId: 'workflow-attempt-1',
|
||||
workflowStepRunId: 'workflow-step-1',
|
||||
admittedWorkflowStepVersion: 2,
|
||||
admittedWorkflowStepDigest: 'a'.repeat(64),
|
||||
}],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (
|
||||
normalized.includes('FROM "ql3"."step_runs"') &&
|
||||
normalized.includes('FOR UPDATE')
|
||||
) {
|
||||
return {
|
||||
rows: [{
|
||||
workflowStepStatus: 'ready',
|
||||
workflowStepVersion: 2,
|
||||
workflowStepDigest: 'a'.repeat(64),
|
||||
}],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (
|
||||
normalized.startsWith('SELECT') &&
|
||||
normalized.includes('FROM "ql3"."run_dispatch_leases"') &&
|
||||
normalized.includes('WHERE attempt_id = $1 FOR UPDATE')
|
||||
) {
|
||||
return { rows: [], rowCount: 0 };
|
||||
}
|
||||
if (normalized.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: 10_000 }], rowCount: 1 };
|
||||
}
|
||||
if (normalized.includes('count(*)::integer AS "activeCount"')) {
|
||||
return { rows: [{ activeCount: 0 }], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
normalized.startsWith(
|
||||
'INSERT INTO "ql3"."run_dispatch_leases"',
|
||||
)
|
||||
) {
|
||||
return {
|
||||
rows: [{
|
||||
attemptId: 'workflow-attempt-1',
|
||||
runId: 'workflow-run-1',
|
||||
status: 'leased',
|
||||
version: 0,
|
||||
leaseGeneration: 1,
|
||||
workerId: 'edge-1',
|
||||
workerSessionId: SESSION_ID,
|
||||
workerGeneration: 2,
|
||||
leaseTokenDigest: LEASE_DIGEST,
|
||||
acquiredAtMs: 10_000,
|
||||
renewedAtMs: 10_000,
|
||||
expiresAtMs: 40_000,
|
||||
releasedAtMs: null,
|
||||
releaseReason: null,
|
||||
completedAtMs: null,
|
||||
updatedAtMs: 10_000,
|
||||
}],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (
|
||||
normalized.startsWith('UPDATE ') ||
|
||||
normalized.startsWith('INSERT INTO ')
|
||||
) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`Unexpected SQL: ${normalized}`);
|
||||
},
|
||||
release() {
|
||||
calls.push({ sql: 'RELEASE', params: [] });
|
||||
},
|
||||
};
|
||||
const repository = new PostgresRunDispatchLeaseRepository({
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
});
|
||||
const result = await repository.claim({
|
||||
runId: 'workflow-run-1',
|
||||
attemptId: 'workflow-attempt-1',
|
||||
workerId: 'edge-1',
|
||||
workerSessionId: SESSION_ID,
|
||||
workerGeneration: 2,
|
||||
leaseToken: LEASE_TOKEN,
|
||||
leaseDurationMs: 30_000,
|
||||
eventId: '018f5c64-9b9d-7f1a-8c2d-1234567890a1',
|
||||
offerId: 'workflow-offer-1',
|
||||
});
|
||||
|
||||
assert.equal(result.status, 'claimed');
|
||||
const runUpdate = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"'));
|
||||
assert.equal(runUpdate.params[1], 'running');
|
||||
const event = calls.find(
|
||||
({ sql, params }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"') &&
|
||||
params[3] === 'workflow.task_dispatch_leased',
|
||||
);
|
||||
assert.equal(event.params[7], 'workflow-step-1');
|
||||
assert.match(event.params[8], /"execution_scope":"workflow_task"/);
|
||||
assert.equal(calls.at(-2).sql, 'COMMIT');
|
||||
assert.equal(calls.at(-1).sql, 'RELEASE');
|
||||
});
|
||||
Reference in New Issue
Block a user