mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-22 10:32:40 +08:00
feat(ql3): add cluster log retention authority
This commit is contained in:
whyour
@@ -103,6 +103,7 @@ test('defines the immutable PostgreSQL capability and Run core stream', async ()
|
||||
'pg-0052-automation-management-identity-keyset-ledger',
|
||||
'pg-0053-plugin-package-workflow-run-list-index',
|
||||
'pg-0054-approval-management-boundary',
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
],
|
||||
);
|
||||
for (const migration of postgresqlMainMigrationStream.migrations) {
|
||||
@@ -508,6 +509,11 @@ test('freezes every published PostgreSQL migration checksum', () => {
|
||||
checksum:
|
||||
'5e3e6b222269f095e0d7a985fdeb0ea154510e59dfe15873192af8c8d603fca3',
|
||||
},
|
||||
{
|
||||
id: 'pg-0055-run-attempt-log-retention',
|
||||
checksum:
|
||||
'c775c65ec03ae3a1606f899064d2d38fa63fd136ce52cbd1b1172c3a51e6bf30',
|
||||
},
|
||||
];
|
||||
assert.deepEqual(
|
||||
postgresqlMainMigrationStream.migrations.map(({ id, checksum }) => ({
|
||||
@@ -1875,3 +1881,26 @@ test('advances capability v53 with isolated human Approval management authority'
|
||||
/migration_id = 'pg-0053-plugin-package-workflow-run-list-index'/,
|
||||
);
|
||||
});
|
||||
|
||||
test('advances capability v54 with durable Cluster log retention authority', async () => {
|
||||
const migration = migrationById('pg-0055-run-attempt-log-retention');
|
||||
const statements = [];
|
||||
await migration.up({
|
||||
async query(statement) {
|
||||
statements.push(statement);
|
||||
return { rows: [] };
|
||||
},
|
||||
});
|
||||
const sql = statements.join('\n');
|
||||
assert.match(sql, /run_attempt_log_retention_controls/);
|
||||
assert.match(sql, /run_attempt_log_artifact_tombstones/);
|
||||
assert.match(sql, /FOR UPDATE|SKIP LOCKED|claim_expires_at_ms/);
|
||||
assert.match(sql, /TO ql3_runtime/);
|
||||
assert.match(sql, /contract_version = 54/);
|
||||
assert.match(sql, /"run_attempt_log_retention":1/);
|
||||
assert.match(sql, /contract_version = 53/);
|
||||
assert.match(
|
||||
sql,
|
||||
/migration_id = 'pg-0054-approval-management-boundary'/,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -58,6 +58,8 @@ function validPrivileges() {
|
||||
tool_invocation_input_artifacts: [true, true, false, false],
|
||||
tool_invocation_preview_artifacts: [true, true, false, false],
|
||||
run_attempts: [true, true, true, false],
|
||||
run_attempt_log_retention_controls: [true, true, true, true],
|
||||
run_attempt_log_artifact_tombstones: [true, true, false, false],
|
||||
worker_sessions: [true, true, true, false],
|
||||
run_dispatch_leases: [true, true, true, false],
|
||||
worker_credentials: [false, false, false, false],
|
||||
@@ -178,6 +180,8 @@ function validAdminPrivileges() {
|
||||
tool_invocation_input_artifacts: [false, false, false, false],
|
||||
tool_invocation_preview_artifacts: [false, false, false, false],
|
||||
run_attempts: [false, false, false, false],
|
||||
run_attempt_log_retention_controls: [false, false, false, false],
|
||||
run_attempt_log_artifact_tombstones: [false, false, false, false],
|
||||
worker_sessions: [false, false, false, false],
|
||||
run_dispatch_leases: [false, false, false, false],
|
||||
worker_credentials: [true, true, true, false],
|
||||
@@ -692,7 +696,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
|
||||
serverMajor: 16,
|
||||
currentUser: 'ql3_runtime',
|
||||
contractName: 'control-core',
|
||||
contractVersion: 53,
|
||||
contractVersion: 54,
|
||||
migrationIds: [
|
||||
'pg-0001-schema-capability',
|
||||
'pg-0002-run-core',
|
||||
@@ -748,6 +752,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
|
||||
'pg-0052-automation-management-identity-keyset-ledger',
|
||||
'pg-0053-plugin-package-workflow-run-list-index',
|
||||
'pg-0054-approval-management-boundary',
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
],
|
||||
});
|
||||
});
|
||||
@@ -778,10 +783,10 @@ test('accepts the exact schema and isolated least-privilege admin role', async (
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_admin');
|
||||
assert.equal(report.contractVersion, 53);
|
||||
assert.equal(report.contractVersion, 54);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0054-approval-management-boundary',
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -794,10 +799,10 @@ test('accepts the isolated least-privilege automation manager role', async () =>
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_automation_manager');
|
||||
assert.equal(report.contractVersion, 53);
|
||||
assert.equal(report.contractVersion, 54);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0054-approval-management-boundary',
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
);
|
||||
|
||||
const widened = automationManagerPrivileges();
|
||||
@@ -826,10 +831,10 @@ test('accepts the isolated least-privilege human Approval manager role', async (
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_approval_manager');
|
||||
assert.equal(report.contractVersion, 53);
|
||||
assert.equal(report.contractVersion, 54);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0054-approval-management-boundary',
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
);
|
||||
|
||||
const widened = approvalManagerPrivileges();
|
||||
@@ -936,10 +941,10 @@ test('accepts the exact schema and isolated Worker ingress role', async () => {
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_worker_ingress');
|
||||
assert.equal(report.contractVersion, 53);
|
||||
assert.equal(report.contractVersion, 54);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0054-approval-management-boundary',
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,330 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
createRunAttemptLogRetirementRecord,
|
||||
RunAttemptLogRetentionUnavailableError,
|
||||
} = require('@qinglong/runtime-core/run-attempt-log-retention');
|
||||
const {
|
||||
PostgresRunAttemptLogRetentionClaimRepository,
|
||||
} = require('../dist/entrypoints/runtime');
|
||||
|
||||
const TOKEN = '00000000-0000-4000-8000-000000000055';
|
||||
const ARTIFACT_ID = `wlog-${'a'.repeat(30)}`;
|
||||
|
||||
function claimRow(overrides = {}) {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
logArtifactId: ARTIFACT_ID,
|
||||
executorType: 'remote_worker',
|
||||
finishedAtMs: '1000',
|
||||
eligibleAtMs: '61000',
|
||||
observedAtMs: '70000',
|
||||
claimOwner: 'replica-a',
|
||||
claimToken: TOKEN,
|
||||
claimVersion: 1,
|
||||
claimExpiresAtMs: '100000',
|
||||
failureCount: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function claim() {
|
||||
return Object.freeze({
|
||||
candidate: Object.freeze({
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
logArtifactId: ARTIFACT_ID,
|
||||
executorType: 'remote_worker',
|
||||
finishedAtMs: 1000,
|
||||
}),
|
||||
eligibleAtMs: 61000,
|
||||
observedAtMs: 70000,
|
||||
ownerId: 'replica-a',
|
||||
token: TOKEN,
|
||||
version: 1,
|
||||
expiresAtMs: 100000,
|
||||
failureCount: 0,
|
||||
});
|
||||
}
|
||||
|
||||
test('claims one bounded remote log page under a short database lease', async () => {
|
||||
const calls = [];
|
||||
let released = false;
|
||||
const repository = new PostgresRunAttemptLogRetentionClaimRepository(
|
||||
{
|
||||
async connect() {
|
||||
return {
|
||||
async query(text, values = []) {
|
||||
calls.push({ text, values });
|
||||
if (text.includes('FOR UPDATE OF attempt SKIP LOCKED')) {
|
||||
return { rows: [claimRow()], rowCount: 1 };
|
||||
}
|
||||
return { rows: [], rowCount: 0 };
|
||||
},
|
||||
release() {
|
||||
released = true;
|
||||
},
|
||||
};
|
||||
},
|
||||
async query() {
|
||||
throw new Error('pool query not expected');
|
||||
},
|
||||
},
|
||||
() => TOKEN,
|
||||
);
|
||||
|
||||
const page = await repository.claim({
|
||||
ownerId: 'replica-a',
|
||||
retentionMs: 60000,
|
||||
limit: 4,
|
||||
leaseMs: 30000,
|
||||
});
|
||||
|
||||
assert.deepEqual(page, { claims: [claim()], hasMore: false });
|
||||
assert.deepEqual(
|
||||
calls.map(({ text }) => text.split('\n', 1)[0]),
|
||||
[
|
||||
'BEGIN ISOLATION LEVEL READ COMMITTED',
|
||||
"SET LOCAL statement_timeout = '5000ms'",
|
||||
"SET LOCAL lock_timeout = '1000ms'",
|
||||
'WITH observation AS (',
|
||||
'COMMIT',
|
||||
],
|
||||
);
|
||||
assert.deepEqual(calls[3].values, [
|
||||
60000,
|
||||
4,
|
||||
'replica-a',
|
||||
TOKEN,
|
||||
30000,
|
||||
]);
|
||||
assert.match(calls[3].text, /ON CONFLICT \(attempt_id\) DO UPDATE/);
|
||||
assert.match(calls[3].text, /claim_expires_at_ms <= EXCLUDED\.updated_at_ms/);
|
||||
assert.equal(released, true);
|
||||
});
|
||||
|
||||
test('fences retry settlement by owner token version and database expiry', async () => {
|
||||
const calls = [];
|
||||
const repository = new PostgresRunAttemptLogRetentionClaimRepository({
|
||||
async connect() {
|
||||
throw new Error('not expected');
|
||||
},
|
||||
async query(text, values) {
|
||||
calls.push({ text, values });
|
||||
return { rows: [{ attemptId: 'attempt-1' }], rowCount: 1 };
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(
|
||||
await repository.settle(claim(), {
|
||||
status: 'retry',
|
||||
delayMs: 2500,
|
||||
failureCode: 'artifact_unavailable',
|
||||
}),
|
||||
'settled',
|
||||
);
|
||||
assert.deepEqual(calls[0].values, [
|
||||
'attempt-1',
|
||||
'replica-a',
|
||||
TOKEN,
|
||||
1,
|
||||
100000,
|
||||
'retry',
|
||||
2500,
|
||||
'artifact_unavailable',
|
||||
]);
|
||||
assert.match(calls[0].text, /claim_expires_at_ms > observation\.observed_at_ms/);
|
||||
});
|
||||
|
||||
test('reads an exact durable tombstone for the profile-aware log route', async () => {
|
||||
const record = createRunAttemptLogRetirementRecord({
|
||||
...claim().candidate,
|
||||
eligibleAtMs: 61000,
|
||||
retiredAtMs: 80000,
|
||||
disposition: 'already_absent',
|
||||
byteLength: 0,
|
||||
truncation: { truncated: 'unknown' },
|
||||
});
|
||||
const rows = [
|
||||
{
|
||||
...record,
|
||||
finishedAtMs: String(record.finishedAtMs),
|
||||
eligibleAtMs: String(record.eligibleAtMs),
|
||||
retiredAtMs: String(record.retiredAtMs),
|
||||
byteLength: String(record.byteLength),
|
||||
truncated: 'unknown',
|
||||
maximumBytes: null,
|
||||
truncationObservedAtMs: null,
|
||||
},
|
||||
];
|
||||
const repository = new PostgresRunAttemptLogRetentionClaimRepository({
|
||||
async connect() {
|
||||
throw new Error('not expected');
|
||||
},
|
||||
async query(text, values) {
|
||||
assert.match(text, /artifact_tombstones/);
|
||||
assert.deepEqual(values, [ARTIFACT_ID]);
|
||||
return { rows: rows.splice(0), rowCount: 1 };
|
||||
},
|
||||
});
|
||||
const identity = {
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
logArtifactId: ARTIFACT_ID,
|
||||
};
|
||||
|
||||
assert.deepEqual(await repository.inspect(identity), {
|
||||
status: 'retired',
|
||||
record,
|
||||
});
|
||||
assert.deepEqual(await repository.inspect(identity), { status: 'active' });
|
||||
});
|
||||
|
||||
test('records the exact tombstone and removes its claim atomically', async () => {
|
||||
const record = createRunAttemptLogRetirementRecord({
|
||||
...claim().candidate,
|
||||
eligibleAtMs: 61000,
|
||||
retiredAtMs: 80000,
|
||||
disposition: 'deleted',
|
||||
byteLength: 11,
|
||||
truncation: {
|
||||
truncated: false,
|
||||
maximumBytes: 1048576,
|
||||
observedAtMs: 80000,
|
||||
},
|
||||
});
|
||||
const calls = [];
|
||||
let released = false;
|
||||
const repository = new PostgresRunAttemptLogRetentionClaimRepository({
|
||||
async connect() {
|
||||
return {
|
||||
async query(text, values = []) {
|
||||
calls.push({ text, values });
|
||||
if (text.includes('FOR UPDATE OF control')) {
|
||||
return {
|
||||
rows: [claimRow({ observedAtMs: '80000' })],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (text.startsWith('INSERT INTO "ql3"."run_attempt_log_artifact_tombstones"')) {
|
||||
return {
|
||||
rows: [{ recordDigest: record.recordDigest }],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (text.startsWith('DELETE FROM "ql3"."run_attempt_log_retention_controls"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
return { rows: [], rowCount: 0 };
|
||||
},
|
||||
release() {
|
||||
released = true;
|
||||
},
|
||||
};
|
||||
},
|
||||
async query() {
|
||||
throw new Error('not expected');
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(
|
||||
await repository.settle(claim(), { status: 'retired', record }),
|
||||
'settled',
|
||||
);
|
||||
assert.deepEqual(
|
||||
calls.map(({ text }) => text.split('\n', 1)[0]),
|
||||
[
|
||||
'BEGIN ISOLATION LEVEL READ COMMITTED',
|
||||
"SET LOCAL statement_timeout = '5000ms'",
|
||||
"SET LOCAL lock_timeout = '1000ms'",
|
||||
'WITH observation AS (',
|
||||
'INSERT INTO "ql3"."run_attempt_log_artifact_tombstones" (',
|
||||
'DELETE FROM "ql3"."run_attempt_log_retention_controls"',
|
||||
'COMMIT',
|
||||
],
|
||||
);
|
||||
assert.equal(calls[4].values.at(-1), record.recordDigest);
|
||||
assert.equal(released, true);
|
||||
});
|
||||
|
||||
test('returns fenced without writing when the durable lease changed', async () => {
|
||||
const calls = [];
|
||||
const repository = new PostgresRunAttemptLogRetentionClaimRepository({
|
||||
async connect() {
|
||||
return {
|
||||
async query(text) {
|
||||
calls.push(text);
|
||||
if (text.includes('FOR UPDATE OF control')) {
|
||||
return { rows: [], rowCount: 0 };
|
||||
}
|
||||
return { rows: [], rowCount: 0 };
|
||||
},
|
||||
release() {},
|
||||
};
|
||||
},
|
||||
async query() {
|
||||
throw new Error('not expected');
|
||||
},
|
||||
});
|
||||
const record = createRunAttemptLogRetirementRecord({
|
||||
...claim().candidate,
|
||||
eligibleAtMs: 61000,
|
||||
retiredAtMs: 80000,
|
||||
disposition: 'already_absent',
|
||||
byteLength: 0,
|
||||
truncation: { truncated: 'unknown' },
|
||||
});
|
||||
|
||||
assert.equal(
|
||||
await repository.settle(claim(), { status: 'retired', record }),
|
||||
'fenced',
|
||||
);
|
||||
assert.equal(
|
||||
calls.some((text) => text.includes('artifact_tombstones" (')),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('rolls back and wraps claim failures without leaking the client', async () => {
|
||||
const calls = [];
|
||||
let released = false;
|
||||
const repository = new PostgresRunAttemptLogRetentionClaimRepository(
|
||||
{
|
||||
async connect() {
|
||||
return {
|
||||
async query(text) {
|
||||
calls.push(text);
|
||||
if (text.includes('FOR UPDATE OF attempt')) throw new Error('offline');
|
||||
return { rows: [], rowCount: 0 };
|
||||
},
|
||||
release() {
|
||||
released = true;
|
||||
},
|
||||
};
|
||||
},
|
||||
async query() {
|
||||
throw new Error('not expected');
|
||||
},
|
||||
},
|
||||
() => TOKEN,
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
repository.claim({
|
||||
ownerId: 'replica-a',
|
||||
retentionMs: 60000,
|
||||
limit: 1,
|
||||
leaseMs: 5000,
|
||||
}),
|
||||
RunAttemptLogRetentionUnavailableError,
|
||||
);
|
||||
assert.equal(calls.at(-1), 'ROLLBACK');
|
||||
assert.equal(released, true);
|
||||
});
|
||||
Reference in New Issue
Block a user