mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-23 12:05:27 +08:00
feat(ql3): complete cluster log retention lifecycle
This commit is contained in:
whyour
@@ -388,6 +388,7 @@ function bootstrapOptions(events, overrides = {}) {
|
||||
authenticator,
|
||||
policies,
|
||||
runs,
|
||||
runAttemptLogRetention,
|
||||
runCancellation,
|
||||
taskDefinitions,
|
||||
taskExecutionRevisions,
|
||||
@@ -408,6 +409,7 @@ function bootstrapOptions(events, overrides = {}) {
|
||||
assert.equal(typeof authenticator.authenticate, 'function');
|
||||
assert.equal(typeof policies.resolve, 'function');
|
||||
assert.equal(typeof runs.transaction, 'function');
|
||||
assert.equal(typeof runAttemptLogRetention.inspect, 'function');
|
||||
assert.equal(typeof runCancellation.requestUserCancellation, 'function');
|
||||
assert.equal(
|
||||
typeof taskDefinitions.findCurrentTaskDefinition,
|
||||
|
||||
@@ -114,6 +114,18 @@ test('builds an exact runtime-only TLS-verified Pool configuration', async () =>
|
||||
assert.deepEqual(config.security, {
|
||||
apiCredentialPepper: BASE_ENV.QL3_API_CREDENTIAL_PEPPER,
|
||||
});
|
||||
assert.deepEqual(config.logRetention, {
|
||||
enabled: true,
|
||||
retentionMs: 30 * 24 * 60 * 60_000,
|
||||
claimLimit: 4,
|
||||
leaseMs: 30_000,
|
||||
maximumCycleMs: 10_000,
|
||||
retryBaseMs: 5_000,
|
||||
retryMaximumMs: 60 * 60_000,
|
||||
maximumFailures: 8,
|
||||
intervalMs: 60_000,
|
||||
stopTimeoutMs: 10_000,
|
||||
});
|
||||
|
||||
const binding = createClusterControlDatabaseBinding(config);
|
||||
assert.equal(binding.availability.status, 'available');
|
||||
@@ -192,6 +204,17 @@ test('rejects TLS query overrides, missing credentials and unbounded values', ()
|
||||
{ ...BASE_ENV, QL3_CLUSTER_AUTH_RATE_GLOBAL: '1000001' },
|
||||
{ ...BASE_ENV, QL3_CLUSTER_AUTH_RATE_MAX_PEERS: '65537' },
|
||||
{ ...BASE_ENV, QL3_API_CREDENTIAL_PEPPER: 'weak' },
|
||||
{ ...BASE_ENV, QL3_CLUSTER_LOG_RETENTION_CLAIM_LIMIT: '17' },
|
||||
{
|
||||
...BASE_ENV,
|
||||
QL3_CLUSTER_LOG_RETENTION_LEASE_MS: '5000',
|
||||
QL3_CLUSTER_LOG_RETENTION_CYCLE_BUDGET_MS: '4501',
|
||||
},
|
||||
{
|
||||
...BASE_ENV,
|
||||
QL3_CLUSTER_LOG_RETENTION_RETRY_BASE_MS: '5000',
|
||||
QL3_CLUSTER_LOG_RETENTION_RETRY_MAX_MS: '4999',
|
||||
},
|
||||
]) {
|
||||
assert.throws(
|
||||
() => loadClusterControlConfig(environment),
|
||||
@@ -199,3 +222,37 @@ test('rejects TLS query overrides, missing credentials and unbounded values', ()
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('loads bounded Cluster log retention policy and permits explicit disable', () => {
|
||||
const disabled = loadClusterControlConfig({
|
||||
...BASE_ENV,
|
||||
QL3_CLUSTER_LOG_RETENTION_ENABLED: 'false',
|
||||
QL3_CLUSTER_LOG_RETENTION_CLAIM_LIMIT: '999',
|
||||
});
|
||||
assert.deepEqual(disabled.logRetention, { enabled: false });
|
||||
|
||||
const configured = loadClusterControlConfig({
|
||||
...BASE_ENV,
|
||||
QL3_CLUSTER_LOG_RETENTION_MS: '60000',
|
||||
QL3_CLUSTER_LOG_RETENTION_CLAIM_LIMIT: '2',
|
||||
QL3_CLUSTER_LOG_RETENTION_LEASE_MS: '5000',
|
||||
QL3_CLUSTER_LOG_RETENTION_CYCLE_BUDGET_MS: '4000',
|
||||
QL3_CLUSTER_LOG_RETENTION_RETRY_BASE_MS: '250',
|
||||
QL3_CLUSTER_LOG_RETENTION_RETRY_MAX_MS: '1000',
|
||||
QL3_CLUSTER_LOG_RETENTION_MAX_FAILURES: '3',
|
||||
QL3_CLUSTER_LOG_RETENTION_INTERVAL_MS: '2000',
|
||||
QL3_CLUSTER_LOG_RETENTION_STOP_TIMEOUT_MS: '500',
|
||||
});
|
||||
assert.deepEqual(configured.logRetention, {
|
||||
enabled: true,
|
||||
retentionMs: 60_000,
|
||||
claimLimit: 2,
|
||||
leaseMs: 5_000,
|
||||
maximumCycleMs: 4_000,
|
||||
retryBaseMs: 250,
|
||||
retryMaximumMs: 1_000,
|
||||
maximumFailures: 3,
|
||||
intervalMs: 2_000,
|
||||
stopTimeoutMs: 500,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -75,8 +75,14 @@ test('runs one production replica and drains it on the first signal', async () =
|
||||
|
||||
assert.equal(result, 'stopped');
|
||||
assert.deepEqual(events, ['subscribe', 'start', 'stop', 'unsubscribe']);
|
||||
assert.equal(facts.some((fact) => fact.event === 'activation'), true);
|
||||
assert.equal(facts.some((fact) => fact.event === 'listening'), true);
|
||||
assert.equal(
|
||||
facts.some((fact) => fact.event === 'activation'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
facts.some((fact) => fact.event === 'listening'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
facts.some(
|
||||
(fact) =>
|
||||
@@ -105,7 +111,11 @@ test('fails closed before startup for a disabled profile or invalid replica id',
|
||||
await assert.rejects(
|
||||
runProductionClusterControlProcess({
|
||||
environment,
|
||||
signals: { subscribe() { return () => {}; } },
|
||||
signals: {
|
||||
subscribe() {
|
||||
return () => {};
|
||||
},
|
||||
},
|
||||
emit() {},
|
||||
async start() {
|
||||
starts += 1;
|
||||
@@ -154,6 +164,7 @@ test('starts the optional Worker listener and closes its lazy Artifact binding',
|
||||
const artifactStore = {
|
||||
async put() {},
|
||||
async inspect() {},
|
||||
async retire() {},
|
||||
};
|
||||
const environment = {
|
||||
...BASE_ENV,
|
||||
@@ -187,6 +198,14 @@ test('starts the optional Worker listener and closes its lazy Artifact binding',
|
||||
events.push('start');
|
||||
assert.equal(options.workerIngress.config.enabled, true);
|
||||
assert.equal(options.workerIngress.artifactStore, artifactStore);
|
||||
assert.equal(options.logRetention.store, artifactStore);
|
||||
assert.equal(options.logRetention.ownerId, 'cluster-control-0');
|
||||
assert.equal(options.logRetention.claimLimit, 4);
|
||||
options.logRetention.onDiagnostic(
|
||||
Object.assign(new Error('must-not-be-logged'), {
|
||||
code: 'S3Unavailable',
|
||||
}),
|
||||
);
|
||||
return {
|
||||
status: 'active',
|
||||
address: { host: '0.0.0.0', port: 5800 },
|
||||
@@ -219,6 +238,16 @@ test('starts the optional Worker listener and closes its lazy Artifact binding',
|
||||
facts.some((fact) => fact.event === 'worker_ingress_listening'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
facts.some(
|
||||
(fact) =>
|
||||
fact.event === 'runtime_diagnostic' &&
|
||||
fact.diagnostic.scope === 'log-retention' &&
|
||||
fact.diagnostic.code === 'S3Unavailable' &&
|
||||
JSON.stringify(fact).includes('must-not-be-logged') === false,
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('creates the configured mounted Secret provider before Worker activation', async () => {
|
||||
@@ -226,6 +255,7 @@ test('creates the configured mounted Secret provider before Worker activation',
|
||||
const artifactStore = {
|
||||
async put() {},
|
||||
async inspect() {},
|
||||
async retire() {},
|
||||
};
|
||||
const provider = { async resolve() {} };
|
||||
const result = await runProductionClusterControlProcess({
|
||||
@@ -353,7 +383,10 @@ test('fails the process after a database fence drains the active application', a
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(facts.some(({ event }) => event === 'shutdown_requested'), false);
|
||||
assert.equal(
|
||||
facts.some(({ event }) => event === 'shutdown_requested'),
|
||||
false,
|
||||
);
|
||||
assert.equal(facts.at(-1).event, 'stopped');
|
||||
assert.equal(
|
||||
JSON.stringify(facts).includes('must-not-escape-database-detail'),
|
||||
|
||||
@@ -617,6 +617,87 @@ test('wires the production Worker object reader into the Project-scoped log rout
|
||||
assert.equal(Buffer.from(result.body.content, 'base64').toString(), 'prod');
|
||||
});
|
||||
|
||||
test('wires durable retirement authority into the production log route', async () => {
|
||||
const {
|
||||
createRunAttemptLogRetirementRecord,
|
||||
} = require('@qinglong/runtime-core/run-attempt-log-retention');
|
||||
const { input } = fixture();
|
||||
const run = await input.runs.findRunById('run-1');
|
||||
const logArtifactId = `wlog-${'b'.repeat(30)}`;
|
||||
let objectReads = 0;
|
||||
const stack = createProductionClusterControlApplicationStack({
|
||||
...input,
|
||||
runs: {
|
||||
...input.runs,
|
||||
async findRunById() {
|
||||
return { ...run, status: 'succeeded', finishedAtMs: 10 };
|
||||
},
|
||||
async findAttemptById() {
|
||||
return {
|
||||
id: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
attempt: 1,
|
||||
status: 'succeeded',
|
||||
executorType: 'remote_worker',
|
||||
logArtifactId,
|
||||
callbackSequence: 0,
|
||||
createdAtMs: 1,
|
||||
finishedAtMs: 10,
|
||||
};
|
||||
},
|
||||
},
|
||||
runAttemptLogRetention: {
|
||||
async inspect(identity) {
|
||||
assert.deepEqual(identity, {
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
logArtifactId,
|
||||
});
|
||||
return {
|
||||
status: 'retired',
|
||||
record: createRunAttemptLogRetirementRecord({
|
||||
...identity,
|
||||
executorType: 'remote_worker',
|
||||
finishedAtMs: 10,
|
||||
eligibleAtMs: 20,
|
||||
retiredAtMs: 30,
|
||||
disposition: 'deleted',
|
||||
byteLength: 64,
|
||||
truncation: { truncated: 'unknown' },
|
||||
}),
|
||||
};
|
||||
},
|
||||
},
|
||||
workerRuntime: {
|
||||
offers: { claimNext() {} },
|
||||
activation: {
|
||||
acknowledgeStarting() {},
|
||||
acknowledgeRunning() {},
|
||||
failStart() {},
|
||||
},
|
||||
artifacts: { upload() {} },
|
||||
completion: { complete() {} },
|
||||
leaseControl: { control() {} },
|
||||
runAttemptLogRead: {
|
||||
async read() {
|
||||
objectReads += 1;
|
||||
return { status: 'missing' };
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const result = await invoke(
|
||||
stack,
|
||||
metadata('/api/v3/projects/project-1/runs/run-1/attempts/attempt-1/log'),
|
||||
);
|
||||
assert.equal(result.statusCode, 410);
|
||||
assert.equal(result.body.status, 'retired');
|
||||
assert.equal(result.body.retiredAtMs, 30);
|
||||
assert.equal(result.body.byteLength, 64);
|
||||
assert.equal(objectReads, 0);
|
||||
});
|
||||
|
||||
test('optionally exposes Prompt execution behind shared admission and policy', async () => {
|
||||
const { events, input } = fixture();
|
||||
let command;
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
ClusterRunAttemptLogRetentionCoordinator,
|
||||
ClusterRunAttemptLogRetentionLifecycle,
|
||||
} = require('../dist/run/runAttemptLogRetentionLifecycle');
|
||||
|
||||
function claim(overrides = {}) {
|
||||
return Object.freeze({
|
||||
candidate: Object.freeze({
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
logArtifactId: `wlog-${'a'.repeat(30)}`,
|
||||
executorType: 'remote_worker',
|
||||
finishedAtMs: 1_000,
|
||||
}),
|
||||
eligibleAtMs: 61_000,
|
||||
observedAtMs: 70_000,
|
||||
ownerId: 'replica-a',
|
||||
token: '00000000-0000-4000-8000-000000000055',
|
||||
version: 1,
|
||||
expiresAtMs: 100_000,
|
||||
failureCount: 0,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
function options(overrides = {}) {
|
||||
return {
|
||||
ownerId: 'replica-a',
|
||||
retentionMs: 60_000,
|
||||
claimLimit: 4,
|
||||
leaseMs: 30_000,
|
||||
maximumCycleMs: 10_000,
|
||||
retryBaseMs: 1_000,
|
||||
retryMaximumMs: 8_000,
|
||||
maximumFailures: 3,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function coordinator({
|
||||
claims = [claim()],
|
||||
hasMore = false,
|
||||
retire,
|
||||
settle,
|
||||
} = {}) {
|
||||
const calls = [];
|
||||
const value = new ClusterRunAttemptLogRetentionCoordinator(
|
||||
{
|
||||
async claim(input) {
|
||||
calls.push(['claim', input]);
|
||||
return { claims, hasMore };
|
||||
},
|
||||
async settle(current, settlement) {
|
||||
calls.push(['settle', current, settlement]);
|
||||
return (await settle?.(current, settlement)) ?? 'settled';
|
||||
},
|
||||
async inspect() {
|
||||
return { status: 'active' };
|
||||
},
|
||||
},
|
||||
{
|
||||
async retire(candidate, signal) {
|
||||
calls.push(['retire', candidate, signal]);
|
||||
return (
|
||||
(await retire?.(candidate, signal)) ?? {
|
||||
disposition: 'deleted',
|
||||
byteLength: 11,
|
||||
truncation: { truncated: false },
|
||||
}
|
||||
);
|
||||
},
|
||||
},
|
||||
options(),
|
||||
);
|
||||
return { calls, coordinator: value };
|
||||
}
|
||||
|
||||
test('claims one bounded page and records exact DB-clock retirement evidence', async () => {
|
||||
const { calls, coordinator: value } = coordinator({
|
||||
claims: [
|
||||
claim(),
|
||||
claim({
|
||||
candidate: Object.freeze({
|
||||
...claim().candidate,
|
||||
attemptId: 'attempt-2',
|
||||
logArtifactId: `wlog-${'b'.repeat(30)}`,
|
||||
}),
|
||||
}),
|
||||
],
|
||||
hasMore: true,
|
||||
retire(candidate) {
|
||||
return candidate.attemptId === 'attempt-1'
|
||||
? {
|
||||
disposition: 'deleted',
|
||||
byteLength: 11,
|
||||
truncation: { truncated: false },
|
||||
}
|
||||
: {
|
||||
disposition: 'already_absent',
|
||||
byteLength: 0,
|
||||
truncation: { truncated: 'unknown' },
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
const summary = await value.runOnce();
|
||||
assert.deepEqual(summary, {
|
||||
status: 'saturated',
|
||||
claimed: 2,
|
||||
attempted: 2,
|
||||
retired: 1,
|
||||
alreadyAbsent: 1,
|
||||
retried: 0,
|
||||
manual: 0,
|
||||
fenced: 0,
|
||||
hasMore: true,
|
||||
entries: [
|
||||
{ attemptId: 'attempt-1', outcome: 'deleted' },
|
||||
{ attemptId: 'attempt-2', outcome: 'already_absent' },
|
||||
],
|
||||
});
|
||||
assert.deepEqual(calls[0][1], {
|
||||
ownerId: 'replica-a',
|
||||
retentionMs: 60_000,
|
||||
limit: 4,
|
||||
leaseMs: 30_000,
|
||||
});
|
||||
const records = calls
|
||||
.filter(([kind]) => kind === 'settle')
|
||||
.map(([, , settlement]) => settlement.record);
|
||||
assert.equal(records[0].retiredAtMs, 70_000);
|
||||
assert.equal(records[0].recordDigest.length, 64);
|
||||
assert.equal(records[1].byteLength, 0);
|
||||
});
|
||||
|
||||
test('uses bounded exponential retry then moves repeated failures to manual', async () => {
|
||||
const first = claim({ failureCount: 2 });
|
||||
const second = claim({
|
||||
candidate: Object.freeze({
|
||||
...claim().candidate,
|
||||
attemptId: 'attempt-2',
|
||||
logArtifactId: `wlog-${'b'.repeat(30)}`,
|
||||
}),
|
||||
failureCount: 1,
|
||||
});
|
||||
const { calls, coordinator: value } = coordinator({
|
||||
claims: [first, second],
|
||||
retire(candidate) {
|
||||
const error = new Error('object drift');
|
||||
if (candidate.attemptId === 'attempt-1')
|
||||
error.reason = 'integrity_mismatch';
|
||||
throw error;
|
||||
},
|
||||
});
|
||||
|
||||
const summary = await value.runOnce();
|
||||
assert.equal(summary.manual, 1);
|
||||
assert.equal(summary.retried, 1);
|
||||
const settlements = calls
|
||||
.filter(([kind]) => kind === 'settle')
|
||||
.map(([, , settlement]) => settlement);
|
||||
assert.deepEqual(settlements, [
|
||||
{ status: 'manual', failureCode: 'artifact_integrity_mismatch' },
|
||||
{
|
||||
status: 'retry',
|
||||
delayMs: 2_000,
|
||||
failureCode: 'artifact_unavailable',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('classifies malformed retirement evidence and preserves a fenced settlement', async () => {
|
||||
const { coordinator: value } = coordinator({
|
||||
retire() {
|
||||
return {
|
||||
disposition: 'already_absent',
|
||||
byteLength: 5,
|
||||
truncation: { truncated: 'unknown' },
|
||||
};
|
||||
},
|
||||
settle() {
|
||||
return 'fenced';
|
||||
},
|
||||
});
|
||||
const summary = await value.runOnce();
|
||||
assert.equal(summary.fenced, 1);
|
||||
assert.deepEqual(summary.entries, [
|
||||
{ attemptId: 'attempt-1', outcome: 'fenced' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('cycle budget aborts object work and leaves the durable claim for takeover', async () => {
|
||||
let settlements = 0;
|
||||
const value = new ClusterRunAttemptLogRetentionCoordinator(
|
||||
{
|
||||
async claim() {
|
||||
return { claims: [claim()], hasMore: false };
|
||||
},
|
||||
async settle() {
|
||||
settlements += 1;
|
||||
return 'settled';
|
||||
},
|
||||
async inspect() {
|
||||
return { status: 'active' };
|
||||
},
|
||||
},
|
||||
{
|
||||
retire(_candidate, signal) {
|
||||
return new Promise((_, reject) => {
|
||||
signal.addEventListener('abort', () => reject(signal.reason), {
|
||||
once: true,
|
||||
});
|
||||
});
|
||||
},
|
||||
},
|
||||
options({ maximumCycleMs: 100 }),
|
||||
);
|
||||
const summary = await value.runOnce();
|
||||
assert.equal(summary.status, 'budget_exhausted');
|
||||
assert.equal(summary.attempted, 1);
|
||||
assert.equal(settlements, 0);
|
||||
});
|
||||
|
||||
test('lifecycle coalesces cycles and aborts one in-flight object call on drain', async () => {
|
||||
let calls = 0;
|
||||
let observedAbort = false;
|
||||
const lifecycle = new ClusterRunAttemptLogRetentionLifecycle(
|
||||
{
|
||||
runOnce(signal) {
|
||||
calls += 1;
|
||||
return new Promise((resolve) => {
|
||||
signal.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
observedAbort = true;
|
||||
resolve({ status: 'budget_exhausted' });
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
},
|
||||
},
|
||||
{ intervalMs: 60_000, stopTimeoutMs: 1_000 },
|
||||
);
|
||||
const first = lifecycle.runOnce();
|
||||
assert.equal(lifecycle.runOnce(), first);
|
||||
assert.equal(await lifecycle.stopAndDrain(), 'stopped');
|
||||
await first;
|
||||
assert.equal(calls, 1);
|
||||
assert.equal(observedAbort, true);
|
||||
await assert.rejects(lifecycle.runOnce(), /is stopping/);
|
||||
});
|
||||
|
||||
test('rejects configurations that can outlive the lease settlement budget', () => {
|
||||
const dependencies = [
|
||||
{ claim() {}, settle() {}, inspect() {} },
|
||||
{ retire() {} },
|
||||
];
|
||||
assert.throws(
|
||||
() =>
|
||||
new ClusterRunAttemptLogRetentionCoordinator(
|
||||
dependencies[0],
|
||||
dependencies[1],
|
||||
options({ leaseMs: 5_000, maximumCycleMs: 4_501 }),
|
||||
),
|
||||
/cycle budget/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
new ClusterRunAttemptLogRetentionLifecycle(
|
||||
{ runOnce() {} },
|
||||
{ intervalMs: 999, stopTimeoutMs: 1_000 },
|
||||
),
|
||||
/lifecycle options/,
|
||||
);
|
||||
});
|
||||
@@ -7,7 +7,9 @@ const {
|
||||
CreateBucketCommand,
|
||||
DeleteBucketCommand,
|
||||
DeleteObjectsCommand,
|
||||
ListObjectVersionsCommand,
|
||||
ListObjectsV2Command,
|
||||
PutBucketVersioningCommand,
|
||||
S3Client,
|
||||
} = require('@aws-sdk/client-s3');
|
||||
const {
|
||||
@@ -35,6 +37,7 @@ test(
|
||||
credentials: { accessKeyId, secretAccessKey },
|
||||
});
|
||||
const bucket = `ql3-artifact-${process.pid}-${Date.now()}`.slice(0, 63);
|
||||
const versionedBucket = `${bucket}-v`.slice(0, 63);
|
||||
const command = Object.freeze({
|
||||
projectId: 'project-s3-integration',
|
||||
runId: 'run-s3-integration',
|
||||
@@ -101,27 +104,123 @@ test(
|
||||
);
|
||||
assert.equal(objects.KeyCount, 1);
|
||||
assert.match(objects.Contents[0].Key, /\/objects\//);
|
||||
} finally {
|
||||
try {
|
||||
const objects = await client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
}),
|
||||
);
|
||||
if (objects.Contents?.length) {
|
||||
const retired = await store.retire({
|
||||
projectId: command.projectId,
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
logArtifactId: command.logArtifactId,
|
||||
executorType: 'remote_worker',
|
||||
finishedAtMs: 1,
|
||||
});
|
||||
assert.deepEqual(retired, {
|
||||
disposition: 'deleted',
|
||||
byteLength: content.byteLength,
|
||||
truncation: { truncated: true },
|
||||
});
|
||||
assert.equal(
|
||||
(
|
||||
await client.send(
|
||||
new DeleteObjectsCommand({
|
||||
new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
Delete: {
|
||||
Objects: objects.Contents.map(({ Key }) => ({ Key })),
|
||||
Quiet: true,
|
||||
},
|
||||
Prefix: 'qinglong/integration/',
|
||||
}),
|
||||
)
|
||||
).KeyCount,
|
||||
0,
|
||||
);
|
||||
assert.deepEqual(
|
||||
await store.retire({
|
||||
projectId: command.projectId,
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
logArtifactId: command.logArtifactId,
|
||||
executorType: 'remote_worker',
|
||||
finishedAtMs: 1,
|
||||
}),
|
||||
{
|
||||
disposition: 'already_absent',
|
||||
byteLength: 0,
|
||||
truncation: { truncated: 'unknown' },
|
||||
},
|
||||
);
|
||||
|
||||
await client.send(new CreateBucketCommand({ Bucket: versionedBucket }));
|
||||
await client.send(
|
||||
new PutBucketVersioningCommand({
|
||||
Bucket: versionedBucket,
|
||||
VersioningConfiguration: { Status: 'Enabled' },
|
||||
}),
|
||||
);
|
||||
const versionedStore = new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket: versionedBucket,
|
||||
prefix: 'qinglong/integration',
|
||||
encryption: { mode: 's3' },
|
||||
});
|
||||
assert.equal(
|
||||
(await versionedStore.put(command, body(content))).status,
|
||||
'stored',
|
||||
);
|
||||
const beforeVersionedRetirement = await client.send(
|
||||
new ListObjectVersionsCommand({ Bucket: versionedBucket }),
|
||||
);
|
||||
assert.equal(beforeVersionedRetirement.Versions?.length, 1);
|
||||
assert.equal(beforeVersionedRetirement.DeleteMarkers?.length ?? 0, 0);
|
||||
assert.match(beforeVersionedRetirement.Versions[0].Key, /\/objects\//);
|
||||
assert.equal(
|
||||
(
|
||||
await versionedStore.retire({
|
||||
projectId: command.projectId,
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
logArtifactId: command.logArtifactId,
|
||||
executorType: 'remote_worker',
|
||||
finishedAtMs: 1,
|
||||
})
|
||||
).disposition,
|
||||
'deleted',
|
||||
);
|
||||
const afterVersionedRetirement = await client.send(
|
||||
new ListObjectVersionsCommand({ Bucket: versionedBucket }),
|
||||
);
|
||||
assert.equal(afterVersionedRetirement.Versions?.length ?? 0, 0);
|
||||
assert.equal(afterVersionedRetirement.DeleteMarkers?.length ?? 0, 0);
|
||||
} finally {
|
||||
for (const cleanupBucket of [versionedBucket, bucket]) {
|
||||
try {
|
||||
const versions = await client.send(
|
||||
new ListObjectVersionsCommand({ Bucket: cleanupBucket }),
|
||||
);
|
||||
const versionedObjects = [
|
||||
...(versions.Versions ?? []),
|
||||
...(versions.DeleteMarkers ?? []),
|
||||
].map(({ Key, VersionId }) => ({ Key, VersionId }));
|
||||
if (versionedObjects.length) {
|
||||
await client.send(
|
||||
new DeleteObjectsCommand({
|
||||
Bucket: cleanupBucket,
|
||||
Delete: { Objects: versionedObjects, Quiet: true },
|
||||
}),
|
||||
);
|
||||
}
|
||||
const objects = await client.send(
|
||||
new ListObjectsV2Command({ Bucket: cleanupBucket }),
|
||||
);
|
||||
if (objects.Contents?.length) {
|
||||
await client.send(
|
||||
new DeleteObjectsCommand({
|
||||
Bucket: cleanupBucket,
|
||||
Delete: {
|
||||
Objects: objects.Contents.map(({ Key }) => ({ Key })),
|
||||
Quiet: true,
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
await client.send(new DeleteBucketCommand({ Bucket: cleanupBucket }));
|
||||
} catch {
|
||||
// Preserve the integration assertion; the ephemeral container is removed.
|
||||
}
|
||||
await client.send(new DeleteBucketCommand({ Bucket: bucket }));
|
||||
} catch {
|
||||
// Preserve the integration assertion; the ephemeral container is removed.
|
||||
}
|
||||
client.destroy();
|
||||
}
|
||||
|
||||
@@ -73,6 +73,14 @@ class MemoryS3Client {
|
||||
? Buffer.alloc(32, 9).toString('base64')
|
||||
: checksum(object.content),
|
||||
Metadata: metadata,
|
||||
...(input.Key.includes('/objects/') &&
|
||||
this.options.headVersionId !== undefined
|
||||
? { VersionId: this.options.headVersionId }
|
||||
: {}),
|
||||
...(input.Key.includes('/temporary/') &&
|
||||
this.options.temporaryHeadVersionId !== undefined
|
||||
? { VersionId: this.options.temporaryHeadVersionId }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
if (command instanceof GetObjectCommand) {
|
||||
@@ -168,7 +176,33 @@ class MemoryS3Client {
|
||||
}
|
||||
if (command instanceof DeleteObjectCommand) {
|
||||
if (this.options.failDelete) throw new Error('delete unavailable');
|
||||
if (input.Key.includes('/objects/')) {
|
||||
if (this.options.permanentDeletePreconditionFailure) {
|
||||
const error = new Error('precondition failed');
|
||||
error.name = 'PreconditionFailed';
|
||||
error.$metadata = { httpStatusCode: 412 };
|
||||
throw error;
|
||||
}
|
||||
const object = this.objects.get(input.Key);
|
||||
if (!object) throw notFound();
|
||||
if (this.options.headVersionId === undefined) {
|
||||
assert.equal(
|
||||
input.IfMatch,
|
||||
`"${checksum(object.content).slice(0, 32)}"`,
|
||||
);
|
||||
assert.equal(input.VersionId, undefined);
|
||||
} else {
|
||||
assert.equal(input.IfMatch, undefined);
|
||||
assert.equal(input.VersionId, this.options.headVersionId);
|
||||
}
|
||||
}
|
||||
this.objects.delete(input.Key);
|
||||
if (
|
||||
input.Key.includes('/objects/') &&
|
||||
this.options.throwAfterPermanentDelete
|
||||
) {
|
||||
throw new Error('lost delete response');
|
||||
}
|
||||
return {};
|
||||
}
|
||||
throw new Error(`unexpected command: ${command.constructor.name}`);
|
||||
@@ -201,6 +235,15 @@ function permanentKey(client) {
|
||||
return [...client.objects.keys()].find((key) => key.includes('/objects/'));
|
||||
}
|
||||
|
||||
function retentionCandidate(overrides = {}) {
|
||||
return Object.freeze({
|
||||
...LOOKUP,
|
||||
executorType: 'remote_worker',
|
||||
finishedAtMs: 1_000,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
test('streams to a checksummed temporary object then conditionally promotes it', async () => {
|
||||
const client = new MemoryS3Client();
|
||||
const adapter = store(client);
|
||||
@@ -233,6 +276,13 @@ test('streams to a checksummed temporary object then conditionally promotes it',
|
||||
const copy = client.commands.find(
|
||||
(command) => command instanceof CopyObjectCommand,
|
||||
);
|
||||
const cleanup = client.commands.find(
|
||||
(command) =>
|
||||
command instanceof DeleteObjectCommand &&
|
||||
command.input.Key.includes('/temporary/'),
|
||||
);
|
||||
assert.match(cleanup.input.IfMatch, /^"[A-Za-z0-9+/=]+"$/);
|
||||
assert.equal(cleanup.input.VersionId, undefined);
|
||||
assert.equal(copy.input.Metadata['ql3-content-sha256'], CONTENT_SHA256);
|
||||
assert.equal(
|
||||
JSON.stringify(copy.input.Metadata).includes(COMMAND.projectId),
|
||||
@@ -247,6 +297,20 @@ test('streams to a checksummed temporary object then conditionally promotes it',
|
||||
assert.deepEqual(inspected, { ...receipt, status: 'already_stored' });
|
||||
});
|
||||
|
||||
test('cleans one exact temporary object version after validated HEAD', async () => {
|
||||
const client = new MemoryS3Client({
|
||||
temporaryHeadVersionId: 'temporary/version+1=',
|
||||
});
|
||||
await store(client).put(COMMAND, chunks());
|
||||
const cleanup = client.commands.find(
|
||||
(command) =>
|
||||
command instanceof DeleteObjectCommand &&
|
||||
command.input.Key.includes('/temporary/'),
|
||||
);
|
||||
assert.equal(cleanup.input.VersionId, 'temporary/version+1=');
|
||||
assert.equal(cleanup.input.IfMatch, undefined);
|
||||
});
|
||||
|
||||
test('exact replay consumes and hashes the whole body without another write', async () => {
|
||||
const client = new MemoryS3Client();
|
||||
const adapter = store(client);
|
||||
@@ -510,3 +574,101 @@ test('a pre-aborted request performs no object-store operation', async () => {
|
||||
);
|
||||
assert.equal(client.commands.length, 0);
|
||||
});
|
||||
|
||||
test('retires an unversioned Artifact only with its validated ETag', async () => {
|
||||
const client = new MemoryS3Client();
|
||||
const adapter = store(client, { expectedBucketOwner: '123456789012' });
|
||||
await adapter.put(COMMAND, chunks());
|
||||
client.commands.length = 0;
|
||||
|
||||
assert.deepEqual(await adapter.retire(retentionCandidate()), {
|
||||
disposition: 'deleted',
|
||||
byteLength: CONTENT.byteLength,
|
||||
truncation: { truncated: false },
|
||||
});
|
||||
assert.deepEqual(
|
||||
client.commands.map((command) => command.constructor.name),
|
||||
['HeadObjectCommand', 'DeleteObjectCommand'],
|
||||
);
|
||||
assert.equal(client.commands[1].input.ExpectedBucketOwner, '123456789012');
|
||||
assert.equal(permanentKey(client), undefined);
|
||||
});
|
||||
|
||||
test('retires one exact version when HEAD returns an opaque VersionId', async () => {
|
||||
const client = new MemoryS3Client({ headVersionId: 'version/opaque+1=' });
|
||||
const adapter = store(client);
|
||||
await adapter.put(COMMAND, chunks());
|
||||
client.commands.length = 0;
|
||||
|
||||
const result = await adapter.retire(retentionCandidate());
|
||||
assert.equal(result.disposition, 'deleted');
|
||||
assert.equal(client.commands[1].input.VersionId, 'version/opaque+1=');
|
||||
assert.equal(client.commands[1].input.IfMatch, undefined);
|
||||
assert.equal(permanentKey(client), undefined);
|
||||
});
|
||||
|
||||
test('returns durable absent evidence without issuing a delete', async () => {
|
||||
const client = new MemoryS3Client();
|
||||
const adapter = store(client);
|
||||
|
||||
assert.deepEqual(await adapter.retire(retentionCandidate()), {
|
||||
disposition: 'already_absent',
|
||||
byteLength: 0,
|
||||
truncation: { truncated: 'unknown' },
|
||||
});
|
||||
assert.deepEqual(
|
||||
client.commands.map((command) => command.constructor.name),
|
||||
['HeadObjectCommand'],
|
||||
);
|
||||
});
|
||||
|
||||
test('fails closed on conditional-delete drift and malformed version authority', async () => {
|
||||
for (const options of [
|
||||
{ permanentDeletePreconditionFailure: true },
|
||||
{ headVersionId: 'invalid\nversion' },
|
||||
]) {
|
||||
const client = new MemoryS3Client(options);
|
||||
const adapter = store(client);
|
||||
await adapter.put(COMMAND, chunks());
|
||||
client.commands.length = 0;
|
||||
|
||||
await assert.rejects(
|
||||
adapter.retire(retentionCandidate()),
|
||||
(error) =>
|
||||
error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
|
||||
error.reason === 'integrity_mismatch',
|
||||
);
|
||||
assert.notEqual(permanentKey(client), undefined);
|
||||
}
|
||||
|
||||
const wrongExecutor = new MemoryS3Client();
|
||||
await assert.rejects(
|
||||
store(wrongExecutor).retire(
|
||||
retentionCandidate({ executorType: 'local_process' }),
|
||||
),
|
||||
(error) =>
|
||||
error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
|
||||
error.reason === 'integrity_mismatch',
|
||||
);
|
||||
assert.equal(wrongExecutor.commands.length, 0);
|
||||
});
|
||||
|
||||
test('lost delete response converges through a later absent inspection', async () => {
|
||||
const client = new MemoryS3Client({ throwAfterPermanentDelete: true });
|
||||
const adapter = store(client);
|
||||
await adapter.put(COMMAND, chunks());
|
||||
client.commands.length = 0;
|
||||
|
||||
await assert.rejects(
|
||||
adapter.retire(retentionCandidate()),
|
||||
(error) =>
|
||||
error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
|
||||
error.reason === 'unavailable',
|
||||
);
|
||||
client.options.throwAfterPermanentDelete = false;
|
||||
assert.deepEqual(await adapter.retire(retentionCandidate()), {
|
||||
disposition: 'already_absent',
|
||||
byteLength: 0,
|
||||
truncation: { truncated: 'unknown' },
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user