feat(ql3): add local run log retention

2026-09-20 16:07:11 +08:00 · 2026-08-12 02:57:43 +08:00
parent 308aa75d89
commit 2bfa8ca279
50 changed files with 2752 additions and 85 deletions
@@ -67,14 +67,20 @@ function service(overrides = {}) {
  };
  return {
    calls,
-    value: new RunAttemptLogReadService(runs, reader, {
-      executorType: overrides.executorType ?? 'local_process',
-      artifactIdPattern: overrides.artifactIdPattern ?? /^local-[a-f0-9]{30}$/,
-      maximumReadBytes: overrides.maximumReadBytes ?? 32 * 1024,
-      ...(overrides.activeMissingIsPending === undefined
-        ? {}
-        : { activeMissingIsPending: overrides.activeMissingIsPending }),
-    }),
+    value: new RunAttemptLogReadService(
+      runs,
+      reader,
+      {
+        executorType: overrides.executorType ?? 'local_process',
+        artifactIdPattern:
+          overrides.artifactIdPattern ?? /^local-[a-f0-9]{30}$/,
+        maximumReadBytes: overrides.maximumReadBytes ?? 32 * 1024,
+        ...(overrides.activeMissingIsPending === undefined
+          ? {}
+          : { activeMissingIsPending: overrides.activeMissingIsPending }),
+      },
+      overrides.retention,
+    ),
  };
 }

@@ -224,6 +230,72 @@ test('returns a validated bounded snapshot without copying storage bytes', async
  assert.equal(calls.length, 0);
 });

+test('returns a durable retirement before storage and rechecks after a missing read', async () => {
+  const {
+    createRunAttemptLogRetirementRecord,
+  } = require('../dist/run/log-retention/runAttemptLogRetention.js');
+  const tombstone = createRunAttemptLogRetirementRecord({
+    projectId: 'prj_default',
+    runId: 'run_123',
+    attemptId: 'attempt_123',
+    logArtifactId: `local-${'a'.repeat(30)}`,
+    executorType: 'local_process',
+    finishedAtMs: 10,
+    eligibleAtMs: 20,
+    retiredAtMs: 30,
+    disposition: 'deleted',
+    byteLength: 42,
+    truncation: { truncated: false, maximumBytes: 1024, observedAtMs: 9 },
+  });
+  let inspections = 0;
+  let reads = 0;
+  const before = service({
+    retention: {
+      async inspect() {
+        inspections += 1;
+        return { status: 'retired', record: tombstone };
+      },
+    },
+    reader: {
+      async read() {
+        reads += 1;
+        return { status: 'missing' };
+      },
+    },
+  });
+  assert.deepEqual(await before.value.read(request()), {
+    status: 'retired',
+    projectId: 'prj_default',
+    runId: 'run_123',
+    attemptId: 'attempt_123',
+    logArtifactId: `local-${'a'.repeat(30)}`,
+    retiredAtMs: 30,
+    byteLength: 42,
+    truncation: { truncated: false, maximumBytes: 1024, observedAtMs: 9 },
+  });
+  assert.equal(inspections, 1);
+  assert.equal(reads, 0);
+
+  inspections = 0;
+  const after = service({
+    retention: {
+      async inspect() {
+        inspections += 1;
+        return inspections === 1
+          ? { status: 'active' }
+          : { status: 'retired', record: tombstone };
+      },
+    },
+    reader: {
+      async read() {
+        return { status: 'missing' };
+      },
+    },
+  });
+  assert.equal((await after.value.read(request())).status, 'retired');
+  assert.equal(inspections, 2);
+});
+
 test('fails closed on malformed storage results and dependency failures', async () => {
  const malformed = service({
    reader: {
@@ -0,0 +1,150 @@
+const assert = require('node:assert/strict');
+const { test } = require('node:test');
+
+const {
+  InvalidRunAttemptLogRetentionError,
+  RunAttemptLogRetentionService,
+  createRunAttemptLogRetirementRecord,
+  normalizeRunAttemptLogRetirementRecord,
+} = require('../dist/run/log-retention/runAttemptLogRetention.js');
+
+function candidate(index = 1) {
+  return {
+    projectId: 'prj_default',
+    runId: `run_${index}`,
+    attemptId: `attempt_${index}`,
+    logArtifactId: `local-${String(index).padStart(30, 'a')}`,
+    executorType: 'local_process',
+    finishedAtMs: index,
+  };
+}
+
+test('creates tamper-evident exact retirement records', () => {
+  const record = createRunAttemptLogRetirementRecord({
+    ...candidate(),
+    eligibleAtMs: 2,
+    retiredAtMs: 3,
+    disposition: 'deleted',
+    byteLength: 10,
+    truncation: { truncated: true, maximumBytes: 64, observedAtMs: 1 },
+  });
+  assert.match(record.recordDigest, /^[a-f0-9]{64}$/);
+  assert.deepEqual(normalizeRunAttemptLogRetirementRecord(record), record);
+  assert.throws(
+    () =>
+      normalizeRunAttemptLogRetirementRecord({
+        ...record,
+        byteLength: 11,
+      }),
+    InvalidRunAttemptLogRetentionError,
+  );
+});
+
+test('uses pressure policy and persists a bounded resume cursor', async () => {
+  let saved;
+  const recorded = [];
+  const values = [candidate(1), candidate(2), candidate(3)];
+  const service = new RunAttemptLogRetentionService(
+    {
+      async inspect() {
+        return { status: 'active' };
+      },
+      async loadCursor() {
+        return undefined;
+      },
+      async list(input) {
+        assert.equal(input.cutoffMs, 9 * 60_000);
+        assert.equal(input.limit, 3);
+        return {
+          candidates: values,
+          truncated: false,
+        };
+      },
+      async record(record) {
+        recorded.push(record);
+        return 'recorded';
+      },
+      async saveCursor(cursor) {
+        saved = cursor;
+      },
+    },
+    {
+      async retire(value) {
+        return {
+          disposition: 'deleted',
+          byteLength: value.finishedAtMs,
+          truncation: { truncated: 'unknown' },
+        };
+      },
+    },
+    {
+      async inspect() {
+        return { availableBytes: 9n, totalBytes: 100n };
+      },
+    },
+    {
+      normalRetentionMs: 5 * 60_000,
+      pressureRetentionMs: 60_000,
+      minimumFreeBytes: 10,
+      pageSize: 3,
+      maximumDeletions: 2,
+      clock: { now: () => 10 * 60_000 },
+    },
+  );
+  const result = await service.sweep();
+  assert.equal(result.status, 'deletion_budget_exhausted');
+  assert.equal(result.pressure, true);
+  assert.equal(result.deletionsAttempted, 2);
+  assert.equal(result.bytesReclaimed, 3);
+  assert.equal(recorded.length, 2);
+  assert.deepEqual(saved, { finishedAtMs: 2, attemptId: 'attempt_2' });
+});
+
+test('advances past failures and clears the cursor after a complete page', async () => {
+  const saved = [];
+  const service = new RunAttemptLogRetentionService(
+    {
+      async inspect() {
+        return { status: 'active' };
+      },
+      async loadCursor() {
+        return { finishedAtMs: 1, attemptId: 'attempt_1' };
+      },
+      async list() {
+        return { candidates: [candidate(2)], truncated: false };
+      },
+      async record() {
+        throw new Error('database unavailable');
+      },
+      async saveCursor(cursor) {
+        saved.push(cursor);
+      },
+    },
+    {
+      async retire() {
+        return {
+          disposition: 'already_absent',
+          byteLength: 0,
+          truncation: { truncated: 'unknown' },
+        };
+      },
+    },
+    {
+      async inspect() {
+        return { availableBytes: 100n, totalBytes: 100n };
+      },
+    },
+    {
+      normalRetentionMs: 60_000,
+      pressureRetentionMs: 60_000,
+      minimumFreeBytes: 0,
+      pageSize: 2,
+      maximumDeletions: 2,
+      clock: { now: () => 10 * 60_000 },
+    },
+  );
+  const result = await service.sweep();
+  assert.equal(result.failedCandidates, 1);
+  assert.equal(result.entries[0].outcome, 'record_failed');
+  assert.deepEqual(saved, [undefined]);
+});