feat(ql3): add local run log retention

packages/ql3-local-execution/src/artifact-read/index.ts

@@ -1 +1,2 @@
 export * from './localRunAttemptLogRangeReader';
+export * from './localRunAttemptLogRetirementStore';

packages/ql3-local-execution/src/artifact-read/localRunAttemptLogRetirementStore.ts

@@ -0,0 +1,351 @@
+import { constants, type Stats } from 'node:fs';
+import fs, { type FileHandle } from 'node:fs/promises';
+import path from 'node:path';
+
+import {
+  normalizeRunAttemptLogRetentionCandidate,
+  type RunAttemptLogCapacitySource,
+  type RunAttemptLogRetentionCandidate,
+  type RunAttemptLogRetirementStore,
+  type RunAttemptLogRetirementStoreResult,
+} from '@qinglong/runtime-core/run-attempt-log-retention';
+import type {
+  RunAttemptLogReadIdentity,
+  RunAttemptLogTruncationView,
+} from '@qinglong/runtime-core/run-attempt-log-read';
+
+const LOCAL_ARTIFACT_ID = /^local-[a-f0-9]{30}$/;
+const MAXIMUM_ARTIFACT_BYTES = 1024 * 1024 * 1024;
+const MAXIMUM_FACT_BYTES = 1024;
+
+export class LocalRunAttemptLogRetirementError extends Error {
+  constructor(
+    readonly reason:
+      | 'invalid_configuration'
+      | 'unsafe_path'
+      | 'integrity_mismatch',
+    options?: ErrorOptions,
+  ) {
+    super(`Local Run Attempt log retirement failed: ${reason}`, options);
+    this.name = 'LocalRunAttemptLogRetirementError';
+  }
+}
+
+function isCode(error: unknown, code: string): boolean {
+  return (
+    !!error &&
+    typeof error === 'object' &&
+    'code' in error &&
+    (error as { code?: unknown }).code === code
+  );
+}
+
+function currentUid(): number | undefined {
+  return typeof process.getuid === 'function' ? process.getuid() : undefined;
+}
+
+function artifactRoot(value: string): string {
+  if (
+    typeof value !== 'string' ||
+    !path.isAbsolute(value) ||
+    path.parse(value).root === value ||
+    value.includes('\0') ||
+    Buffer.byteLength(value, 'utf8') > 4096
+  ) {
+    throw new LocalRunAttemptLogRetirementError('invalid_configuration');
+  }
+  return path.resolve(value);
+}
+
+function assertOwnedDirectory(stat: Stats): void {
+  const uid = currentUid();
+  if (
+    !stat.isDirectory() ||
+    stat.isSymbolicLink() ||
+    (stat.mode & 0o777) !== 0o700 ||
+    (uid !== undefined && stat.uid !== uid)
+  ) {
+    throw new LocalRunAttemptLogRetirementError('unsafe_path');
+  }
+}
+
+function assertOwnedFile(stat: Stats, maximumBytes: number): void {
+  const uid = currentUid();
+  if (
+    !stat.isFile() ||
+    stat.isSymbolicLink() ||
+    stat.nlink !== 1 ||
+    (stat.mode & 0o777) !== 0o600 ||
+    (uid !== undefined && stat.uid !== uid) ||
+    !Number.isSafeInteger(stat.size) ||
+    stat.size < 0 ||
+    stat.size > maximumBytes
+  ) {
+    throw new LocalRunAttemptLogRetirementError('unsafe_path');
+  }
+}
+
+async function optionalOwnedDirectory(directory: string): Promise<boolean> {
+  try {
+    assertOwnedDirectory(await fs.lstat(directory));
+    return true;
+  } catch (error) {
+    if (isCode(error, 'ENOENT')) return false;
+    if (error instanceof LocalRunAttemptLogRetirementError) throw error;
+    throw new LocalRunAttemptLogRetirementError('unsafe_path', {
+      cause: error,
+    });
+  }
+}
+
+async function openPrivateFile(
+  filePath: string,
+): Promise<FileHandle | undefined> {
+  try {
+    return await fs.open(
+      filePath,
+      constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
+    );
+  } catch (error) {
+    if (isCode(error, 'ENOENT')) return undefined;
+    throw new LocalRunAttemptLogRetirementError('unsafe_path', {
+      cause: error,
+    });
+  }
+}
+
+function sameFile(left: Stats, right: Stats): boolean {
+  return left.dev === right.dev && left.ino === right.ino;
+}
+
+function exactFact(
+  value: unknown,
+  expected: Readonly<RunAttemptLogReadIdentity>,
+): Readonly<RunAttemptLogTruncationView> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new LocalRunAttemptLogRetirementError('integrity_mismatch');
+  }
+  const fact = value as Record<string, unknown>;
+  if (
+    Object.keys(fact).sort().join(',') !==
+      'attemptId,logArtifactId,maximumBytes,observedAtMs,quotaReached,runId,schemaVersion' ||
+    fact.schemaVersion !== 1 ||
+    fact.runId !== expected.runId ||
+    fact.attemptId !== expected.attemptId ||
+    fact.logArtifactId !== expected.logArtifactId ||
+    !Number.isSafeInteger(fact.maximumBytes) ||
+    Number(fact.maximumBytes) < 64 * 1024 ||
+    Number(fact.maximumBytes) > MAXIMUM_ARTIFACT_BYTES ||
+    typeof fact.quotaReached !== 'boolean' ||
+    !Number.isSafeInteger(fact.observedAtMs) ||
+    Number(fact.observedAtMs) < 0
+  ) {
+    throw new LocalRunAttemptLogRetirementError('integrity_mismatch');
+  }
+  return Object.freeze({
+    truncated: fact.quotaReached,
+    maximumBytes: fact.maximumBytes as number,
+    observedAtMs: fact.observedAtMs as number,
+  });
+}
+
+async function readFact(
+  factPath: string,
+  expected: Readonly<RunAttemptLogReadIdentity>,
+): Promise<
+  Readonly<{
+    truncation: Readonly<RunAttemptLogTruncationView>;
+    stat?: Stats;
+  }>
+> {
+  const handle = await openPrivateFile(factPath);
+  if (!handle) {
+    return Object.freeze({
+      truncation: Object.freeze({ truncated: 'unknown' as const }),
+    });
+  }
+  try {
+    const before = await handle.stat();
+    assertOwnedFile(before, MAXIMUM_FACT_BYTES);
+    if (before.size < 2) {
+      throw new LocalRunAttemptLogRetirementError('integrity_mismatch');
+    }
+    const content = Buffer.allocUnsafe(before.size);
+    try {
+      let read = 0;
+      while (read < content.byteLength) {
+        const result = await handle.read(
+          content,
+          read,
+          content.byteLength - read,
+          read,
+        );
+        if (result.bytesRead < 1) {
+          throw new LocalRunAttemptLogRetirementError('integrity_mismatch');
+        }
+        read += result.bytesRead;
+      }
+      const after = await handle.stat();
+      assertOwnedFile(after, MAXIMUM_FACT_BYTES);
+      if (!sameFile(before, after) || before.size !== after.size) {
+        throw new LocalRunAttemptLogRetirementError('integrity_mismatch');
+      }
+      const decoded = new TextDecoder('utf-8', { fatal: true }).decode(content);
+      return Object.freeze({
+        truncation: exactFact(JSON.parse(decoded), expected),
+        stat: before,
+      });
+    } catch (error) {
+      if (error instanceof LocalRunAttemptLogRetirementError) throw error;
+      throw new LocalRunAttemptLogRetirementError('integrity_mismatch', {
+        cause: error,
+      });
+    } finally {
+      content.fill(0);
+    }
+  } finally {
+    await handle.close().catch(() => undefined);
+  }
+}
+
+async function assertPathStillMatches(
+  filePath: string,
+  expected: Stats,
+  maximumBytes: number,
+): Promise<void> {
+  try {
+    const current = await fs.lstat(filePath);
+    assertOwnedFile(current, maximumBytes);
+    if (!sameFile(current, expected)) {
+      throw new LocalRunAttemptLogRetirementError('integrity_mismatch');
+    }
+  } catch (error) {
+    if (error instanceof LocalRunAttemptLogRetirementError) throw error;
+    throw new LocalRunAttemptLogRetirementError('unsafe_path', {
+      cause: error,
+    });
+  }
+}
+
+async function syncDirectory(directory: string): Promise<void> {
+  const handle = await fs.open(directory, constants.O_RDONLY);
+  try {
+    await handle.sync();
+  } finally {
+    await handle.close().catch(() => undefined);
+  }
+}
+
+export class LocalRunAttemptLogRetirementStore
+  implements RunAttemptLogRetirementStore
+{
+  private readonly root: string;
+
+  constructor(artifactRootPath: string) {
+    this.root = artifactRoot(artifactRootPath);
+  }
+
+  async retire(
+    raw: Readonly<RunAttemptLogRetentionCandidate>,
+  ): Promise<Readonly<RunAttemptLogRetirementStoreResult>> {
+    const candidate = normalizeRunAttemptLogRetentionCandidate(raw);
+    if (
+      candidate.executorType !== 'local_process' ||
+      !LOCAL_ARTIFACT_ID.test(candidate.logArtifactId)
+    ) {
+      throw new LocalRunAttemptLogRetirementError('integrity_mismatch');
+    }
+    if (!(await optionalOwnedDirectory(this.root))) {
+      return Object.freeze({
+        disposition: 'already_absent' as const,
+        byteLength: 0,
+        truncation: Object.freeze({ truncated: 'unknown' as const }),
+      });
+    }
+    const directory = path.join(
+      this.root,
+      candidate.logArtifactId.slice('local-'.length, 'local-'.length + 2),
+    );
+    if (!(await optionalOwnedDirectory(directory))) {
+      return Object.freeze({
+        disposition: 'already_absent' as const,
+        byteLength: 0,
+        truncation: Object.freeze({ truncated: 'unknown' as const }),
+      });
+    }
+
+    const target = path.join(directory, `${candidate.logArtifactId}.log`);
+    const factPath = path.join(
+      directory,
+      `.${candidate.logArtifactId}.log.truncated.json`,
+    );
+    const fact = await readFact(factPath, candidate);
+    const handle = await openPrivateFile(target);
+    if (!handle) {
+      if (fact.stat) {
+        await assertPathStillMatches(factPath, fact.stat, MAXIMUM_FACT_BYTES);
+        await fs.unlink(factPath);
+        await syncDirectory(directory);
+      }
+      return Object.freeze({
+        disposition: 'already_absent' as const,
+        byteLength: 0,
+        truncation: fact.truncation,
+      });
+    }
+    try {
+      const before = await handle.stat();
+      assertOwnedFile(before, MAXIMUM_ARTIFACT_BYTES);
+      await assertPathStillMatches(target, before, MAXIMUM_ARTIFACT_BYTES);
+      if (fact.stat) {
+        await assertPathStillMatches(factPath, fact.stat, MAXIMUM_FACT_BYTES);
+      }
+      await fs.unlink(target);
+      if (fact.stat) await fs.unlink(factPath);
+      await syncDirectory(directory);
+      return Object.freeze({
+        disposition: 'deleted' as const,
+        byteLength: before.size,
+        truncation: fact.truncation,
+      });
+    } finally {
+      await handle.close().catch(() => undefined);
+    }
+  }
+}
+
+export class LocalRunAttemptLogCapacityProbe
+  implements RunAttemptLogCapacitySource
+{
+  private readonly root: string;
+
+  constructor(artifactRootPath: string) {
+    this.root = artifactRoot(artifactRootPath);
+  }
+
+  async inspect() {
+    let current = this.root;
+    while (true) {
+      try {
+        const stat = await fs.statfs(current, { bigint: true });
+        return Object.freeze({
+          availableBytes: stat.bavail * stat.bsize,
+          totalBytes: stat.blocks * stat.bsize,
+        });
+      } catch (error) {
+        if (!isCode(error, 'ENOENT')) {
+          throw new LocalRunAttemptLogRetirementError('unsafe_path', {
+            cause: error,
+          });
+        }
+        const parent = path.dirname(current);
+        if (parent === current) {
+          throw new LocalRunAttemptLogRetirementError('unsafe_path', {
+            cause: error,
+          });
+        }
+        current = parent;
+      }
+    }
+  }
+}

packages/ql3-local-execution/src/control/lifecycle.ts

@@ -1,5 +1,6 @@
 import type { LocalCompletionReceiptJournalCursor } from '@qinglong/runtime-core/local-completion-receipt-journal';
 import { assertLocalExecutionControlLimit } from '@qinglong/runtime-core/local-execution-control';
+import type { RunAttemptLogRetentionSweepSummary } from '@qinglong/runtime-core/run-attempt-log-retention';
 import type {
   LocalCompletionReceiptCleanupScanner,
   LocalCompletionReceiptCleanupSummary,
@@ -21,6 +22,9 @@ export interface LocalExecutionControlLifecycleOptions {
   readonly stopTimeoutMs: number;
   readonly maxDrainPages: number;
   readonly maxNotifications?: number;
+  readonly artifactRetention?: Readonly<{
+    sweep(): Promise<RunAttemptLogRetentionSweepSummary>;
+  }>;
   readonly clock?: { now(): number };
   readonly onDiagnostic?: (
     error: unknown,
@@ -33,6 +37,7 @@ export interface LocalExecutionControlCycleSummary {
   readonly completionFailures: number;
   readonly control: LocalExecutionControlScanSummary;
   readonly cleanup?: LocalCompletionReceiptCleanupSummary;
+  readonly artifactRetention?: RunAttemptLogRetentionSweepSummary;
 }
 
 export interface LocalExecutionControlStopSummary {
@@ -111,6 +116,12 @@ export class LocalExecutionControlLifecycle {
     ) {
       throw new RangeError('Local completion notification budget is invalid');
     }
+    if (
+      options.artifactRetention !== undefined &&
+      typeof options.artifactRetention.sweep !== 'function'
+    ) {
+      throw new TypeError('Local Artifact retention lifecycle is invalid');
+    }
     this.clock = options.clock ?? { now: Date.now };
   }
 
@@ -234,6 +245,7 @@ export class LocalExecutionControlLifecycle {
       );
     }
     let cleanup: LocalCompletionReceiptCleanupSummary | undefined;
+    let artifactRetention: RunAttemptLogRetentionSweepSummary | undefined;
     if (
       forceCleanup ||
       this.lastCleanupAtMs === undefined ||
@@ -246,6 +258,7 @@ export class LocalExecutionControlLifecycle {
           : { cursor: this.cleanupCursor }),
       });
       this.cleanupCursor = cleanup.truncated ? cleanup.nextCursor : undefined;
+      artifactRetention = await this.options.artifactRetention?.sweep();
       this.lastCleanupAtMs = now;
     }
     if (this.pending.size > 0) this.kick();
@@ -254,6 +267,7 @@ export class LocalExecutionControlLifecycle {
       completionFailures: completion.failed,
       control,
       ...(cleanup === undefined ? {} : { cleanup }),
+      ...(artifactRetention === undefined ? {} : { artifactRetention }),
     });
   }
 

packages/ql3-local-execution/test/control.test.cjs

@@ -367,6 +367,7 @@ test('coalesces completion notifications and owns one idempotent shutdown drain'
   let scans = 0;
   let drains = 0;
   let cleanups = 0;
+  let retentionSweeps = 0;
   const lifecycle = new LocalExecutionControlLifecycle(
     {
       async process(attemptId) {
@@ -420,6 +421,25 @@ test('coalesces completion notifications and owns one idempotent shutdown drain'
       stopTimeoutMs: 1_000,
       maxDrainPages: 1,
       clock: { now: () => 100 },
+      artifactRetention: {
+        async sweep() {
+          retentionSweeps += 1;
+          return {
+            status: 'complete',
+            pressure: false,
+            observedAtMs: 100,
+            retentionMs: 60_000,
+            availableBytes: '100',
+            totalBytes: '100',
+            candidatesScanned: 0,
+            deletionsAttempted: 0,
+            recordsWritten: 0,
+            failedCandidates: 0,
+            bytesReclaimed: 0,
+            entries: [],
+          };
+        },
+      },
     },
   );
   assert.equal(lifecycle.notifyCompletion(IDS.completionAttempt), true);
@@ -428,10 +448,12 @@ test('coalesces completion notifications and owns one idempotent shutdown drain'
   assert.deepEqual(completions, [IDS.completionAttempt]);
   assert.equal(scans, 1);
   assert.equal(cleanups, 1);
+  assert.equal(retentionSweeps, 1);
   const first = lifecycle.stopAndDrain();
   const second = lifecycle.stopAndDrain();
   assert.equal(first, second);
   assert.equal((await first).status, 'stopped');
   assert.equal(drains, 1);
   assert.equal(cleanups, 2);
+  assert.equal(retentionSweeps, 1);
 });

packages/ql3-local-execution/test/localRunAttemptLogRetirementStore.test.cjs

@@ -0,0 +1,131 @@
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { test } = require('node:test');
+
+const {
+  LocalRunAttemptLogCapacityProbe,
+  LocalRunAttemptLogRetirementError,
+  LocalRunAttemptLogRetirementStore,
+} = require('../dist/artifact-read/localRunAttemptLogRetirementStore.js');
+
+const ARTIFACT_ID = `local-${'a'.repeat(30)}`;
+
+function fixture(t) {
+  const parent = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-log-retire-'));
+  const root = path.join(parent, 'artifacts');
+  fs.mkdirSync(root, { mode: 0o700 });
+  fs.chmodSync(root, 0o700);
+  const directory = path.join(root, 'aa');
+  fs.mkdirSync(directory, { mode: 0o700 });
+  fs.chmodSync(directory, 0o700);
+  t.after(() => fs.rmSync(parent, { recursive: true, force: true }));
+  return {
+    parent,
+    root,
+    directory,
+    target: path.join(directory, `${ARTIFACT_ID}.log`),
+    fact: path.join(directory, `.${ARTIFACT_ID}.log.truncated.json`),
+  };
+}
+
+function candidate() {
+  return {
+    projectId: 'prj_default',
+    runId: 'run_1',
+    attemptId: 'attempt_1',
+    logArtifactId: ARTIFACT_ID,
+    executorType: 'local_process',
+    finishedAtMs: 1,
+  };
+}
+
+function privateFile(filePath, content) {
+  fs.writeFileSync(filePath, content, { mode: 0o600 });
+  fs.chmodSync(filePath, 0o600);
+}
+
+function fact() {
+  return JSON.stringify({
+    schemaVersion: 1,
+    runId: 'run_1',
+    attemptId: 'attempt_1',
+    logArtifactId: ARTIFACT_ID,
+    maximumBytes: 64 * 1024,
+    quotaReached: true,
+    observedAtMs: 2,
+  });
+}
+
+test('deletes only the exact private log and truncation fact', async (t) => {
+  const value = fixture(t);
+  privateFile(value.target, 'hello');
+  privateFile(value.fact, fact());
+  privateFile(path.join(value.directory, 'unrelated'), 'keep');
+
+  const retired = await new LocalRunAttemptLogRetirementStore(
+    value.root,
+  ).retire(candidate());
+  assert.deepEqual(retired, {
+    disposition: 'deleted',
+    byteLength: 5,
+    truncation: {
+      truncated: true,
+      maximumBytes: 64 * 1024,
+      observedAtMs: 2,
+    },
+  });
+  assert.equal(fs.existsSync(value.target), false);
+  assert.equal(fs.existsSync(value.fact), false);
+  assert.equal(fs.existsSync(path.join(value.directory, 'unrelated')), true);
+});
+
+test('converges an unlink-before-tombstone crash and removes its orphan fact', async (t) => {
+  const value = fixture(t);
+  privateFile(value.fact, fact());
+  const retired = await new LocalRunAttemptLogRetirementStore(
+    value.root,
+  ).retire(candidate());
+  assert.equal(retired.disposition, 'already_absent');
+  assert.equal(retired.byteLength, 0);
+  assert.equal(retired.truncation.truncated, true);
+  assert.equal(fs.existsSync(value.fact), false);
+});
+
+test('fails closed for links, unsafe modes and fact identity drift', async (t) => {
+  const cases = [];
+
+  const hardLink = fixture(t);
+  privateFile(hardLink.target, 'hello');
+  fs.linkSync(hardLink.target, path.join(hardLink.directory, 'second-link'));
+  cases.push(hardLink);
+
+  const unsafeMode = fixture(t);
+  privateFile(unsafeMode.target, 'hello');
+  fs.chmodSync(unsafeMode.target, 0o644);
+  cases.push(unsafeMode);
+
+  const drift = fixture(t);
+  privateFile(drift.target, 'hello');
+  privateFile(drift.fact, fact().replace('"attempt_1"', '"attempt_other"'));
+  cases.push(drift);
+
+  for (const value of cases) {
+    await assert.rejects(
+      new LocalRunAttemptLogRetirementStore(value.root).retire(candidate()),
+      LocalRunAttemptLogRetirementError,
+    );
+    assert.equal(fs.existsSync(value.target), true);
+  }
+});
+
+test('capacity probe uses the nearest existing parent without creating roots', async (t) => {
+  const value = fixture(t);
+  const missing = path.join(value.parent, 'future', 'artifacts');
+  const snapshot = await new LocalRunAttemptLogCapacityProbe(missing).inspect();
+  assert.equal(snapshot.totalBytes > 0n, true);
+  assert.equal(snapshot.availableBytes >= 0n, true);
+  assert.equal(snapshot.availableBytes <= snapshot.totalBytes, true);
+  assert.equal(fs.existsSync(missing), false);
+});