feat(ql3): complete cluster log retention lifecycle

packages/ql3-cluster-control/src/application-runtime/application.ts

@@ -13,6 +13,7 @@ import {
   type ClusterControlAssemblyInput,
   type ClusterControlRecoveryRuntimeOptions,
   type ClusterRunCancellationConvergenceRuntimeOptions,
+  type ClusterRunAttemptLogRetentionRuntimeOptions,
   type ClusterSchedulerRuntimeOptions,
   type ClusterWorkerRuntimeDependencies,
 } from './clusterControlRuntime';
@@ -39,6 +40,7 @@ export interface ClusterControlApplicationOptions {
   readonly recovery?: ClusterControlRecoveryRuntimeOptions;
   readonly scheduler?: ClusterSchedulerRuntimeOptions;
   readonly cancellationConvergence?: ClusterRunCancellationConvergenceRuntimeOptions;
+  readonly logRetention?: ClusterRunAttemptLogRetentionRuntimeOptions;
   readonly workerRuntime?: ClusterWorkerRuntimeDependencies;
   readonly openDatabase: OpenPostgresDatabase;
   readonly availability: ClusterControlAvailabilitySource;
@@ -88,6 +90,9 @@ function inactiveBootstrap(
     ...(options.cancellationConvergence === undefined
       ? {}
       : { cancellationConvergence: options.cancellationConvergence }),
+    ...(options.logRetention === undefined
+      ? {}
+      : { logRetention: options.logRetention }),
     ...(options.workerRuntime === undefined
       ? {}
       : { workerRuntime: options.workerRuntime }),
@@ -163,6 +168,9 @@ export async function startClusterControlApplication(
       ...(options.cancellationConvergence === undefined
         ? {}
         : { cancellationConvergence: options.cancellationConvergence }),
+      ...(options.logRetention === undefined
+        ? {}
+        : { logRetention: options.logRetention }),
       ...(options.workerRuntime === undefined
         ? {}
         : { workerRuntime: options.workerRuntime }),

packages/ql3-cluster-control/src/application-runtime/clusterControlRuntime.ts

@@ -30,6 +30,17 @@ import {
   type ClusterRunCancellationConvergenceCycleResult,
 } from '@qinglong/runtime-core';
 import type { ClusterRunCancellationRepository } from '@qinglong/runtime-core/cluster-run-cancellation';
+import {
+  MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_CLAIMS,
+  MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_LEASE_MS,
+  MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_RETRY_DELAY_MS,
+  MIN_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_LEASE_MS,
+} from '@qinglong/runtime-core/cluster-run-attempt-log-retention';
+import {
+  MAX_RUN_ATTEMPT_LOG_RETENTION_MS,
+  MIN_RUN_ATTEMPT_LOG_RETENTION_MS,
+  type RunAttemptLogRetentionStateReader,
+} from '@qinglong/runtime-core/run-attempt-log-retention';
 import type { ProjectRunListReader } from '@qinglong/runtime-core/project-run-list';
 import type { ClusterScheduleStore } from '@qinglong/runtime-core/cluster-scheduler';
 import type { TaskDefinitionSource } from '@qinglong/runtime-core/task-definition';
@@ -60,6 +71,7 @@ import {
   PostgresSecurityAuditRepository,
   PostgresRunRepository,
   PostgresWorkerExecutionAttestationRepository,
+  PostgresRunAttemptLogRetentionClaimRepository,
   PostgresTaskDefinitionSource,
   PostgresTaskExecutionRevisionSource,
   PostgresTriggerSource,
@@ -104,6 +116,12 @@ import {
 import { ClusterWorkflowSchedulerCoordinator } from '../scheduling/workflowScheduler';
 import { ClusterRuntimeSchedulerCoordinator } from '../scheduling/runtimeScheduler';
 import { ClusterRunCancellationConvergenceLifecycle } from '../run/runCancellationLifecycle';
+import {
+  ClusterRunAttemptLogRetentionCoordinator,
+  ClusterRunAttemptLogRetentionLifecycle,
+  type ClusterRunAttemptLogRetirementStore,
+  type ClusterRunAttemptLogRetentionCycleSummary,
+} from '../run/runAttemptLogRetentionLifecycle';
 import type { TaskStartRepository } from '@qinglong/runtime-core/task-start';
 import {
   createClusterWorkerRuntimePort,
@@ -131,6 +149,7 @@ export interface ClusterControlAssemblyInput {
   readonly evidence: ClusterControlReadinessEvidence;
   readonly policies: ProjectPolicyRepository;
   readonly runs: RunRepository & ProjectRunListReader;
+  readonly runAttemptLogRetention: RunAttemptLogRetentionStateReader;
   readonly runCancellation: ClusterRunCancellationRepository;
   readonly taskStart: TaskStartRepository;
   readonly taskDefinitions: TaskDefinitionSource;
@@ -178,6 +197,24 @@ export interface ClusterRunCancellationConvergenceRuntimeOptions {
   ) => void | Promise<void>;
 }
 
+export interface ClusterRunAttemptLogRetentionRuntimeOptions {
+  readonly store: ClusterRunAttemptLogRetirementStore;
+  readonly ownerId?: string;
+  readonly retentionMs?: number;
+  readonly claimLimit?: number;
+  readonly leaseMs?: number;
+  readonly maximumCycleMs?: number;
+  readonly retryBaseMs?: number;
+  readonly retryMaximumMs?: number;
+  readonly maximumFailures?: number;
+  readonly intervalMs?: number;
+  readonly stopTimeoutMs?: number;
+  readonly onDiagnostic?: (
+    error: unknown,
+    summary?: Readonly<ClusterRunAttemptLogRetentionCycleSummary>,
+  ) => void | Promise<void>;
+}
+
 export interface ClusterControlBootstrapOptions {
   readonly enabled?: boolean;
   readonly profile: DeploymentProfile;
@@ -185,6 +222,7 @@ export interface ClusterControlBootstrapOptions {
   readonly recovery?: ClusterControlRecoveryRuntimeOptions;
   readonly scheduler?: ClusterSchedulerRuntimeOptions;
   readonly cancellationConvergence?: ClusterRunCancellationConvergenceRuntimeOptions;
+  readonly logRetention?: ClusterRunAttemptLogRetentionRuntimeOptions;
   readonly workerRuntime?: ClusterWorkerRuntimeDependencies;
   readonly openDatabase: OpenPostgresDatabase;
   readonly create: (
@@ -223,6 +261,21 @@ interface PreparedCancellationConvergenceRuntime {
   readonly onDiagnostic?: ClusterRunCancellationConvergenceRuntimeOptions['onDiagnostic'];
 }
 
+interface PreparedLogRetentionRuntime {
+  readonly store: ClusterRunAttemptLogRetirementStore;
+  readonly ownerId: string;
+  readonly retentionMs: number;
+  readonly claimLimit: number;
+  readonly leaseMs: number;
+  readonly maximumCycleMs: number;
+  readonly retryBaseMs: number;
+  readonly retryMaximumMs: number;
+  readonly maximumFailures: number;
+  readonly intervalMs: number;
+  readonly stopTimeoutMs: number;
+  readonly onDiagnostic?: ClusterRunAttemptLogRetentionRuntimeOptions['onDiagnostic'];
+}
+
 function boundedInteger(
   name: string,
   value: number | undefined,
@@ -439,6 +492,116 @@ function prepareCancellationConvergenceRuntime(
   });
 }
 
+function prepareLogRetentionRuntime(
+  options: ClusterRunAttemptLogRetentionRuntimeOptions | undefined,
+  fallbackOwnerId: string,
+): PreparedLogRetentionRuntime | undefined {
+  if (options === undefined) return undefined;
+  const allowedKeys = new Set([
+    'claimLimit',
+    'intervalMs',
+    'leaseMs',
+    'maximumCycleMs',
+    'maximumFailures',
+    'onDiagnostic',
+    'ownerId',
+    'retentionMs',
+    'retryBaseMs',
+    'retryMaximumMs',
+    'stopTimeoutMs',
+    'store',
+  ]);
+  if (
+    !options ||
+    typeof options !== 'object' ||
+    Array.isArray(options) ||
+    Object.keys(options).some((key) => !allowedKeys.has(key)) ||
+    typeof options.store?.retire !== 'function' ||
+    (options.onDiagnostic !== undefined &&
+      typeof options.onDiagnostic !== 'function')
+  ) {
+    throw new TypeError(
+      'Cluster Run Attempt log retention configuration is invalid',
+    );
+  }
+  const ownerId = options.ownerId ?? fallbackOwnerId;
+  if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(ownerId)) {
+    throw new TypeError('Cluster Run Attempt log retention ownerId is invalid');
+  }
+  const leaseMs = boundedInteger(
+    'Cluster Run Attempt log retention lease',
+    options.leaseMs,
+    30_000,
+    MIN_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_LEASE_MS,
+    MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_LEASE_MS,
+  );
+  const retryBaseMs = boundedInteger(
+    'Cluster Run Attempt log retention retry base',
+    options.retryBaseMs,
+    5_000,
+    0,
+    MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_RETRY_DELAY_MS,
+  );
+  return Object.freeze({
+    store: options.store,
+    ownerId,
+    retentionMs: boundedInteger(
+      'Cluster Run Attempt log retention duration',
+      options.retentionMs,
+      30 * 24 * 60 * 60_000,
+      MIN_RUN_ATTEMPT_LOG_RETENTION_MS,
+      MAX_RUN_ATTEMPT_LOG_RETENTION_MS,
+    ),
+    claimLimit: boundedInteger(
+      'Cluster Run Attempt log retention claim limit',
+      options.claimLimit,
+      4,
+      1,
+      MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_CLAIMS,
+    ),
+    leaseMs,
+    maximumCycleMs: boundedInteger(
+      'Cluster Run Attempt log retention cycle budget',
+      options.maximumCycleMs,
+      10_000,
+      100,
+      leaseMs - 500,
+    ),
+    retryBaseMs,
+    retryMaximumMs: boundedInteger(
+      'Cluster Run Attempt log retention retry maximum',
+      options.retryMaximumMs,
+      60 * 60_000,
+      retryBaseMs,
+      MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_RETRY_DELAY_MS,
+    ),
+    maximumFailures: boundedInteger(
+      'Cluster Run Attempt log retention failure limit',
+      options.maximumFailures,
+      8,
+      1,
+      32,
+    ),
+    intervalMs: boundedInteger(
+      'Cluster Run Attempt log retention interval',
+      options.intervalMs,
+      60_000,
+      1_000,
+      24 * 60 * 60_000,
+    ),
+    stopTimeoutMs: boundedInteger(
+      'Cluster Run Attempt log retention stop timeout',
+      options.stopTimeoutMs,
+      10_000,
+      100,
+      30_000,
+    ),
+    ...(options.onDiagnostic === undefined
+      ? {}
+      : { onDiagnostic: options.onDiagnostic }),
+  });
+}
+
 function readinessEvidence(
   report: PostgresSchemaReadinessReport,
 ): ClusterControlReadinessEvidence {
@@ -463,6 +626,7 @@ export async function bootstrapClusterControlRuntime(
   let cancellationConvergenceRuntime:
     | PreparedCancellationConvergenceRuntime
     | undefined;
+  let logRetentionRuntime: PreparedLogRetentionRuntime | undefined;
   let recoveryRegistry: ClusterControlRecoveryEvidenceRegistry | undefined;
   if ((options.enabled ?? false) && options.profile === 'cluster-control') {
     assertClusterControlApiCredentialPepper(options.apiCredentialPepper ?? '');
@@ -474,6 +638,10 @@ export async function bootstrapClusterControlRuntime(
     cancellationConvergenceRuntime = prepareCancellationConvergenceRuntime(
       options.cancellationConvergence,
     );
+    logRetentionRuntime = prepareLogRetentionRuntime(
+      options.logRetention,
+      recoveryRuntime.ownerId,
+    );
   }
   let database: PostgresDatabaseResource | undefined;
   let closePromise: Promise<void> | undefined;
@@ -576,6 +744,8 @@ export async function bootstrapClusterControlRuntime(
         );
         const schedules = new PostgresClusterScheduleRepository(database.pool);
         const runs = new PostgresRunRepository(database.pool);
+        const runAttemptLogRetention =
+          new PostgresRunAttemptLogRetentionClaimRepository(database.pool);
         const trustedToolStorage: ClusterTrustedToolStorage = Object.freeze({
           invocationArtifacts: new PostgresToolInvocationArtifactRepository(
             database.pool,
@@ -653,6 +823,32 @@ export async function bootstrapClusterControlRuntime(
                   }),
             },
           );
+        const logRetentionLifecycle =
+          logRetentionRuntime === undefined
+            ? undefined
+            : new ClusterRunAttemptLogRetentionLifecycle(
+                new ClusterRunAttemptLogRetentionCoordinator(
+                  runAttemptLogRetention,
+                  logRetentionRuntime.store,
+                  {
+                    ownerId: logRetentionRuntime.ownerId,
+                    retentionMs: logRetentionRuntime.retentionMs,
+                    claimLimit: logRetentionRuntime.claimLimit,
+                    leaseMs: logRetentionRuntime.leaseMs,
+                    maximumCycleMs: logRetentionRuntime.maximumCycleMs,
+                    retryBaseMs: logRetentionRuntime.retryBaseMs,
+                    retryMaximumMs: logRetentionRuntime.retryMaximumMs,
+                    maximumFailures: logRetentionRuntime.maximumFailures,
+                  },
+                ),
+                {
+                  intervalMs: logRetentionRuntime.intervalMs,
+                  stopTimeoutMs: logRetentionRuntime.stopTimeoutMs,
+                  ...(logRetentionRuntime.onDiagnostic === undefined
+                    ? {}
+                    : { onDiagnostic: logRetentionRuntime.onDiagnostic }),
+                },
+              );
         const runCancellation = new PostgresClusterRunCancellationRepository(
           database.pool,
         );
@@ -665,6 +861,7 @@ export async function bootstrapClusterControlRuntime(
           ),
           policies: new PostgresProjectPolicyRepository(database.pool),
           runs,
+          runAttemptLogRetention,
           runCancellation,
           taskStart,
           taskDefinitions: new PostgresTaskDefinitionSource(database.pool),
@@ -739,6 +936,7 @@ export async function bootstrapClusterControlRuntime(
             if (!(await application.startLifecycles())) return false;
             schedulerLifecycle.start();
             cancellationConvergenceLifecycle.start();
+            logRetentionLifecycle?.start();
             return true;
           },
           installAdmission: () => application.installAdmission(),
@@ -746,14 +944,21 @@ export async function bootstrapClusterControlRuntime(
             recoveryRegistry?.dispose();
             let schedulerStatus: 'stopped' | 'timed_out' = 'stopped';
             let cancellationStatus: 'stopped' | 'timed_out' = 'stopped';
+            let logRetentionStatus: 'stopped' | 'timed_out' = 'stopped';
             let applicationStatus: ClusterControlStopResult = 'stopped';
             let primaryError: unknown;
+            try {
+              logRetentionStatus =
+                (await logRetentionLifecycle?.stopAndDrain()) ?? 'stopped';
+            } catch (error) {
+              primaryError = error;
+            }
             try {
               cancellationStatus = (
                 await cancellationConvergenceLifecycle.stopAndDrain()
               ).status;
             } catch (error) {
-              primaryError = error;
+              primaryError ??= error;
             }
             try {
               schedulerStatus = (await schedulerLifecycle.stopAndDrain())
@@ -768,6 +973,7 @@ export async function bootstrapClusterControlRuntime(
             }
             if (primaryError) throw primaryError;
             return cancellationStatus === 'timed_out' ||
+              logRetentionStatus === 'timed_out' ||
               schedulerStatus === 'timed_out' ||
               applicationStatus === 'timed_out'
               ? 'timed_out'

packages/ql3-cluster-control/src/application-runtime/productionApplication.ts

@@ -200,6 +200,7 @@ export function createProductionClusterControlApplicationStack(
     createClusterControlRunAttemptLogReadRoute(
       input.runs,
       input.workerRuntime?.runAttemptLogRead,
+      input.runAttemptLogRetention,
     ),
     createClusterControlRunCancellationRoute(
       input.runCancellation,

packages/ql3-cluster-control/src/artifact/s3ArtifactStore.ts

@@ -19,6 +19,12 @@ import {
   type RunAttemptLogReadIdentity,
   type RunAttemptLogReadRange,
 } from '@qinglong/runtime-core/run-attempt-log-read';
+import {
+  normalizeRunAttemptLogRetentionCandidate,
+  type RunAttemptLogRetentionCandidate,
+  type RunAttemptLogRetirementStore,
+  type RunAttemptLogRetirementStoreResult,
+} from '@qinglong/runtime-core/run-attempt-log-retention';
 import {
   MAX_REMOTE_WORKER_ARTIFACT_BYTES,
   REMOTE_WORKER_ARTIFACT_CONTENT_TYPE,
@@ -124,6 +130,7 @@ type ArtifactAuthority = Readonly<{
 type StoredArtifactHead = Readonly<{
   receipt: Readonly<RemoteWorkerArtifactReceipt>;
   eTag?: string;
+  versionId?: string;
 }>;
 
 type NormalizedStorageCommand = ArtifactAuthority &
@@ -484,6 +491,18 @@ function canonicalETag(value: unknown): string {
   return value;
 }
 
+function canonicalVersionId(value: unknown): string {
+  if (
+    typeof value !== 'string' ||
+    value.length < 1 ||
+    Buffer.byteLength(value, 'utf8') > 1024 ||
+    /[\u0000-\u001f\u007f]/.test(value)
+  ) {
+    throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
+  }
+  return value;
+}
+
 function assertRangeMetadata(
   authority: ArtifactAuthority,
   receipt: Readonly<RemoteWorkerArtifactReceipt>,
@@ -580,6 +599,20 @@ function isNotFound(error: unknown): boolean {
   );
 }
 
+function isPreconditionFailed(error: unknown): boolean {
+  if (!error || typeof error !== 'object') return false;
+  const value = error as {
+    name?: unknown;
+    Code?: unknown;
+    $metadata?: { httpStatusCode?: unknown };
+  };
+  return (
+    value.name === 'PreconditionFailed' ||
+    value.Code === 'PreconditionFailed' ||
+    value.$metadata?.httpStatusCode === 412
+  );
+}
+
 function requestOptions(
   signal?: AbortSignal,
 ): { abortSignal: AbortSignal } | undefined {
@@ -655,10 +688,11 @@ class ArtifactContentDigest {
 /**
  * Shared immutable S3 adapter. A unique temporary upload is checksummed first,
  * then promoted by one destination-conditional server-side copy. Permanent
- * objects are never overwritten or deleted by this adapter.
+ * objects are never overwritten and are retired only after an identity-checked
+ * HEAD followed by a VersionId- or ETag-fenced delete.
  */
 export class S3ClusterRemoteWorkerArtifactStore
-  implements ClusterRemoteWorkerArtifactStore
+  implements ClusterRemoteWorkerArtifactStore, RunAttemptLogRetirementStore
 {
   private readonly options: PreparedOptions;
 
@@ -751,6 +785,75 @@ export class S3ClusterRemoteWorkerArtifactStore
     }
   }
 
+  async retire(
+    rawCandidate: Readonly<RunAttemptLogRetentionCandidate>,
+    signal?: AbortSignal,
+  ): Promise<Readonly<RunAttemptLogRetirementStoreResult>> {
+    const candidate = normalizeRunAttemptLogRetentionCandidate(rawCandidate);
+    if (candidate.executorType !== 'remote_worker') {
+      throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
+    }
+    const authority = Object.freeze({
+      projectId: candidate.projectId,
+      runId: candidate.runId,
+      attemptId: candidate.attemptId,
+      logArtifactId: candidate.logArtifactId,
+    });
+    const stored = await this.head(authority, signal);
+    if (!stored) {
+      return Object.freeze({
+        disposition: 'already_absent' as const,
+        byteLength: 0,
+        truncation: Object.freeze({ truncated: 'unknown' as const }),
+      });
+    }
+    const versionId =
+      stored.versionId === undefined
+        ? undefined
+        : canonicalVersionId(stored.versionId);
+    const eTag =
+      versionId === undefined ? canonicalETag(stored.eTag) : undefined;
+    try {
+      await this.options.client.send(
+        new DeleteObjectCommand({
+          Bucket: this.options.bucket,
+          Key: finalObjectKey(this.options.prefix, authority),
+          ...(versionId === undefined
+            ? { IfMatch: eTag }
+            : { VersionId: versionId }),
+          ...(this.options.expectedBucketOwner === undefined
+            ? {}
+            : { ExpectedBucketOwner: this.options.expectedBucketOwner }),
+        }),
+        requestOptions(signal),
+      );
+    } catch (error) {
+      if (isNotFound(error)) {
+        return Object.freeze({
+          disposition: 'already_absent' as const,
+          byteLength: 0,
+          truncation: Object.freeze({ truncated: 'unknown' as const }),
+        });
+      }
+      if (isPreconditionFailed(error)) {
+        throw new S3ClusterRemoteWorkerArtifactStoreError(
+          'integrity_mismatch',
+          { cause: error },
+        );
+      }
+      throw new S3ClusterRemoteWorkerArtifactStoreError('unavailable', {
+        cause: error,
+      });
+    }
+    return Object.freeze({
+      disposition: 'deleted' as const,
+      byteLength: stored.receipt.byteLength,
+      truncation: Object.freeze({
+        truncated: stored.receipt.truncated ?? ('unknown' as const),
+      }),
+    });
+  }
+
   private async head(
     authority: ArtifactAuthority,
     signal?: AbortSignal,
@@ -771,6 +874,9 @@ export class S3ClusterRemoteWorkerArtifactStore
       return Object.freeze({
         receipt: parseStoredReceipt(authority, output),
         ...(output.ETag === undefined ? {} : { eTag: output.ETag }),
+        ...(output.VersionId === undefined
+          ? {}
+          : { versionId: output.VersionId }),
       });
     } catch (error) {
       if (isNotFound(error)) return undefined;
@@ -811,7 +917,9 @@ export class S3ClusterRemoteWorkerArtifactStore
       this.options.createTemporaryId,
     );
     const temporaryOwner = temporaryOwnershipDigest();
-    let temporaryOwned = false;
+    let temporaryDeleteAuthority:
+      | Readonly<{ readonly eTag: string; readonly versionId?: string }>
+      | undefined;
     let result: Readonly<RemoteWorkerArtifactReceipt> | undefined;
     let primaryError: unknown;
     try {
@@ -837,21 +945,19 @@ export class S3ClusterRemoteWorkerArtifactStore
           }),
           requestOptions(signal),
         );
-        temporaryOwned = true;
       } catch (error) {
         if (!digest.isComplete()) throw error;
       } finally {
         body.destroy();
       }
       const sha256 = digest.digest();
-      await this.assertTemporaryObject(
+      temporaryDeleteAuthority = await this.assertTemporaryObject(
         temporaryKey,
         temporaryOwner,
         command.byteLength,
         sha256,
         signal,
       );
-      temporaryOwned = true;
 
       let copied = false;
       try {
@@ -898,12 +1004,15 @@ export class S3ClusterRemoteWorkerArtifactStore
       primaryError = error;
     }
 
-    if (temporaryOwned) {
+    if (temporaryDeleteAuthority !== undefined) {
       try {
         await this.options.client.send(
           new DeleteObjectCommand({
             Bucket: this.options.bucket,
             Key: temporaryKey,
+            ...(temporaryDeleteAuthority.versionId === undefined
+              ? { IfMatch: temporaryDeleteAuthority.eTag }
+              : { VersionId: temporaryDeleteAuthority.versionId }),
             ...(this.options.expectedBucketOwner === undefined
               ? {}
               : { ExpectedBucketOwner: this.options.expectedBucketOwner }),
@@ -937,7 +1046,7 @@ export class S3ClusterRemoteWorkerArtifactStore
     byteLength: number,
     sha256: string,
     signal?: AbortSignal,
-  ): Promise<void> {
+  ): Promise<Readonly<{ readonly eTag: string; readonly versionId?: string }>> {
     let output;
     try {
       output = await this.options.client.send(
@@ -965,5 +1074,14 @@ export class S3ClusterRemoteWorkerArtifactStore
     ) {
       throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
     }
+    const eTag = canonicalETag(output.ETag);
+    const versionId =
+      output.VersionId === undefined
+        ? undefined
+        : canonicalVersionId(output.VersionId);
+    return Object.freeze({
+      eTag,
+      ...(versionId === undefined ? {} : { versionId }),
+    });
   }
 }

packages/ql3-cluster-control/src/artifact/workerArtifactBinding.ts

@@ -5,9 +5,11 @@ import {
 } from './s3ArtifactStore';
 import type { ClusterRemoteWorkerArtifactStore } from '../remote-execution/remoteWorkerCompletionService';
 import type { ClusterWorkerArtifactS3Config } from '../worker-ingress/workerIngressConfig';
+import type { ClusterRunAttemptLogRetirementStore } from '../run/runAttemptLogRetentionLifecycle';
 
 export interface ClusterWorkerArtifactBinding {
-  readonly store: ClusterRemoteWorkerArtifactStore;
+  readonly store: ClusterRemoteWorkerArtifactStore &
+    ClusterRunAttemptLogRetirementStore;
   close(): Promise<void>;
 }
 
@@ -19,9 +21,7 @@ export function createClusterWorkerArtifactBinding(
   }
   const client = createS3ClusterRemoteWorkerArtifactClient({
     region: config.region,
-    ...(config.endpoint === undefined
-      ? {}
-      : { endpoint: config.endpoint }),
+    ...(config.endpoint === undefined ? {} : { endpoint: config.endpoint }),
     forcePathStyle: config.forcePathStyle,
   });
   const store = new S3ClusterRemoteWorkerArtifactStore({

packages/ql3-cluster-control/src/production-process/config.ts

@@ -12,6 +12,16 @@ import {
 } from '@qinglong/cluster-postgres/runtime';
 import { ClusterControlAvailabilityFence } from '../database/availability';
 import type { ClusterControlHttpSurfaceOptions } from '../transport/httpSurface';
+import {
+  MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_CLAIMS,
+  MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_LEASE_MS,
+  MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_RETRY_DELAY_MS,
+  MIN_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_LEASE_MS,
+} from '@qinglong/runtime-core/cluster-run-attempt-log-retention';
+import {
+  MAX_RUN_ATTEMPT_LOG_RETENTION_MS,
+  MIN_RUN_ATTEMPT_LOG_RETENTION_MS,
+} from '@qinglong/runtime-core/run-attempt-log-retention';
 
 export type ClusterControlEnvironment = Readonly<
   Record<string, string | undefined>
@@ -33,6 +43,20 @@ export interface EnabledClusterControlConfig {
   readonly security: Readonly<{
     apiCredentialPepper: string;
   }>;
+  readonly logRetention:
+    | Readonly<{ readonly enabled: false }>
+    | Readonly<{
+        readonly enabled: true;
+        readonly retentionMs: number;
+        readonly claimLimit: number;
+        readonly leaseMs: number;
+        readonly maximumCycleMs: number;
+        readonly retryBaseMs: number;
+        readonly retryMaximumMs: number;
+        readonly maximumFailures: number;
+        readonly intervalMs: number;
+        readonly stopTimeoutMs: number;
+      }>;
 }
 
 export type ClusterControlConfig =
@@ -222,6 +246,82 @@ function apiCredentialPepper(environment: ClusterControlEnvironment): string {
   return value;
 }
 
+function logRetentionConfig(
+  environment: ClusterControlEnvironment,
+): EnabledClusterControlConfig['logRetention'] {
+  if (!booleanValue(environment, 'QL3_CLUSTER_LOG_RETENTION_ENABLED', true)) {
+    return Object.freeze({ enabled: false as const });
+  }
+  const leaseMs = integerValue(
+    environment,
+    'QL3_CLUSTER_LOG_RETENTION_LEASE_MS',
+    30_000,
+    MIN_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_LEASE_MS,
+    MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_LEASE_MS,
+  );
+  const retryBaseMs = integerValue(
+    environment,
+    'QL3_CLUSTER_LOG_RETENTION_RETRY_BASE_MS',
+    5_000,
+    0,
+    MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_RETRY_DELAY_MS,
+  );
+  return Object.freeze({
+    enabled: true as const,
+    retentionMs: integerValue(
+      environment,
+      'QL3_CLUSTER_LOG_RETENTION_MS',
+      30 * 24 * 60 * 60_000,
+      MIN_RUN_ATTEMPT_LOG_RETENTION_MS,
+      MAX_RUN_ATTEMPT_LOG_RETENTION_MS,
+    ),
+    claimLimit: integerValue(
+      environment,
+      'QL3_CLUSTER_LOG_RETENTION_CLAIM_LIMIT',
+      4,
+      1,
+      MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_CLAIMS,
+    ),
+    leaseMs,
+    maximumCycleMs: integerValue(
+      environment,
+      'QL3_CLUSTER_LOG_RETENTION_CYCLE_BUDGET_MS',
+      10_000,
+      100,
+      leaseMs - 500,
+    ),
+    retryBaseMs,
+    retryMaximumMs: integerValue(
+      environment,
+      'QL3_CLUSTER_LOG_RETENTION_RETRY_MAX_MS',
+      60 * 60_000,
+      retryBaseMs,
+      MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_RETRY_DELAY_MS,
+    ),
+    maximumFailures: integerValue(
+      environment,
+      'QL3_CLUSTER_LOG_RETENTION_MAX_FAILURES',
+      8,
+      1,
+      32,
+    ),
+    intervalMs: integerValue(
+      environment,
+      'QL3_CLUSTER_LOG_RETENTION_INTERVAL_MS',
+      60_000,
+      1_000,
+      24 * 60 * 60_000,
+    ),
+    stopTimeoutMs: integerValue(
+      environment,
+      'QL3_CLUSTER_LOG_RETENTION_STOP_TIMEOUT_MS',
+      10_000,
+      100,
+      30_000,
+    ),
+  });
+}
+
 /**
  * Parses the profile gate before reading PostgreSQL configuration. A disabled
  * cluster-control therefore does not touch its runtime credential source.
@@ -345,6 +445,7 @@ export function loadClusterControlConfig(
     security: Object.freeze({
       apiCredentialPepper: apiCredentialPepper(environment),
     }),
+    logRetention: logRetentionConfig(environment),
   };
   return Object.freeze(config);
 }

packages/ql3-cluster-control/src/production-process/processApplication.ts

@@ -38,6 +38,7 @@ export interface ClusterControlProcessEvent {
     scope:
       | 'scheduler'
       | 'cancellation-convergence'
+      | 'log-retention'
       | 'database'
       | 'worker-ingress';
     name: string;
@@ -78,10 +79,7 @@ export class ClusterControlProcessError extends Error {
     | 'QL3_CLUSTER_CONTROL_PROCESS_CONFIG_INVALID'
     | 'QL3_CLUSTER_CONTROL_PROCESS_DISABLED';
 
-  constructor(
-    code: ClusterControlProcessError['code'],
-    message: string,
-  ) {
+  constructor(code: ClusterControlProcessError['code'], message: string) {
     super(message);
     this.name = 'ClusterControlProcessError';
     this.code = code;
@@ -103,10 +101,7 @@ function processConfiguration(environment: ClusterControlEnvironment): {
     );
   }
   const replicaId = environment.QL3_CLUSTER_REPLICA_ID;
-  if (
-    typeof replicaId !== 'string' ||
-    !REPLICA_ID_PATTERN.test(replicaId)
-  ) {
+  if (typeof replicaId !== 'string' || !REPLICA_ID_PATTERN.test(replicaId)) {
     throw new ClusterControlProcessError(
       'QL3_CLUSTER_CONTROL_PROCESS_CONFIG_INVALID',
       'QL3_CLUSTER_REPLICA_ID must be a stable safe identifier',
@@ -205,9 +200,11 @@ export async function runProductionClusterControlProcess(
   let resolveSignal:
     | ((signal: ClusterControlProcessSignal) => void)
     | undefined;
-  const requestedSignal = new Promise<ClusterControlProcessSignal>((resolve) => {
-    resolveSignal = resolve;
-  });
+  const requestedSignal = new Promise<ClusterControlProcessSignal>(
+    (resolve) => {
+      resolveSignal = resolve;
+    },
+  );
   let acceptedSignal = false;
   const unsubscribe = options.signals.subscribe((signal) => {
     if (acceptedSignal) return;
@@ -230,6 +227,14 @@ export async function runProductionClusterControlProcess(
         );
       }
       artifactBinding = await createBinding(workerIngress.artifact);
+      if (
+        config.logRetention.enabled &&
+        typeof artifactBinding?.store?.retire !== 'function'
+      ) {
+        throw new TypeError(
+          'Cluster Worker Artifact binding has no log retirement capability',
+        );
+      }
       if (
         workerIngress.secret !== undefined &&
         workerSecretProvider === undefined
@@ -274,15 +279,40 @@ export async function runProductionClusterControlProcess(
               event(replicaId, {
                 level: 'error',
                 event: 'runtime_diagnostic',
-                diagnostic: diagnosticFact(
-                  'cancellation-convergence',
-                  error,
-                ),
+                diagnostic: diagnosticFact('cancellation-convergence', error),
               }),
             ),
           ).catch(() => undefined);
         },
       },
+      ...(workerIngress !== undefined && config.logRetention.enabled
+        ? {
+            logRetention: {
+              store: artifactBinding!.store,
+              ownerId: replicaId,
+              retentionMs: config.logRetention.retentionMs,
+              claimLimit: config.logRetention.claimLimit,
+              leaseMs: config.logRetention.leaseMs,
+              maximumCycleMs: config.logRetention.maximumCycleMs,
+              retryBaseMs: config.logRetention.retryBaseMs,
+              retryMaximumMs: config.logRetention.retryMaximumMs,
+              maximumFailures: config.logRetention.maximumFailures,
+              intervalMs: config.logRetention.intervalMs,
+              stopTimeoutMs: config.logRetention.stopTimeoutMs,
+              onDiagnostic(error: unknown) {
+                void Promise.resolve(
+                  options.emit(
+                    event(replicaId, {
+                      level: 'error',
+                      event: 'runtime_diagnostic',
+                      diagnostic: diagnosticFact('log-retention', error),
+                    }),
+                  ),
+                ).catch(() => undefined);
+              },
+            },
+          }
+        : {}),
       ...(workerIngress === undefined
         ? {}
         : {
@@ -298,10 +328,7 @@ export async function runProductionClusterControlProcess(
                     event(replicaId, {
                       level: 'error',
                       event: 'runtime_diagnostic',
-                      diagnostic: diagnosticFact(
-                        'worker-ingress',
-                        error,
-                      ),
+                      diagnostic: diagnosticFact('worker-ingress', error),
                     }),
                   ),
                 ).catch(() => undefined);
@@ -395,10 +422,7 @@ export async function runProductionClusterControlProcess(
     unsubscribe();
     resolveSignal = undefined;
     let cleanupError: unknown;
-    if (
-      application?.status === 'active' &&
-      !applicationStopStarted
-    ) {
+    if (application?.status === 'active' && !applicationStopStarted) {
       try {
         applicationStopStarted = true;
         await application.stop();

packages/ql3-cluster-control/src/run/runAttemptLogRetentionLifecycle.ts

@@ -0,0 +1,490 @@
+// Run owns bounded multi-replica Cluster log retirement and one lifecycle timer.
+import {
+  MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_CLAIMS,
+  MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_LEASE_MS,
+  MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_RETRY_DELAY_MS,
+  MIN_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_LEASE_MS,
+  type ClusterRunAttemptLogRetentionClaim,
+  type ClusterRunAttemptLogRetentionClaimRepository,
+  type ClusterRunAttemptLogRetentionFailureCode,
+} from '@qinglong/runtime-core/cluster-run-attempt-log-retention';
+import {
+  createRunAttemptLogRetirementRecord,
+  MAX_RUN_ATTEMPT_LOG_RETENTION_MS,
+  MIN_RUN_ATTEMPT_LOG_RETENTION_MS,
+  type RunAttemptLogRetentionCandidate,
+  type RunAttemptLogRetirementStore,
+  type RunAttemptLogRetirementStoreResult,
+} from '@qinglong/runtime-core/run-attempt-log-retention';
+
+const OWNER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+const MIN_SETTLEMENT_BUDGET_MS = 500;
+
+export interface ClusterRunAttemptLogRetirementStore
+  extends RunAttemptLogRetirementStore {
+  retire(
+    candidate: Readonly<RunAttemptLogRetentionCandidate>,
+    signal?: AbortSignal,
+  ): Promise<Readonly<RunAttemptLogRetirementStoreResult>>;
+}
+
+export interface ClusterRunAttemptLogRetentionCoordinatorOptions {
+  readonly ownerId: string;
+  readonly retentionMs: number;
+  readonly claimLimit: number;
+  readonly leaseMs: number;
+  readonly maximumCycleMs: number;
+  readonly retryBaseMs: number;
+  readonly retryMaximumMs: number;
+  readonly maximumFailures: number;
+}
+
+export interface ClusterRunAttemptLogRetentionCycleEntry {
+  readonly attemptId: string;
+  readonly outcome:
+    | 'deleted'
+    | 'already_absent'
+    | 'retry'
+    | 'manual'
+    | 'fenced';
+}
+
+export interface ClusterRunAttemptLogRetentionCycleSummary {
+  readonly status: 'complete' | 'saturated' | 'budget_exhausted';
+  readonly claimed: number;
+  readonly attempted: number;
+  readonly retired: number;
+  readonly alreadyAbsent: number;
+  readonly retried: number;
+  readonly manual: number;
+  readonly fenced: number;
+  readonly hasMore: boolean;
+  readonly entries: readonly Readonly<ClusterRunAttemptLogRetentionCycleEntry>[];
+}
+
+function integer(
+  name: string,
+  value: number,
+  minimum: number,
+  maximum: number,
+): number {
+  if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
+    throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
+  }
+  return value;
+}
+
+function prepareOptions(
+  options: ClusterRunAttemptLogRetentionCoordinatorOptions,
+): Readonly<ClusterRunAttemptLogRetentionCoordinatorOptions> {
+  const allowed = new Set([
+    'claimLimit',
+    'leaseMs',
+    'maximumCycleMs',
+    'maximumFailures',
+    'ownerId',
+    'retentionMs',
+    'retryBaseMs',
+    'retryMaximumMs',
+  ]);
+  if (
+    !options ||
+    typeof options !== 'object' ||
+    Array.isArray(options) ||
+    Object.keys(options).some((key) => !allowed.has(key)) ||
+    !OWNER_PATTERN.test(options.ownerId)
+  ) {
+    throw new TypeError(
+      'Cluster Run Attempt log retention options are invalid',
+    );
+  }
+  const leaseMs = integer(
+    'Cluster Run Attempt log retention lease',
+    options.leaseMs,
+    MIN_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_LEASE_MS,
+    MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_LEASE_MS,
+  );
+  const maximumCycleMs = integer(
+    'Cluster Run Attempt log retention cycle budget',
+    options.maximumCycleMs,
+    100,
+    leaseMs - MIN_SETTLEMENT_BUDGET_MS,
+  );
+  const retryBaseMs = integer(
+    'Cluster Run Attempt log retention retry base',
+    options.retryBaseMs,
+    0,
+    MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_RETRY_DELAY_MS,
+  );
+  const retryMaximumMs = integer(
+    'Cluster Run Attempt log retention retry maximum',
+    options.retryMaximumMs,
+    retryBaseMs,
+    MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_RETRY_DELAY_MS,
+  );
+  return Object.freeze({
+    ownerId: options.ownerId,
+    retentionMs: integer(
+      'Cluster Run Attempt log retention duration',
+      options.retentionMs,
+      MIN_RUN_ATTEMPT_LOG_RETENTION_MS,
+      MAX_RUN_ATTEMPT_LOG_RETENTION_MS,
+    ),
+    claimLimit: integer(
+      'Cluster Run Attempt log retention claim limit',
+      options.claimLimit,
+      1,
+      MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_CLAIMS,
+    ),
+    leaseMs,
+    maximumCycleMs,
+    retryBaseMs,
+    retryMaximumMs,
+    maximumFailures: integer(
+      'Cluster Run Attempt log retention failure limit',
+      options.maximumFailures,
+      1,
+      32,
+    ),
+  });
+}
+
+function failureCode(error: unknown): ClusterRunAttemptLogRetentionFailureCode {
+  return (error as { readonly reason?: unknown })?.reason ===
+    'integrity_mismatch'
+    ? 'artifact_integrity_mismatch'
+    : 'artifact_unavailable';
+}
+
+function retryDelay(
+  failureCount: number,
+  options: Readonly<ClusterRunAttemptLogRetentionCoordinatorOptions>,
+): number {
+  const multiplier = 2 ** Math.min(failureCount, 30);
+  return Math.min(options.retryMaximumMs, options.retryBaseMs * multiplier);
+}
+
+function isAbortSignal(value: unknown): value is AbortSignal {
+  return (
+    !!value &&
+    typeof value === 'object' &&
+    typeof (value as AbortSignal).aborted === 'boolean' &&
+    typeof (value as AbortSignal).addEventListener === 'function' &&
+    typeof (value as AbortSignal).removeEventListener === 'function'
+  );
+}
+
+export class ClusterRunAttemptLogRetentionCoordinator {
+  private readonly options: Readonly<ClusterRunAttemptLogRetentionCoordinatorOptions>;
+
+  constructor(
+    private readonly repository: ClusterRunAttemptLogRetentionClaimRepository,
+    private readonly store: ClusterRunAttemptLogRetirementStore,
+    options: ClusterRunAttemptLogRetentionCoordinatorOptions,
+  ) {
+    if (
+      !repository ||
+      typeof repository.claim !== 'function' ||
+      typeof repository.settle !== 'function' ||
+      !store ||
+      typeof store.retire !== 'function'
+    ) {
+      throw new TypeError(
+        'Cluster Run Attempt log retention dependencies are invalid',
+      );
+    }
+    this.options = prepareOptions(options);
+  }
+
+  async runOnce(
+    externalSignal?: AbortSignal,
+  ): Promise<Readonly<ClusterRunAttemptLogRetentionCycleSummary>> {
+    if (externalSignal !== undefined && !isAbortSignal(externalSignal)) {
+      throw new TypeError(
+        'Cluster Run Attempt log retention signal is invalid',
+      );
+    }
+    if (externalSignal?.aborted) throw externalSignal.reason;
+    const controller = new AbortController();
+    const forwardAbort = () => controller.abort(externalSignal?.reason);
+    externalSignal?.addEventListener('abort', forwardAbort, { once: true });
+    const timeout = setTimeout(
+      () =>
+        controller.abort(
+          new Error('Cluster log retention cycle budget expired'),
+        ),
+      this.options.maximumCycleMs,
+    );
+    timeout.unref?.();
+    try {
+      const page = await this.repository.claim({
+        ownerId: this.options.ownerId,
+        retentionMs: this.options.retentionMs,
+        limit: this.options.claimLimit,
+        leaseMs: this.options.leaseMs,
+      });
+      const entries: ClusterRunAttemptLogRetentionCycleEntry[] = [];
+      let attempted = 0;
+      let retired = 0;
+      let alreadyAbsent = 0;
+      let retried = 0;
+      let manual = 0;
+      let fenced = 0;
+      for (const claim of page.claims) {
+        if (controller.signal.aborted) break;
+        attempted += 1;
+        let result: Readonly<RunAttemptLogRetirementStoreResult>;
+        try {
+          result = await this.store.retire(claim.candidate, controller.signal);
+        } catch (error) {
+          if (controller.signal.aborted) break;
+          const settled = await this.settleFailure(claim, failureCode(error));
+          if (settled === 'fenced') {
+            fenced += 1;
+            entries.push({
+              attemptId: claim.candidate.attemptId,
+              outcome: 'fenced',
+            });
+          } else if (claim.failureCount + 1 >= this.options.maximumFailures) {
+            manual += 1;
+            entries.push({
+              attemptId: claim.candidate.attemptId,
+              outcome: 'manual',
+            });
+          } else {
+            retried += 1;
+            entries.push({
+              attemptId: claim.candidate.attemptId,
+              outcome: 'retry',
+            });
+          }
+          continue;
+        }
+        if (controller.signal.aborted) break;
+        let record;
+        try {
+          record = createRunAttemptLogRetirementRecord({
+            ...claim.candidate,
+            eligibleAtMs: claim.eligibleAtMs,
+            retiredAtMs: claim.observedAtMs,
+            ...result,
+          });
+        } catch {
+          const settled = await this.settleFailure(
+            claim,
+            'retirement_record_unavailable',
+          );
+          if (settled === 'fenced') {
+            fenced += 1;
+            entries.push({
+              attemptId: claim.candidate.attemptId,
+              outcome: 'fenced',
+            });
+          } else if (claim.failureCount + 1 >= this.options.maximumFailures) {
+            manual += 1;
+            entries.push({
+              attemptId: claim.candidate.attemptId,
+              outcome: 'manual',
+            });
+          } else {
+            retried += 1;
+            entries.push({
+              attemptId: claim.candidate.attemptId,
+              outcome: 'retry',
+            });
+          }
+          continue;
+        }
+        const settled = await this.repository.settle(claim, {
+          status: 'retired',
+          record,
+        });
+        if (settled === 'fenced') {
+          fenced += 1;
+          entries.push({
+            attemptId: claim.candidate.attemptId,
+            outcome: 'fenced',
+          });
+          continue;
+        }
+        if (record.disposition === 'already_absent') alreadyAbsent += 1;
+        else retired += 1;
+        entries.push({
+          attemptId: claim.candidate.attemptId,
+          outcome: record.disposition,
+        });
+      }
+      const budgetExhausted = controller.signal.aborted;
+      return Object.freeze({
+        status: budgetExhausted
+          ? ('budget_exhausted' as const)
+          : page.hasMore
+          ? ('saturated' as const)
+          : ('complete' as const),
+        claimed: page.claims.length,
+        attempted,
+        retired,
+        alreadyAbsent,
+        retried,
+        manual,
+        fenced,
+        hasMore: page.hasMore,
+        entries: Object.freeze(entries.map((entry) => Object.freeze(entry))),
+      });
+    } finally {
+      clearTimeout(timeout);
+      externalSignal?.removeEventListener('abort', forwardAbort);
+    }
+  }
+
+  private settleFailure(
+    claim: Readonly<ClusterRunAttemptLogRetentionClaim>,
+    code: ClusterRunAttemptLogRetentionFailureCode,
+  ): Promise<'settled' | 'fenced'> {
+    if (claim.failureCount + 1 >= this.options.maximumFailures) {
+      return this.repository.settle(claim, {
+        status: 'manual',
+        failureCode: code,
+      });
+    }
+    return this.repository.settle(claim, {
+      status: 'retry',
+      delayMs: retryDelay(claim.failureCount, this.options),
+      failureCode: code,
+    });
+  }
+}
+
+export interface ClusterRunAttemptLogRetentionLifecycleOptions {
+  readonly intervalMs: number;
+  readonly stopTimeoutMs: number;
+  readonly onDiagnostic?: (
+    error: unknown,
+    summary?: Readonly<ClusterRunAttemptLogRetentionCycleSummary>,
+  ) => void | Promise<void>;
+}
+
+export class ClusterRunAttemptLogRetentionLifecycle {
+  private timer: NodeJS.Timeout | undefined;
+  private inFlight:
+    | Promise<Readonly<ClusterRunAttemptLogRetentionCycleSummary>>
+    | undefined;
+  private controller: AbortController | undefined;
+  private stopPromise: Promise<'stopped' | 'timed_out'> | undefined;
+  private running = false;
+  private stopping = false;
+
+  constructor(
+    private readonly coordinator: Pick<
+      ClusterRunAttemptLogRetentionCoordinator,
+      'runOnce'
+    >,
+    private readonly options: ClusterRunAttemptLogRetentionLifecycleOptions,
+  ) {
+    if (
+      !coordinator ||
+      typeof coordinator.runOnce !== 'function' ||
+      !options ||
+      typeof options !== 'object' ||
+      Array.isArray(options) ||
+      !Number.isSafeInteger(options.intervalMs) ||
+      options.intervalMs < 1_000 ||
+      options.intervalMs > 24 * 60 * 60_000 ||
+      !Number.isSafeInteger(options.stopTimeoutMs) ||
+      options.stopTimeoutMs < 100 ||
+      options.stopTimeoutMs > 30_000 ||
+      (options.onDiagnostic !== undefined &&
+        typeof options.onDiagnostic !== 'function')
+    ) {
+      throw new TypeError(
+        'Cluster Run Attempt log retention lifecycle options are invalid',
+      );
+    }
+  }
+
+  start(): 'started' {
+    if (!this.running && !this.stopping) {
+      this.running = true;
+      this.schedule();
+    }
+    return 'started';
+  }
+
+  runOnce(): Promise<Readonly<ClusterRunAttemptLogRetentionCycleSummary>> {
+    if (this.stopping) {
+      return Promise.reject(
+        new Error('Cluster Run Attempt log retention lifecycle is stopping'),
+      );
+    }
+    if (this.inFlight) return this.inFlight;
+    const controller = new AbortController();
+    this.controller = controller;
+    const work = this.coordinator.runOnce(controller.signal).finally(() => {
+      if (this.inFlight === work) {
+        this.inFlight = undefined;
+        this.controller = undefined;
+      }
+    });
+    this.inFlight = work;
+    return work;
+  }
+
+  stopAndDrain(): Promise<'stopped' | 'timed_out'> {
+    if (this.stopPromise) return this.stopPromise;
+    this.stopping = true;
+    this.running = false;
+    if (this.timer) clearTimeout(this.timer);
+    this.timer = undefined;
+    this.controller?.abort(
+      new Error('Cluster Run Attempt log retention lifecycle is stopping'),
+    );
+    this.stopPromise = (async () => {
+      const work = this.inFlight;
+      if (!work) return 'stopped' as const;
+      let timeout: NodeJS.Timeout | undefined;
+      try {
+        return await Promise.race([
+          work.then(
+            () => 'stopped' as const,
+            () => 'stopped' as const,
+          ),
+          new Promise<'timed_out'>((resolve) => {
+            timeout = setTimeout(
+              () => resolve('timed_out' as const),
+              this.options.stopTimeoutMs,
+            );
+            timeout.unref?.();
+          }),
+        ]);
+      } finally {
+        if (timeout) clearTimeout(timeout);
+      }
+    })();
+    return this.stopPromise;
+  }
+
+  private schedule(): void {
+    if (!this.running || this.timer) return;
+    this.timer = setTimeout(() => {
+      this.timer = undefined;
+      if (!this.running) return;
+      void this.runOnce()
+        .then((summary) => this.diagnostic(undefined, summary))
+        .catch((error) => this.diagnostic(error))
+        .finally(() => this.schedule());
+    }, this.options.intervalMs);
+    this.timer.unref?.();
+  }
+
+  private async diagnostic(
+    error: unknown,
+    summary?: Readonly<ClusterRunAttemptLogRetentionCycleSummary>,
+  ): Promise<void> {
+    if (this.stopping) return;
+    try {
+      await this.options.onDiagnostic?.(error, summary);
+    } catch {
+      // Diagnostics cannot own or stop retention.
+    }
+  }
+}

packages/ql3-cluster-control/test/bootstrap.test.cjs

@@ -388,6 +388,7 @@ function bootstrapOptions(events, overrides = {}) {
         authenticator,
         policies,
         runs,
+        runAttemptLogRetention,
         runCancellation,
         taskDefinitions,
         taskExecutionRevisions,
@@ -408,6 +409,7 @@ function bootstrapOptions(events, overrides = {}) {
       assert.equal(typeof authenticator.authenticate, 'function');
       assert.equal(typeof policies.resolve, 'function');
       assert.equal(typeof runs.transaction, 'function');
+      assert.equal(typeof runAttemptLogRetention.inspect, 'function');
       assert.equal(typeof runCancellation.requestUserCancellation, 'function');
       assert.equal(
         typeof taskDefinitions.findCurrentTaskDefinition,

packages/ql3-cluster-control/test/config.test.cjs

@@ -114,6 +114,18 @@ test('builds an exact runtime-only TLS-verified Pool configuration', async () =>
   assert.deepEqual(config.security, {
     apiCredentialPepper: BASE_ENV.QL3_API_CREDENTIAL_PEPPER,
   });
+  assert.deepEqual(config.logRetention, {
+    enabled: true,
+    retentionMs: 30 * 24 * 60 * 60_000,
+    claimLimit: 4,
+    leaseMs: 30_000,
+    maximumCycleMs: 10_000,
+    retryBaseMs: 5_000,
+    retryMaximumMs: 60 * 60_000,
+    maximumFailures: 8,
+    intervalMs: 60_000,
+    stopTimeoutMs: 10_000,
+  });
 
   const binding = createClusterControlDatabaseBinding(config);
   assert.equal(binding.availability.status, 'available');
@@ -192,6 +204,17 @@ test('rejects TLS query overrides, missing credentials and unbounded values', ()
     { ...BASE_ENV, QL3_CLUSTER_AUTH_RATE_GLOBAL: '1000001' },
     { ...BASE_ENV, QL3_CLUSTER_AUTH_RATE_MAX_PEERS: '65537' },
     { ...BASE_ENV, QL3_API_CREDENTIAL_PEPPER: 'weak' },
+    { ...BASE_ENV, QL3_CLUSTER_LOG_RETENTION_CLAIM_LIMIT: '17' },
+    {
+      ...BASE_ENV,
+      QL3_CLUSTER_LOG_RETENTION_LEASE_MS: '5000',
+      QL3_CLUSTER_LOG_RETENTION_CYCLE_BUDGET_MS: '4501',
+    },
+    {
+      ...BASE_ENV,
+      QL3_CLUSTER_LOG_RETENTION_RETRY_BASE_MS: '5000',
+      QL3_CLUSTER_LOG_RETENTION_RETRY_MAX_MS: '4999',
+    },
   ]) {
     assert.throws(
       () => loadClusterControlConfig(environment),
@@ -199,3 +222,37 @@ test('rejects TLS query overrides, missing credentials and unbounded values', ()
     );
   }
 });
+
+test('loads bounded Cluster log retention policy and permits explicit disable', () => {
+  const disabled = loadClusterControlConfig({
+    ...BASE_ENV,
+    QL3_CLUSTER_LOG_RETENTION_ENABLED: 'false',
+    QL3_CLUSTER_LOG_RETENTION_CLAIM_LIMIT: '999',
+  });
+  assert.deepEqual(disabled.logRetention, { enabled: false });
+
+  const configured = loadClusterControlConfig({
+    ...BASE_ENV,
+    QL3_CLUSTER_LOG_RETENTION_MS: '60000',
+    QL3_CLUSTER_LOG_RETENTION_CLAIM_LIMIT: '2',
+    QL3_CLUSTER_LOG_RETENTION_LEASE_MS: '5000',
+    QL3_CLUSTER_LOG_RETENTION_CYCLE_BUDGET_MS: '4000',
+    QL3_CLUSTER_LOG_RETENTION_RETRY_BASE_MS: '250',
+    QL3_CLUSTER_LOG_RETENTION_RETRY_MAX_MS: '1000',
+    QL3_CLUSTER_LOG_RETENTION_MAX_FAILURES: '3',
+    QL3_CLUSTER_LOG_RETENTION_INTERVAL_MS: '2000',
+    QL3_CLUSTER_LOG_RETENTION_STOP_TIMEOUT_MS: '500',
+  });
+  assert.deepEqual(configured.logRetention, {
+    enabled: true,
+    retentionMs: 60_000,
+    claimLimit: 2,
+    leaseMs: 5_000,
+    maximumCycleMs: 4_000,
+    retryBaseMs: 250,
+    retryMaximumMs: 1_000,
+    maximumFailures: 3,
+    intervalMs: 2_000,
+    stopTimeoutMs: 500,
+  });
+});

packages/ql3-cluster-control/test/processApplication.test.cjs

@@ -75,8 +75,14 @@ test('runs one production replica and drains it on the first signal', async () =
 
   assert.equal(result, 'stopped');
   assert.deepEqual(events, ['subscribe', 'start', 'stop', 'unsubscribe']);
-  assert.equal(facts.some((fact) => fact.event === 'activation'), true);
-  assert.equal(facts.some((fact) => fact.event === 'listening'), true);
+  assert.equal(
+    facts.some((fact) => fact.event === 'activation'),
+    true,
+  );
+  assert.equal(
+    facts.some((fact) => fact.event === 'listening'),
+    true,
+  );
   assert.equal(
     facts.some(
       (fact) =>
@@ -105,7 +111,11 @@ test('fails closed before startup for a disabled profile or invalid replica id',
     await assert.rejects(
       runProductionClusterControlProcess({
         environment,
-        signals: { subscribe() { return () => {}; } },
+        signals: {
+          subscribe() {
+            return () => {};
+          },
+        },
         emit() {},
         async start() {
           starts += 1;
@@ -154,6 +164,7 @@ test('starts the optional Worker listener and closes its lazy Artifact binding',
   const artifactStore = {
     async put() {},
     async inspect() {},
+    async retire() {},
   };
   const environment = {
     ...BASE_ENV,
@@ -187,6 +198,14 @@ test('starts the optional Worker listener and closes its lazy Artifact binding',
       events.push('start');
       assert.equal(options.workerIngress.config.enabled, true);
       assert.equal(options.workerIngress.artifactStore, artifactStore);
+      assert.equal(options.logRetention.store, artifactStore);
+      assert.equal(options.logRetention.ownerId, 'cluster-control-0');
+      assert.equal(options.logRetention.claimLimit, 4);
+      options.logRetention.onDiagnostic(
+        Object.assign(new Error('must-not-be-logged'), {
+          code: 'S3Unavailable',
+        }),
+      );
       return {
         status: 'active',
         address: { host: '0.0.0.0', port: 5800 },
@@ -219,6 +238,16 @@ test('starts the optional Worker listener and closes its lazy Artifact binding',
     facts.some((fact) => fact.event === 'worker_ingress_listening'),
     true,
   );
+  assert.equal(
+    facts.some(
+      (fact) =>
+        fact.event === 'runtime_diagnostic' &&
+        fact.diagnostic.scope === 'log-retention' &&
+        fact.diagnostic.code === 'S3Unavailable' &&
+        JSON.stringify(fact).includes('must-not-be-logged') === false,
+    ),
+    true,
+  );
 });
 
 test('creates the configured mounted Secret provider before Worker activation', async () => {
@@ -226,6 +255,7 @@ test('creates the configured mounted Secret provider before Worker activation',
   const artifactStore = {
     async put() {},
     async inspect() {},
+    async retire() {},
   };
   const provider = { async resolve() {} };
   const result = await runProductionClusterControlProcess({
@@ -353,7 +383,10 @@ test('fails the process after a database fence drains the active application', a
     ),
     true,
   );
-  assert.equal(facts.some(({ event }) => event === 'shutdown_requested'), false);
+  assert.equal(
+    facts.some(({ event }) => event === 'shutdown_requested'),
+    false,
+  );
   assert.equal(facts.at(-1).event, 'stopped');
   assert.equal(
     JSON.stringify(facts).includes('must-not-escape-database-detail'),

packages/ql3-cluster-control/test/productionApplication.test.cjs

@@ -617,6 +617,87 @@ test('wires the production Worker object reader into the Project-scoped log rout
   assert.equal(Buffer.from(result.body.content, 'base64').toString(), 'prod');
 });
 
+test('wires durable retirement authority into the production log route', async () => {
+  const {
+    createRunAttemptLogRetirementRecord,
+  } = require('@qinglong/runtime-core/run-attempt-log-retention');
+  const { input } = fixture();
+  const run = await input.runs.findRunById('run-1');
+  const logArtifactId = `wlog-${'b'.repeat(30)}`;
+  let objectReads = 0;
+  const stack = createProductionClusterControlApplicationStack({
+    ...input,
+    runs: {
+      ...input.runs,
+      async findRunById() {
+        return { ...run, status: 'succeeded', finishedAtMs: 10 };
+      },
+      async findAttemptById() {
+        return {
+          id: 'attempt-1',
+          runId: 'run-1',
+          attempt: 1,
+          status: 'succeeded',
+          executorType: 'remote_worker',
+          logArtifactId,
+          callbackSequence: 0,
+          createdAtMs: 1,
+          finishedAtMs: 10,
+        };
+      },
+    },
+    runAttemptLogRetention: {
+      async inspect(identity) {
+        assert.deepEqual(identity, {
+          projectId: 'project-1',
+          runId: 'run-1',
+          attemptId: 'attempt-1',
+          logArtifactId,
+        });
+        return {
+          status: 'retired',
+          record: createRunAttemptLogRetirementRecord({
+            ...identity,
+            executorType: 'remote_worker',
+            finishedAtMs: 10,
+            eligibleAtMs: 20,
+            retiredAtMs: 30,
+            disposition: 'deleted',
+            byteLength: 64,
+            truncation: { truncated: 'unknown' },
+          }),
+        };
+      },
+    },
+    workerRuntime: {
+      offers: { claimNext() {} },
+      activation: {
+        acknowledgeStarting() {},
+        acknowledgeRunning() {},
+        failStart() {},
+      },
+      artifacts: { upload() {} },
+      completion: { complete() {} },
+      leaseControl: { control() {} },
+      runAttemptLogRead: {
+        async read() {
+          objectReads += 1;
+          return { status: 'missing' };
+        },
+      },
+    },
+  });
+  const result = await invoke(
+    stack,
+    metadata('/api/v3/projects/project-1/runs/run-1/attempts/attempt-1/log'),
+  );
+  assert.equal(result.statusCode, 410);
+  assert.equal(result.body.status, 'retired');
+  assert.equal(result.body.retiredAtMs, 30);
+  assert.equal(result.body.byteLength, 64);
+  assert.equal(objectReads, 0);
+});
+
 test('optionally exposes Prompt execution behind shared admission and policy', async () => {
   const { events, input } = fixture();
   let command;

packages/ql3-cluster-control/test/runAttemptLogRetentionLifecycle.test.cjs

@@ -0,0 +1,281 @@
+'use strict';
+
+const assert = require('node:assert/strict');
+const { test } = require('node:test');
+const {
+  ClusterRunAttemptLogRetentionCoordinator,
+  ClusterRunAttemptLogRetentionLifecycle,
+} = require('../dist/run/runAttemptLogRetentionLifecycle');
+
+function claim(overrides = {}) {
+  return Object.freeze({
+    candidate: Object.freeze({
+      projectId: 'project-1',
+      runId: 'run-1',
+      attemptId: 'attempt-1',
+      logArtifactId: `wlog-${'a'.repeat(30)}`,
+      executorType: 'remote_worker',
+      finishedAtMs: 1_000,
+    }),
+    eligibleAtMs: 61_000,
+    observedAtMs: 70_000,
+    ownerId: 'replica-a',
+    token: '00000000-0000-4000-8000-000000000055',
+    version: 1,
+    expiresAtMs: 100_000,
+    failureCount: 0,
+    ...overrides,
+  });
+}
+
+function options(overrides = {}) {
+  return {
+    ownerId: 'replica-a',
+    retentionMs: 60_000,
+    claimLimit: 4,
+    leaseMs: 30_000,
+    maximumCycleMs: 10_000,
+    retryBaseMs: 1_000,
+    retryMaximumMs: 8_000,
+    maximumFailures: 3,
+    ...overrides,
+  };
+}
+
+function coordinator({
+  claims = [claim()],
+  hasMore = false,
+  retire,
+  settle,
+} = {}) {
+  const calls = [];
+  const value = new ClusterRunAttemptLogRetentionCoordinator(
+    {
+      async claim(input) {
+        calls.push(['claim', input]);
+        return { claims, hasMore };
+      },
+      async settle(current, settlement) {
+        calls.push(['settle', current, settlement]);
+        return (await settle?.(current, settlement)) ?? 'settled';
+      },
+      async inspect() {
+        return { status: 'active' };
+      },
+    },
+    {
+      async retire(candidate, signal) {
+        calls.push(['retire', candidate, signal]);
+        return (
+          (await retire?.(candidate, signal)) ?? {
+            disposition: 'deleted',
+            byteLength: 11,
+            truncation: { truncated: false },
+          }
+        );
+      },
+    },
+    options(),
+  );
+  return { calls, coordinator: value };
+}
+
+test('claims one bounded page and records exact DB-clock retirement evidence', async () => {
+  const { calls, coordinator: value } = coordinator({
+    claims: [
+      claim(),
+      claim({
+        candidate: Object.freeze({
+          ...claim().candidate,
+          attemptId: 'attempt-2',
+          logArtifactId: `wlog-${'b'.repeat(30)}`,
+        }),
+      }),
+    ],
+    hasMore: true,
+    retire(candidate) {
+      return candidate.attemptId === 'attempt-1'
+        ? {
+            disposition: 'deleted',
+            byteLength: 11,
+            truncation: { truncated: false },
+          }
+        : {
+            disposition: 'already_absent',
+            byteLength: 0,
+            truncation: { truncated: 'unknown' },
+          };
+    },
+  });
+
+  const summary = await value.runOnce();
+  assert.deepEqual(summary, {
+    status: 'saturated',
+    claimed: 2,
+    attempted: 2,
+    retired: 1,
+    alreadyAbsent: 1,
+    retried: 0,
+    manual: 0,
+    fenced: 0,
+    hasMore: true,
+    entries: [
+      { attemptId: 'attempt-1', outcome: 'deleted' },
+      { attemptId: 'attempt-2', outcome: 'already_absent' },
+    ],
+  });
+  assert.deepEqual(calls[0][1], {
+    ownerId: 'replica-a',
+    retentionMs: 60_000,
+    limit: 4,
+    leaseMs: 30_000,
+  });
+  const records = calls
+    .filter(([kind]) => kind === 'settle')
+    .map(([, , settlement]) => settlement.record);
+  assert.equal(records[0].retiredAtMs, 70_000);
+  assert.equal(records[0].recordDigest.length, 64);
+  assert.equal(records[1].byteLength, 0);
+});
+
+test('uses bounded exponential retry then moves repeated failures to manual', async () => {
+  const first = claim({ failureCount: 2 });
+  const second = claim({
+    candidate: Object.freeze({
+      ...claim().candidate,
+      attemptId: 'attempt-2',
+      logArtifactId: `wlog-${'b'.repeat(30)}`,
+    }),
+    failureCount: 1,
+  });
+  const { calls, coordinator: value } = coordinator({
+    claims: [first, second],
+    retire(candidate) {
+      const error = new Error('object drift');
+      if (candidate.attemptId === 'attempt-1')
+        error.reason = 'integrity_mismatch';
+      throw error;
+    },
+  });
+
+  const summary = await value.runOnce();
+  assert.equal(summary.manual, 1);
+  assert.equal(summary.retried, 1);
+  const settlements = calls
+    .filter(([kind]) => kind === 'settle')
+    .map(([, , settlement]) => settlement);
+  assert.deepEqual(settlements, [
+    { status: 'manual', failureCode: 'artifact_integrity_mismatch' },
+    {
+      status: 'retry',
+      delayMs: 2_000,
+      failureCode: 'artifact_unavailable',
+    },
+  ]);
+});
+
+test('classifies malformed retirement evidence and preserves a fenced settlement', async () => {
+  const { coordinator: value } = coordinator({
+    retire() {
+      return {
+        disposition: 'already_absent',
+        byteLength: 5,
+        truncation: { truncated: 'unknown' },
+      };
+    },
+    settle() {
+      return 'fenced';
+    },
+  });
+  const summary = await value.runOnce();
+  assert.equal(summary.fenced, 1);
+  assert.deepEqual(summary.entries, [
+    { attemptId: 'attempt-1', outcome: 'fenced' },
+  ]);
+});
+
+test('cycle budget aborts object work and leaves the durable claim for takeover', async () => {
+  let settlements = 0;
+  const value = new ClusterRunAttemptLogRetentionCoordinator(
+    {
+      async claim() {
+        return { claims: [claim()], hasMore: false };
+      },
+      async settle() {
+        settlements += 1;
+        return 'settled';
+      },
+      async inspect() {
+        return { status: 'active' };
+      },
+    },
+    {
+      retire(_candidate, signal) {
+        return new Promise((_, reject) => {
+          signal.addEventListener('abort', () => reject(signal.reason), {
+            once: true,
+          });
+        });
+      },
+    },
+    options({ maximumCycleMs: 100 }),
+  );
+  const summary = await value.runOnce();
+  assert.equal(summary.status, 'budget_exhausted');
+  assert.equal(summary.attempted, 1);
+  assert.equal(settlements, 0);
+});
+
+test('lifecycle coalesces cycles and aborts one in-flight object call on drain', async () => {
+  let calls = 0;
+  let observedAbort = false;
+  const lifecycle = new ClusterRunAttemptLogRetentionLifecycle(
+    {
+      runOnce(signal) {
+        calls += 1;
+        return new Promise((resolve) => {
+          signal.addEventListener(
+            'abort',
+            () => {
+              observedAbort = true;
+              resolve({ status: 'budget_exhausted' });
+            },
+            { once: true },
+          );
+        });
+      },
+    },
+    { intervalMs: 60_000, stopTimeoutMs: 1_000 },
+  );
+  const first = lifecycle.runOnce();
+  assert.equal(lifecycle.runOnce(), first);
+  assert.equal(await lifecycle.stopAndDrain(), 'stopped');
+  await first;
+  assert.equal(calls, 1);
+  assert.equal(observedAbort, true);
+  await assert.rejects(lifecycle.runOnce(), /is stopping/);
+});
+
+test('rejects configurations that can outlive the lease settlement budget', () => {
+  const dependencies = [
+    { claim() {}, settle() {}, inspect() {} },
+    { retire() {} },
+  ];
+  assert.throws(
+    () =>
+      new ClusterRunAttemptLogRetentionCoordinator(
+        dependencies[0],
+        dependencies[1],
+        options({ leaseMs: 5_000, maximumCycleMs: 4_501 }),
+      ),
+    /cycle budget/,
+  );
+  assert.throws(
+    () =>
+      new ClusterRunAttemptLogRetentionLifecycle(
+        { runOnce() {} },
+        { intervalMs: 999, stopTimeoutMs: 1_000 },
+      ),
+    /lifecycle options/,
+  );
+});

packages/ql3-cluster-control/test/s3ArtifactStore.integration.test.cjs

@@ -7,7 +7,9 @@ const {
   CreateBucketCommand,
   DeleteBucketCommand,
   DeleteObjectsCommand,
+  ListObjectVersionsCommand,
   ListObjectsV2Command,
+  PutBucketVersioningCommand,
   S3Client,
 } = require('@aws-sdk/client-s3');
 const {
@@ -35,6 +37,7 @@ test(
       credentials: { accessKeyId, secretAccessKey },
     });
     const bucket = `ql3-artifact-${process.pid}-${Date.now()}`.slice(0, 63);
+    const versionedBucket = `${bucket}-v`.slice(0, 63);
     const command = Object.freeze({
       projectId: 'project-s3-integration',
       runId: 'run-s3-integration',
@@ -101,27 +104,123 @@ test(
       );
       assert.equal(objects.KeyCount, 1);
       assert.match(objects.Contents[0].Key, /\/objects\//);
-    } finally {
-      try {
-        const objects = await client.send(
-          new ListObjectsV2Command({
-            Bucket: bucket,
-          }),
-        );
-        if (objects.Contents?.length) {
+      const retired = await store.retire({
+        projectId: command.projectId,
+        runId: command.runId,
+        attemptId: command.attemptId,
+        logArtifactId: command.logArtifactId,
+        executorType: 'remote_worker',
+        finishedAtMs: 1,
+      });
+      assert.deepEqual(retired, {
+        disposition: 'deleted',
+        byteLength: content.byteLength,
+        truncation: { truncated: true },
+      });
+      assert.equal(
+        (
           await client.send(
-            new DeleteObjectsCommand({
+            new ListObjectsV2Command({
               Bucket: bucket,
-              Delete: {
-                Objects: objects.Contents.map(({ Key }) => ({ Key })),
-                Quiet: true,
-              },
+              Prefix: 'qinglong/integration/',
             }),
+          )
+        ).KeyCount,
+        0,
+      );
+      assert.deepEqual(
+        await store.retire({
+          projectId: command.projectId,
+          runId: command.runId,
+          attemptId: command.attemptId,
+          logArtifactId: command.logArtifactId,
+          executorType: 'remote_worker',
+          finishedAtMs: 1,
+        }),
+        {
+          disposition: 'already_absent',
+          byteLength: 0,
+          truncation: { truncated: 'unknown' },
+        },
+      );
+
+      await client.send(new CreateBucketCommand({ Bucket: versionedBucket }));
+      await client.send(
+        new PutBucketVersioningCommand({
+          Bucket: versionedBucket,
+          VersioningConfiguration: { Status: 'Enabled' },
+        }),
+      );
+      const versionedStore = new S3ClusterRemoteWorkerArtifactStore({
+        client,
+        bucket: versionedBucket,
+        prefix: 'qinglong/integration',
+        encryption: { mode: 's3' },
+      });
+      assert.equal(
+        (await versionedStore.put(command, body(content))).status,
+        'stored',
+      );
+      const beforeVersionedRetirement = await client.send(
+        new ListObjectVersionsCommand({ Bucket: versionedBucket }),
+      );
+      assert.equal(beforeVersionedRetirement.Versions?.length, 1);
+      assert.equal(beforeVersionedRetirement.DeleteMarkers?.length ?? 0, 0);
+      assert.match(beforeVersionedRetirement.Versions[0].Key, /\/objects\//);
+      assert.equal(
+        (
+          await versionedStore.retire({
+            projectId: command.projectId,
+            runId: command.runId,
+            attemptId: command.attemptId,
+            logArtifactId: command.logArtifactId,
+            executorType: 'remote_worker',
+            finishedAtMs: 1,
+          })
+        ).disposition,
+        'deleted',
+      );
+      const afterVersionedRetirement = await client.send(
+        new ListObjectVersionsCommand({ Bucket: versionedBucket }),
+      );
+      assert.equal(afterVersionedRetirement.Versions?.length ?? 0, 0);
+      assert.equal(afterVersionedRetirement.DeleteMarkers?.length ?? 0, 0);
+    } finally {
+      for (const cleanupBucket of [versionedBucket, bucket]) {
+        try {
+          const versions = await client.send(
+            new ListObjectVersionsCommand({ Bucket: cleanupBucket }),
           );
+          const versionedObjects = [
+            ...(versions.Versions ?? []),
+            ...(versions.DeleteMarkers ?? []),
+          ].map(({ Key, VersionId }) => ({ Key, VersionId }));
+          if (versionedObjects.length) {
+            await client.send(
+              new DeleteObjectsCommand({
+                Bucket: cleanupBucket,
+                Delete: { Objects: versionedObjects, Quiet: true },
+              }),
+            );
+          }
+          const objects = await client.send(
+            new ListObjectsV2Command({ Bucket: cleanupBucket }),
+          );
+          if (objects.Contents?.length) {
+            await client.send(
+              new DeleteObjectsCommand({
+                Bucket: cleanupBucket,
+                Delete: {
+                  Objects: objects.Contents.map(({ Key }) => ({ Key })),
+                  Quiet: true,
+                },
+              }),
+            );
+          }
+          await client.send(new DeleteBucketCommand({ Bucket: cleanupBucket }));
+        } catch {
+          // Preserve the integration assertion; the ephemeral container is removed.
         }
-        await client.send(new DeleteBucketCommand({ Bucket: bucket }));
-      } catch {
-        // Preserve the integration assertion; the ephemeral container is removed.
       }
       client.destroy();
     }

packages/ql3-cluster-control/test/s3ArtifactStore.test.cjs

@@ -73,6 +73,14 @@ class MemoryS3Client {
             ? Buffer.alloc(32, 9).toString('base64')
             : checksum(object.content),
         Metadata: metadata,
+        ...(input.Key.includes('/objects/') &&
+        this.options.headVersionId !== undefined
+          ? { VersionId: this.options.headVersionId }
+          : {}),
+        ...(input.Key.includes('/temporary/') &&
+        this.options.temporaryHeadVersionId !== undefined
+          ? { VersionId: this.options.temporaryHeadVersionId }
+          : {}),
       };
     }
     if (command instanceof GetObjectCommand) {
@@ -168,7 +176,33 @@ class MemoryS3Client {
     }
     if (command instanceof DeleteObjectCommand) {
       if (this.options.failDelete) throw new Error('delete unavailable');
+      if (input.Key.includes('/objects/')) {
+        if (this.options.permanentDeletePreconditionFailure) {
+          const error = new Error('precondition failed');
+          error.name = 'PreconditionFailed';
+          error.$metadata = { httpStatusCode: 412 };
+          throw error;
+        }
+        const object = this.objects.get(input.Key);
+        if (!object) throw notFound();
+        if (this.options.headVersionId === undefined) {
+          assert.equal(
+            input.IfMatch,
+            `"${checksum(object.content).slice(0, 32)}"`,
+          );
+          assert.equal(input.VersionId, undefined);
+        } else {
+          assert.equal(input.IfMatch, undefined);
+          assert.equal(input.VersionId, this.options.headVersionId);
+        }
+      }
       this.objects.delete(input.Key);
+      if (
+        input.Key.includes('/objects/') &&
+        this.options.throwAfterPermanentDelete
+      ) {
+        throw new Error('lost delete response');
+      }
       return {};
     }
     throw new Error(`unexpected command: ${command.constructor.name}`);
@@ -201,6 +235,15 @@ function permanentKey(client) {
   return [...client.objects.keys()].find((key) => key.includes('/objects/'));
 }
 
+function retentionCandidate(overrides = {}) {
+  return Object.freeze({
+    ...LOOKUP,
+    executorType: 'remote_worker',
+    finishedAtMs: 1_000,
+    ...overrides,
+  });
+}
+
 test('streams to a checksummed temporary object then conditionally promotes it', async () => {
   const client = new MemoryS3Client();
   const adapter = store(client);
@@ -233,6 +276,13 @@ test('streams to a checksummed temporary object then conditionally promotes it',
   const copy = client.commands.find(
     (command) => command instanceof CopyObjectCommand,
   );
+  const cleanup = client.commands.find(
+    (command) =>
+      command instanceof DeleteObjectCommand &&
+      command.input.Key.includes('/temporary/'),
+  );
+  assert.match(cleanup.input.IfMatch, /^"[A-Za-z0-9+/=]+"$/);
+  assert.equal(cleanup.input.VersionId, undefined);
   assert.equal(copy.input.Metadata['ql3-content-sha256'], CONTENT_SHA256);
   assert.equal(
     JSON.stringify(copy.input.Metadata).includes(COMMAND.projectId),
@@ -247,6 +297,20 @@ test('streams to a checksummed temporary object then conditionally promotes it',
   assert.deepEqual(inspected, { ...receipt, status: 'already_stored' });
 });
 
+test('cleans one exact temporary object version after validated HEAD', async () => {
+  const client = new MemoryS3Client({
+    temporaryHeadVersionId: 'temporary/version+1=',
+  });
+  await store(client).put(COMMAND, chunks());
+  const cleanup = client.commands.find(
+    (command) =>
+      command instanceof DeleteObjectCommand &&
+      command.input.Key.includes('/temporary/'),
+  );
+  assert.equal(cleanup.input.VersionId, 'temporary/version+1=');
+  assert.equal(cleanup.input.IfMatch, undefined);
+});
+
 test('exact replay consumes and hashes the whole body without another write', async () => {
   const client = new MemoryS3Client();
   const adapter = store(client);
@@ -510,3 +574,101 @@ test('a pre-aborted request performs no object-store operation', async () => {
   );
   assert.equal(client.commands.length, 0);
 });
+
+test('retires an unversioned Artifact only with its validated ETag', async () => {
+  const client = new MemoryS3Client();
+  const adapter = store(client, { expectedBucketOwner: '123456789012' });
+  await adapter.put(COMMAND, chunks());
+  client.commands.length = 0;
+
+  assert.deepEqual(await adapter.retire(retentionCandidate()), {
+    disposition: 'deleted',
+    byteLength: CONTENT.byteLength,
+    truncation: { truncated: false },
+  });
+  assert.deepEqual(
+    client.commands.map((command) => command.constructor.name),
+    ['HeadObjectCommand', 'DeleteObjectCommand'],
+  );
+  assert.equal(client.commands[1].input.ExpectedBucketOwner, '123456789012');
+  assert.equal(permanentKey(client), undefined);
+});
+
+test('retires one exact version when HEAD returns an opaque VersionId', async () => {
+  const client = new MemoryS3Client({ headVersionId: 'version/opaque+1=' });
+  const adapter = store(client);
+  await adapter.put(COMMAND, chunks());
+  client.commands.length = 0;
+
+  const result = await adapter.retire(retentionCandidate());
+  assert.equal(result.disposition, 'deleted');
+  assert.equal(client.commands[1].input.VersionId, 'version/opaque+1=');
+  assert.equal(client.commands[1].input.IfMatch, undefined);
+  assert.equal(permanentKey(client), undefined);
+});
+
+test('returns durable absent evidence without issuing a delete', async () => {
+  const client = new MemoryS3Client();
+  const adapter = store(client);
+
+  assert.deepEqual(await adapter.retire(retentionCandidate()), {
+    disposition: 'already_absent',
+    byteLength: 0,
+    truncation: { truncated: 'unknown' },
+  });
+  assert.deepEqual(
+    client.commands.map((command) => command.constructor.name),
+    ['HeadObjectCommand'],
+  );
+});
+
+test('fails closed on conditional-delete drift and malformed version authority', async () => {
+  for (const options of [
+    { permanentDeletePreconditionFailure: true },
+    { headVersionId: 'invalid\nversion' },
+  ]) {
+    const client = new MemoryS3Client(options);
+    const adapter = store(client);
+    await adapter.put(COMMAND, chunks());
+    client.commands.length = 0;
+
+    await assert.rejects(
+      adapter.retire(retentionCandidate()),
+      (error) =>
+        error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
+        error.reason === 'integrity_mismatch',
+    );
+    assert.notEqual(permanentKey(client), undefined);
+  }
+
+  const wrongExecutor = new MemoryS3Client();
+  await assert.rejects(
+    store(wrongExecutor).retire(
+      retentionCandidate({ executorType: 'local_process' }),
+    ),
+    (error) =>
+      error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
+      error.reason === 'integrity_mismatch',
+  );
+  assert.equal(wrongExecutor.commands.length, 0);
+});
+
+test('lost delete response converges through a later absent inspection', async () => {
+  const client = new MemoryS3Client({ throwAfterPermanentDelete: true });
+  const adapter = store(client);
+  await adapter.put(COMMAND, chunks());
+  client.commands.length = 0;
+
+  await assert.rejects(
+    adapter.retire(retentionCandidate()),
+    (error) =>
+      error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
+      error.reason === 'unavailable',
+  );
+  client.options.throwAfterPermanentDelete = false;
+  assert.deepEqual(await adapter.retire(retentionCandidate()), {
+    disposition: 'already_absent',
+    byteLength: 0,
+    truncation: { truncated: 'unknown' },
+  });
+});