feat(ql3): add strong local manual run retry

packages/ql3-local-owner-cli/package.json

@@ -60,6 +60,11 @@
       "require": "./dist/automation-management/taskDefinitionCommand.js",
       "default": "./dist/automation-management/taskDefinitionCommand.js"
     },
+    "./run-retry-command": {
+      "types": "./dist/run-management/runRetryCommand.d.ts",
+      "require": "./dist/run-management/runRetryCommand.js",
+      "default": "./dist/run-management/runRetryCommand.js"
+    },
     "./plugin-package-workflow-command": {
       "types": "./dist/plugin-package/pluginPackageWorkflowCommand.d.ts",
       "require": "./dist/plugin-package/pluginPackageWorkflowCommand.js",
@@ -125,6 +130,7 @@
     "ql3-ai-feature": "dist/ai-management/aiFeatureCli.js",
     "ql3-secret": "dist/security-management/secretCli.js",
     "ql3-task": "dist/automation-management/taskDefinitionCli.js",
+    "ql3-run": "dist/run-management/runRetryCli.js",
     "ql3-workflow": "dist/plugin-package/pluginPackageWorkflowCli.js",
     "ql3-trigger": "dist/automation-management/triggerCli.js",
     "ql3-policy": "dist/security-management/projectPolicyCli.js",

packages/ql3-local-owner-cli/src/product-cli/productCommand.ts

@@ -84,6 +84,12 @@ export const QINGLONG3_PRODUCT_COMMANDS: readonly QingLong3ProductCommandDefinit
       target: 'automation-management/taskDefinitionCli.js',
       description: 'manage Task definitions',
     }),
+    Object.freeze({
+      name: 'run',
+      binary: 'ql3-run',
+      target: 'run-management/runRetryCli.js',
+      description: 'retry terminal Runs under strong local authentication',
+    }),
     Object.freeze({
       name: 'trigger',
       binary: 'ql3-trigger',

packages/ql3-local-owner-cli/src/run-management/runRetryCli.ts

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+
+import { runLocalRunRetryCommandFile } from './runRetryCommand';
+
+const USAGE =
+  'Usage: ql3-run retry --command-file /absolute/private-command.json';
+
+async function main(argv: readonly string[]): Promise<void> {
+  if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
+    process.stdout.write(`${USAGE}\n`);
+    return;
+  }
+  const commandFilePath = argv[2];
+  if (
+    argv.length !== 3 ||
+    argv[0] !== 'retry' ||
+    argv[1] !== '--command-file' ||
+    commandFilePath === undefined
+  ) {
+    process.stderr.write(
+      `${JSON.stringify({
+        code: 'LOCAL_RUN_RETRY_CLI_USAGE_INVALID',
+        message: USAGE,
+      })}\n`,
+    );
+    process.exitCode = 64;
+    return;
+  }
+  try {
+    const result = await runLocalRunRetryCommandFile(commandFilePath);
+    process.stdout.write(`${JSON.stringify(result)}\n`);
+  } catch (error) {
+    const candidate = error as {
+      readonly code?: unknown;
+      readonly name?: unknown;
+      readonly message?: unknown;
+      readonly retryAfterMs?: unknown;
+    };
+    process.stderr.write(
+      `${JSON.stringify({
+        code:
+          typeof candidate.code === 'string'
+            ? candidate.code
+            : 'LOCAL_RUN_RETRY_CLI_FAILED',
+        name: typeof candidate.name === 'string' ? candidate.name : 'Error',
+        message:
+          typeof candidate.message === 'string'
+            ? candidate.message
+            : 'Local Run retry command failed',
+        ...(Number.isSafeInteger(candidate.retryAfterMs)
+          ? { retryAfterMs: candidate.retryAfterMs }
+          : {}),
+      })}\n`,
+    );
+    process.exitCode = 1;
+  }
+}
+
+void main(process.argv.slice(2));

packages/ql3-local-owner-cli/src/run-management/runRetryCommand.ts

@@ -0,0 +1,500 @@
+import { randomUUID } from 'node:crypto';
+import path from 'node:path';
+
+import {
+  PrivateLocalCommandFileError,
+  readPrivateLocalCommandFile,
+} from '@qinglong/local-command-file';
+import {
+  AuthenticatedLocalCommandAuthenticationError,
+  establishAuthenticatedLocalCommand,
+  type AuthenticatedLocalCommand,
+} from '@qinglong/local-owner-console/authenticated-command';
+import {
+  LocalSqliteAuthenticatedManagementFenceError,
+  type LocalSqliteAuthenticatedUserCredentialFence,
+} from '@qinglong/local-sqlite/authenticated-management';
+import {
+  openLocalSqliteRunManagementDatabase,
+  type LocalSqliteRunManagementDatabase,
+} from '@qinglong/local-sqlite/run-management';
+import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy';
+import {
+  RUN_MANUAL_RETRY_SCHEMA,
+  RunManualRetryFenceRejectedError,
+  RunManualRetryNotFoundError,
+  RunManualRetryRateLimitedError,
+  RunManualRetryUnavailableError,
+  parseRunManualRetryRequestBody,
+  type RunManualRetryResult,
+  type RunManualRetrySourceStatus,
+} from '@qinglong/runtime-core/run-manual-retry';
+import {
+  normalizeSecurityAuditRecord,
+  type SecurityAuditRecord,
+} from '@qinglong/runtime-core/security-audit';
+
+const MAX_PATH_BYTES = 4_096;
+const UUID_V4_PATTERN =
+  /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
+const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+
+export interface LocalRunRetryCommandOptions {
+  readonly deploymentRoot: string;
+  readonly databasePath: string;
+  readonly profile: 'edge' | 'standalone';
+  readonly ownerPepperKeyringDirectory: string;
+  readonly credentialFilePath: string;
+  readonly busyTimeoutMs?: number;
+}
+
+export interface LocalRunRetryCommand {
+  readonly schemaVersion: 1;
+  readonly operation: 'run.retry';
+  readonly options: LocalRunRetryCommandOptions;
+  readonly request: Readonly<{
+    projectId: string;
+    sourceRunId: string;
+    mutationId: string;
+    expectedRunVersion: number;
+    expectedRunStatus: RunManualRetrySourceStatus;
+    requestId: string;
+    auditEventId: string;
+    failureAuditEventId: string;
+    occurredAtMs: number;
+  }>;
+}
+
+export interface LocalRunRetryCommandResult {
+  readonly schemaVersion: 1;
+  readonly operation: 'run.retry';
+  readonly retry: Readonly<RunManualRetryResult>;
+}
+
+export interface LocalRunRetryCommandRunner {
+  run(commandFilePath: string): Promise<Readonly<LocalRunRetryCommandResult>>;
+}
+
+export interface LocalRunRetryCommandRunnerDependencies {
+  readonly openDatabase: typeof openLocalSqliteRunManagementDatabase;
+  readonly authenticate: typeof establishAuthenticatedLocalCommand;
+  readonly now: () => number;
+  readonly randomUuid: () => string;
+}
+
+export class LocalRunRetryCommandConfigurationError extends TypeError {
+  readonly code = 'LOCAL_RUN_RETRY_COMMAND_CONFIGURATION_INVALID';
+
+  constructor(message: string, options?: ErrorOptions) {
+    super(
+      `Local Run retry command configuration is invalid: ${message}`,
+      options,
+    );
+    this.name = 'LocalRunRetryCommandConfigurationError';
+  }
+}
+
+export class LocalRunRetryCommandAuthorizationError extends Error {
+  readonly code = 'LOCAL_RUN_RETRY_COMMAND_AUTHORIZATION_REJECTED';
+
+  constructor() {
+    super('Local Run retry command authorization was rejected');
+    this.name = 'LocalRunRetryCommandAuthorizationError';
+  }
+}
+
+export class LocalRunRetryCommandUnavailableError extends Error {
+  readonly code = 'LOCAL_RUN_RETRY_COMMAND_UNAVAILABLE';
+
+  constructor(options?: ErrorOptions) {
+    super('Local Run retry command is unavailable', options);
+    this.name = 'LocalRunRetryCommandUnavailableError';
+  }
+}
+
+function exactRecord(
+  value: unknown,
+  required: readonly string[],
+  optional: readonly string[],
+  label: string,
+): asserts value is Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new LocalRunRetryCommandConfigurationError(
+      `${label} must be an object`,
+    );
+  }
+  const keys = Object.keys(value).sort();
+  const allowed = new Set([...required, ...optional]);
+  if (
+    required.some((key) => !keys.includes(key)) ||
+    keys.some((key) => !allowed.has(key))
+  ) {
+    throw new LocalRunRetryCommandConfigurationError(
+      `${label} shape is invalid`,
+    );
+  }
+}
+
+function boundedPath(value: unknown, label: string): string {
+  if (
+    typeof value !== 'string' ||
+    value.length < 1 ||
+    Buffer.byteLength(value, 'utf8') > MAX_PATH_BYTES ||
+    value.includes('\0') ||
+    !path.isAbsolute(value) ||
+    path.normalize(value) !== value ||
+    path.parse(value).root === value
+  ) {
+    throw new LocalRunRetryCommandConfigurationError(
+      `${label} must be a normalized bounded absolute non-root path`,
+    );
+  }
+  return value;
+}
+
+function descendant(root: string, value: string, label: string): void {
+  const relative = path.relative(root, value);
+  if (
+    relative.length === 0 ||
+    relative === '..' ||
+    relative.startsWith(`..${path.sep}`) ||
+    path.isAbsolute(relative)
+  ) {
+    throw new LocalRunRetryCommandConfigurationError(
+      `${label} must be a descendant of deploymentRoot`,
+    );
+  }
+}
+
+function identifier(value: unknown, label: string): string {
+  if (typeof value !== 'string' || !IDENTIFIER_PATTERN.test(value)) {
+    throw new LocalRunRetryCommandConfigurationError(`${label} is invalid`);
+  }
+  return value;
+}
+
+function uuid(value: unknown, label: string): string {
+  if (typeof value !== 'string' || !UUID_V4_PATTERN.test(value)) {
+    throw new LocalRunRetryCommandConfigurationError(`${label} is invalid`);
+  }
+  return value;
+}
+
+function options(value: unknown): Readonly<LocalRunRetryCommandOptions> {
+  exactRecord(
+    value,
+    [
+      'deploymentRoot',
+      'databasePath',
+      'profile',
+      'ownerPepperKeyringDirectory',
+      'credentialFilePath',
+    ],
+    ['busyTimeoutMs'],
+    'options',
+  );
+  const deploymentRoot = boundedPath(value.deploymentRoot, 'deploymentRoot');
+  const databasePath = boundedPath(value.databasePath, 'databasePath');
+  const ownerPepperKeyringDirectory = boundedPath(
+    value.ownerPepperKeyringDirectory,
+    'ownerPepperKeyringDirectory',
+  );
+  const credentialFilePath = boundedPath(
+    value.credentialFilePath,
+    'credentialFilePath',
+  );
+  for (const [target, label] of [
+    [databasePath, 'databasePath'],
+    [ownerPepperKeyringDirectory, 'ownerPepperKeyringDirectory'],
+    [credentialFilePath, 'credentialFilePath'],
+  ] as const) {
+    descendant(deploymentRoot, target, label);
+  }
+  if (
+    new Set([databasePath, ownerPepperKeyringDirectory, credentialFilePath])
+      .size !== 3 ||
+    (value.profile !== 'edge' && value.profile !== 'standalone') ||
+    (value.busyTimeoutMs !== undefined &&
+      (!Number.isSafeInteger(value.busyTimeoutMs) ||
+        (value.busyTimeoutMs as number) < 1 ||
+        (value.busyTimeoutMs as number) > 60_000))
+  ) {
+    throw new LocalRunRetryCommandConfigurationError('options are invalid');
+  }
+  return Object.freeze({
+    deploymentRoot,
+    databasePath,
+    profile: value.profile,
+    ownerPepperKeyringDirectory,
+    credentialFilePath,
+    ...(value.busyTimeoutMs === undefined
+      ? {}
+      : { busyTimeoutMs: value.busyTimeoutMs as number }),
+  });
+}
+
+function normalizeCommand(value: unknown): Readonly<LocalRunRetryCommand> {
+  exactRecord(
+    value,
+    ['schemaVersion', 'operation', 'options', 'request'],
+    [],
+    'command',
+  );
+  if (value.schemaVersion !== 1 || value.operation !== 'run.retry') {
+    throw new LocalRunRetryCommandConfigurationError(
+      'schemaVersion or operation is invalid',
+    );
+  }
+  exactRecord(
+    value.request,
+    [
+      'projectId',
+      'sourceRunId',
+      'mutationId',
+      'expectedRunVersion',
+      'expectedRunStatus',
+      'requestId',
+      'auditEventId',
+      'failureAuditEventId',
+      'occurredAtMs',
+    ],
+    [],
+    'request',
+  );
+  let retry;
+  try {
+    retry = parseRunManualRetryRequestBody({
+      schema: RUN_MANUAL_RETRY_SCHEMA,
+      mutationId: value.request.mutationId,
+      expectedRunVersion: value.request.expectedRunVersion,
+      expectedRunStatus: value.request.expectedRunStatus,
+    });
+  } catch (error) {
+    throw new LocalRunRetryCommandConfigurationError(
+      'retry request is invalid',
+      { cause: error },
+    );
+  }
+  if (
+    !Number.isSafeInteger(value.request.occurredAtMs) ||
+    (value.request.occurredAtMs as number) < 0
+  ) {
+    throw new LocalRunRetryCommandConfigurationError(
+      'request occurredAtMs is invalid',
+    );
+  }
+  return Object.freeze({
+    schemaVersion: 1,
+    operation: 'run.retry',
+    options: options(value.options),
+    request: Object.freeze({
+      projectId: identifier(value.request.projectId, 'projectId'),
+      sourceRunId: identifier(value.request.sourceRunId, 'sourceRunId'),
+      mutationId: retry.mutationId,
+      expectedRunVersion: retry.expectedRunVersion,
+      expectedRunStatus: retry.expectedRunStatus,
+      requestId: identifier(value.request.requestId, 'requestId'),
+      auditEventId: uuid(value.request.auditEventId, 'auditEventId'),
+      failureAuditEventId: uuid(
+        value.request.failureAuditEventId,
+        'failureAuditEventId',
+      ),
+      occurredAtMs: value.request.occurredAtMs as number,
+    }),
+  });
+}
+
+function readCommandFile(
+  commandFilePath: string,
+): Readonly<LocalRunRetryCommand> {
+  try {
+    return normalizeCommand(readPrivateLocalCommandFile(commandFilePath));
+  } catch (error) {
+    if (error instanceof LocalRunRetryCommandConfigurationError) throw error;
+    if (error instanceof PrivateLocalCommandFileError) {
+      throw new LocalRunRetryCommandConfigurationError(
+        'private command file cannot be read',
+        { cause: error },
+      );
+    }
+    throw new LocalRunRetryCommandConfigurationError(
+      'command file is invalid',
+      { cause: error },
+    );
+  }
+}
+
+function dependencies(
+  value: LocalRunRetryCommandRunnerDependencies,
+): LocalRunRetryCommandRunnerDependencies {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    Array.isArray(value) ||
+    Object.keys(value).sort().join('\0') !==
+      ['authenticate', 'now', 'openDatabase', 'randomUuid'].sort().join('\0') ||
+    typeof value.openDatabase !== 'function' ||
+    typeof value.authenticate !== 'function' ||
+    typeof value.now !== 'function' ||
+    typeof value.randomUuid !== 'function'
+  ) {
+    throw new LocalRunRetryCommandConfigurationError(
+      'runner dependencies are invalid',
+    );
+  }
+  return Object.freeze({ ...value });
+}
+
+function failureReason(error: unknown): string {
+  if (error instanceof RunManualRetryRateLimitedError) {
+    return 'run_retry_rate_limited';
+  }
+  if (error instanceof RunManualRetryNotFoundError) return 'run_not_found';
+  if (
+    error instanceof LocalRunRetryCommandAuthorizationError ||
+    error instanceof RunManualRetryFenceRejectedError ||
+    error instanceof LocalSqliteAuthenticatedManagementFenceError
+  ) {
+    return 'run_retry_fence_rejected';
+  }
+  return 'run_retry_unavailable';
+}
+
+function failureAudit(
+  command: Readonly<LocalRunRetryCommand>,
+  authenticated: Readonly<AuthenticatedLocalCommand> | undefined,
+  error: unknown,
+): Readonly<SecurityAuditRecord> {
+  const unauthenticated = authenticated === undefined;
+  return normalizeSecurityAuditRecord({
+    eventId: command.request.failureAuditEventId,
+    requestId: command.request.requestId,
+    operationId: 'run.retry',
+    projectId: command.request.projectId,
+    subject: authenticated?.principal.subject ?? null,
+    authenticationId: authenticated?.principal.authenticationId ?? null,
+    outcome: unauthenticated ? 'authentication_rejected' : 'denied',
+    reasons: [
+      unauthenticated ? 'local_console_required' : failureReason(error),
+    ],
+    fence: null,
+    occurredAtMs: command.request.occurredAtMs,
+  });
+}
+
+export function createLocalRunRetryCommandRunner(
+  candidateDependencies: LocalRunRetryCommandRunnerDependencies = {
+    openDatabase: openLocalSqliteRunManagementDatabase,
+    authenticate: establishAuthenticatedLocalCommand,
+    now: Date.now,
+    randomUuid: randomUUID,
+  },
+): Readonly<LocalRunRetryCommandRunner> {
+  const adapters = dependencies(candidateDependencies);
+  return Object.freeze({
+    async run(commandFilePath: string) {
+      const command = readCommandFile(commandFilePath);
+      const nowMs = adapters.now();
+      if (
+        !Number.isSafeInteger(nowMs) ||
+        nowMs < command.request.occurredAtMs ||
+        nowMs - command.request.occurredAtMs > 5 * 60_000
+      ) {
+        throw new LocalRunRetryCommandConfigurationError(
+          'request time is outside the accepted window',
+        );
+      }
+      const database = await adapters.openDatabase({
+        databasePath: command.options.databasePath,
+        profile: command.options.profile,
+        ...(command.options.busyTimeoutMs === undefined
+          ? {}
+          : { busyTimeoutMs: command.options.busyTimeoutMs }),
+      });
+      let authenticated: Readonly<AuthenticatedLocalCommand> | undefined;
+      try {
+        try {
+          authenticated = await adapters.authenticate(database, {
+            deploymentRoot: command.options.deploymentRoot,
+            databasePath: command.options.databasePath,
+            ownerPepperKeyringDirectory:
+              command.options.ownerPepperKeyringDirectory,
+            credentialFilePath: command.options.credentialFilePath,
+            authenticationNamespace: 'local_run_retry',
+          });
+          database.activateUserCredentialFence(
+            authenticated.databaseFence as Readonly<LocalSqliteAuthenticatedUserCredentialFence>,
+          );
+          const decision = await new ProjectPolicyEngine(
+            database.projectPolicy,
+          ).authorize(
+            authenticated.principal,
+            command.request.projectId,
+            'run.retry',
+          );
+          if (
+            decision.effect !== 'allow' ||
+            !decision.fence ||
+            decision.fence.bindingVersion === null
+          ) {
+            throw new LocalRunRetryCommandAuthorizationError();
+          }
+          await authenticated.confirm();
+          const result = await database.runManualRetry.retryRun({
+            projectId: command.request.projectId,
+            sourceRunId: command.request.sourceRunId,
+            mutationId: command.request.mutationId,
+            expectedRunVersion: command.request.expectedRunVersion,
+            expectedRunStatus: command.request.expectedRunStatus,
+            runId: adapters.randomUuid(),
+            attemptId: adapters.randomUuid(),
+            createdEventId: adapters.randomUuid(),
+            queuedEventId: adapters.randomUuid(),
+            auditEventId: command.request.auditEventId,
+            requestId: command.request.requestId,
+            principal: authenticated.principal,
+            policyFence: decision.fence,
+          });
+          return Object.freeze({
+            schemaVersion: 1 as const,
+            operation: 'run.retry' as const,
+            retry: result,
+          });
+        } catch (error) {
+          try {
+            await database.securityAudit.record(
+              failureAudit(command, authenticated, error),
+            );
+          } catch (auditError) {
+            throw new LocalRunRetryCommandUnavailableError({
+              cause: auditError,
+            });
+          }
+          throw error;
+        }
+      } finally {
+        await database.close();
+      }
+    },
+  });
+}
+
+export function runLocalRunRetryCommandFile(
+  commandFilePath: string,
+): Promise<Readonly<LocalRunRetryCommandResult>> {
+  return createLocalRunRetryCommandRunner().run(commandFilePath);
+}
+
+export function isLocalRunRetryCommandError(error: unknown): boolean {
+  return (
+    error instanceof LocalRunRetryCommandConfigurationError ||
+    error instanceof LocalRunRetryCommandAuthorizationError ||
+    error instanceof LocalRunRetryCommandUnavailableError ||
+    error instanceof AuthenticatedLocalCommandAuthenticationError ||
+    error instanceof LocalSqliteAuthenticatedManagementFenceError ||
+    error instanceof RunManualRetryNotFoundError ||
+    error instanceof RunManualRetryFenceRejectedError ||
+    error instanceof RunManualRetryRateLimitedError ||
+    error instanceof RunManualRetryUnavailableError
+  );
+}

packages/ql3-local-owner-cli/test/productCli.test.cjs

@@ -32,7 +32,7 @@ function runCli(args) {
 
 test('catalog maps every product subcommand to an existing same-package binary', () => {
   assert.equal(manifest.bin.ql3, 'dist/product-cli/cli.js');
-  assert.equal(QINGLONG3_PRODUCT_COMMANDS.length, 20);
+  assert.equal(QINGLONG3_PRODUCT_COMMANDS.length, 21);
   assert.equal(
     new Set(QINGLONG3_PRODUCT_COMMANDS.map(({ name }) => name)).size,
     QINGLONG3_PRODUCT_COMMANDS.length,
@@ -60,6 +60,7 @@ test('help and version are bounded installation-derived product facts', () => {
   const help = qingLong3ProductHelp();
   assert.match(help, /^Usage: ql3 <command> \[arguments\]/);
   assert.match(help, /\n  task\s+manage Task definitions\n/);
+  assert.match(help, /\n  run\s+retry terminal Runs/);
   assert.match(help, /Root service mutation remains isolated/);
   assert.equal(help.includes('ql3-service-bridge  '), false);
   assert.equal(loadQingLong3ProductVersion(moduleDirectory), manifest.version);
@@ -153,6 +154,14 @@ test('product binary exposes help/version and delegates without a shell', () =>
   );
   assert.equal(delegatedHelp.stderr, '');
 
+  const delegatedRunHelp = runCli(['run', '--help']);
+  assert.equal(delegatedRunHelp.status, 0);
+  assert.equal(
+    delegatedRunHelp.stdout.trim(),
+    'Usage: ql3-run retry --command-file /absolute/private-command.json',
+  );
+  assert.equal(delegatedRunHelp.stderr, '');
+
   const delegatedFailure = runCli(['task']);
   assert.equal(delegatedFailure.status, 64);
   assert.equal(delegatedFailure.stdout, '');

packages/ql3-local-owner-cli/test/runRetryCommand.test.cjs

@@ -0,0 +1,216 @@
+const assert = require('node:assert/strict');
+const { spawnSync } = require('node:child_process');
+const { DatabaseSync } = require('node:sqlite');
+const path = require('node:path');
+const { test } = require('node:test');
+
+const {
+  RunManualRetryNotFoundError,
+} = require('@qinglong/runtime-core/run-manual-retry');
+const {
+  openLocalSqliteRuntimeDatabase,
+} = require('@qinglong/local-sqlite/runtime');
+const {
+  runLocalRunRetryCommandFile,
+} = require('../dist/run-management/runRetryCommand.js');
+const {
+  auditRows,
+  localManagementFixture,
+  writeCommand,
+} = require('./localManagementFixture.cjs');
+
+function uuid(value) {
+  return `019f9200-0000-4000-8000-${value.toString(16).padStart(12, '0')}`;
+}
+
+async function createFailedSource(value) {
+  const runtime = await openLocalSqliteRuntimeDatabase({
+    databasePath: value.databasePath,
+    profile: 'edge',
+  });
+  let started;
+  try {
+    const task = (
+      await runtime.taskDefinitions.appendTaskDefinitionRevision({
+        projectId: 'default',
+        taskId: 'retry-product-task',
+        expectedRevision: null,
+        mutationId: uuid(1),
+        name: 'Retry product task',
+        kind: 'command',
+        spec: {
+          schema: 'qinglong/command@v1',
+          config: {
+            command: {
+              kind: 'argv',
+              file: '/bin/echo',
+              args: ['retry-product'],
+            },
+          },
+        },
+        labels: {},
+        enabled: true,
+        occurredAtMs: value.now,
+      })
+    ).definition;
+    started = await (
+      await runtime.taskStartRepository()
+    ).startTask({
+      projectId: 'default',
+      taskId: task.taskId,
+      mutationId: uuid(2),
+      expectedRevision: task.revision,
+      expectedContentDigest: task.contentDigest,
+      runId: uuid(3),
+      attemptId: uuid(4),
+      createdEventId: uuid(5),
+      queuedEventId: uuid(6),
+      subject: { type: 'user', id: 'automation-user' },
+      policyFence: { projectVersion: 1, bindingVersion: 1 },
+    });
+  } finally {
+    await runtime.close();
+  }
+  const database = new DatabaseSync(value.databasePath);
+  try {
+    database
+      .prepare(
+        `UPDATE "Runs"
+            SET "status" = 'failed', "version" = 3, "event_sequence" = 3,
+                "finished_at_ms" = ?, "error_code" = 'PRODUCT_TEST_FAILURE',
+                "error_summary" = 'failed'
+          WHERE "id" = ?`,
+      )
+      .run(value.now + 10, started.runId);
+    database
+      .prepare(
+        `UPDATE "RunAttempts"
+            SET "status" = 'failed', "finished_at_ms" = ?,
+                "error_code" = 'PRODUCT_TEST_FAILURE',
+                "error_summary" = 'failed'
+          WHERE "id" = ?`,
+      )
+      .run(value.now + 10, started.attemptId);
+    database
+      .prepare(
+        `INSERT INTO "RunEvents" (
+           "id", "run_id", "sequence", "type", "dedupe_key",
+           "actor_type", "actor_id", "attempt_id", "payload",
+           "created_at_ms"
+         ) VALUES (?, ?, 3, 'run.failed', 'product-test-failed', 'executor',
+                   'product-test', ?, '{}', ?)`,
+      )
+      .run(uuid(7), started.runId, started.attemptId, value.now + 10);
+  } finally {
+    database.close();
+  }
+  return started;
+}
+
+function request(value, sourceRunId, overrides = {}) {
+  return {
+    projectId: 'default',
+    sourceRunId,
+    mutationId: uuid(10),
+    expectedRunVersion: 3,
+    expectedRunStatus: 'failed',
+    requestId: 'local-run-retry-product-1',
+    auditEventId: uuid(11),
+    failureAuditEventId: uuid(12),
+    occurredAtMs: value.now,
+    ...overrides,
+  };
+}
+
+test('runs a strongly authenticated manual retry and exactly replays it', async (t) => {
+  const value = await localManagementFixture(t);
+  const source = await createFailedSource(value);
+  const commandFile = writeCommand(
+    value,
+    'run.retry',
+    request(value, source.runId),
+    'run-retry',
+  );
+
+  const accepted = await runLocalRunRetryCommandFile(commandFile);
+  assert.equal(accepted.schemaVersion, 1);
+  assert.equal(accepted.operation, 'run.retry');
+  assert.equal(accepted.retry.status, 'accepted');
+  assert.equal(accepted.retry.retryOfRunId, source.runId);
+  assert.equal(accepted.retry.runStatus, 'queued');
+
+  const replay = await runLocalRunRetryCommandFile(commandFile);
+  assert.equal(replay.retry.status, 'existing');
+  assert.equal(replay.retry.runId, accepted.retry.runId);
+  assert.deepEqual(
+    auditRows(value.databasePath)
+      .filter(({ operationId }) => operationId === 'run.retry')
+      .map((row) => ({ ...row })),
+    [
+      {
+        eventId: uuid(11),
+        operationId: 'run.retry',
+        outcome: 'allowed',
+        reasonsJson: '["role_grant","strong_authentication"]',
+      },
+    ],
+  );
+});
+
+test('audits a missing source without creating Run state', async (t) => {
+  const value = await localManagementFixture(t);
+  const commandFile = writeCommand(
+    value,
+    'run.retry',
+    request(value, 'missing-run', {
+      mutationId: uuid(20),
+      requestId: 'local-run-retry-missing',
+      auditEventId: uuid(21),
+      failureAuditEventId: uuid(22),
+    }),
+    'run-retry-missing',
+  );
+  await assert.rejects(
+    runLocalRunRetryCommandFile(commandFile),
+    RunManualRetryNotFoundError,
+  );
+  assert.deepEqual(
+    auditRows(value.databasePath)
+      .filter(({ operationId }) => operationId === 'run.retry')
+      .map((row) => ({ ...row })),
+    [
+      {
+        eventId: uuid(22),
+        operationId: 'run.retry',
+        outcome: 'denied',
+        reasonsJson: '["run_not_found"]',
+      },
+    ],
+  );
+});
+
+test('binary exposes only the private command-file retry interface', () => {
+  const cli = path.join(
+    __dirname,
+    '..',
+    'dist',
+    'run-management',
+    'runRetryCli.js',
+  );
+  const help = spawnSync(process.execPath, [cli, '--help'], {
+    encoding: 'utf8',
+  });
+  assert.equal(help.status, 0);
+  assert.equal(
+    help.stdout.trim(),
+    'Usage: ql3-run retry --command-file /absolute/private-command.json',
+  );
+  const invalid = spawnSync(process.execPath, [cli, 'retry'], {
+    encoding: 'utf8',
+  });
+  assert.equal(invalid.status, 64);
+  assert.equal(
+    JSON.parse(invalid.stderr).code,
+    'LOCAL_RUN_RETRY_CLI_USAGE_INVALID',
+  );
+});