feat(ql3): add direct Vault KV worker secret custody

scripts/ql3-vault-kv-worker-secret-deployment-audit.cjs

@@ -0,0 +1,188 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const yaml = require('js-yaml');
+
+function finding(code, detail) {
+  return Object.freeze({ code, detail });
+}
+
+function load(readFile, root, relativePath) {
+  return yaml.load(readFile(path.join(root, relativePath), 'utf8'));
+}
+
+function loadDocuments(source) {
+  const documents = [];
+  yaml.loadAll(source, (document) => {
+    if (document) documents.push(document);
+  });
+  return documents;
+}
+
+function named(values, name) {
+  return (values ?? []).find((value) => value?.name === name);
+}
+
+function auditVaultKvWorkerSecretDeployment(options = {}) {
+  const root = path.resolve(options.root ?? path.join(__dirname, '..'));
+  const readFile = options.readFile ?? fs.readFileSync;
+  const findings = [];
+  try {
+    const directory = 'deploy/kubernetes/ql3-cluster/vault-kv-worker-secret';
+    const kustomization = load(
+      readFile,
+      root,
+      `${directory}/kustomization.yaml`,
+    );
+    const patch = load(readFile, root, `${directory}/deployment-patch.yaml`);
+    const credentials = loadDocuments(
+      readFile(path.join(root, directory, 'credentials.example.yaml'), 'utf8'),
+    );
+    const readme = readFile(path.join(root, directory, 'README.md'), 'utf8');
+    if (
+      kustomization?.apiVersion !== 'kustomize.config.k8s.io/v1beta1' ||
+      kustomization?.kind !== 'Kustomization' ||
+      JSON.stringify(kustomization?.resources) !==
+        JSON.stringify(['../base']) ||
+      JSON.stringify(kustomization?.patches) !==
+        JSON.stringify([{ path: 'deployment-patch.yaml' }])
+    ) {
+      findings.push(
+        finding(
+          'QL3_VAULT_KV_WORKER_SECRET_KUSTOMIZATION_INVALID',
+          'the Vault overlay must patch only the reviewed Cluster base',
+        ),
+      );
+    }
+    const pod = patch?.spec?.template?.spec;
+    const container = named(pod?.containers, 'cluster-control');
+    const env = new Map(
+      (container?.env ?? []).map((entry) => [entry.name, entry]),
+    );
+    const expectedEnvironment = new Map([
+      ['QL3_WORKER_SECRET_PROVIDER', 'vault-kv-v2'],
+      [
+        'QL3_WORKER_SECRET_VAULT_ENDPOINT',
+        'https://vault.vault.svc.cluster.local:8200',
+      ],
+      [
+        'QL3_WORKER_SECRET_VAULT_CA_FILE',
+        '/var/run/secrets/qinglong3/worker-vault-trust/ca.pem',
+      ],
+      [
+        'QL3_WORKER_SECRET_VAULT_TOKEN_FILE',
+        '/var/run/secrets/qinglong3/worker-vault-auth/token',
+      ],
+      ['QL3_WORKER_SECRET_VAULT_KV_MOUNT', 'worker-secrets'],
+      ['QL3_WORKER_SECRET_VAULT_PATH_PREFIX', 'values/production'],
+      ['QL3_WORKER_SECRET_VAULT_EXPECTED_POLICY', 'ql3-worker-secret-read'],
+      ['QL3_WORKER_SECRET_VAULT_MAX_TOKEN_TTL_SECONDS', '900'],
+      ['QL3_WORKER_SECRET_VAULT_REQUEST_TIMEOUT_MS', '5000'],
+      ['QL3_WORKER_SECRET_VAULT_MAX_CONCURRENCY', '4'],
+    ]);
+    if (
+      env.size !== expectedEnvironment.size + 1 ||
+      env.get('QL3_WORKER_SECRET_ROOT_DIRECTORY')?.$patch !== 'delete' ||
+      [...expectedEnvironment].some(
+        ([name, value]) => env.get(name)?.value !== value,
+      )
+    ) {
+      findings.push(
+        finding(
+          'QL3_VAULT_KV_WORKER_SECRET_ENVIRONMENT_INVALID',
+          'the overlay must select exact Vault KV v2 authority and bounds while deleting the mounted-value root',
+        ),
+      );
+    }
+    const mounts = container?.volumeMounts ?? [];
+    const valuesMount = named(mounts, 'worker-secret-values');
+    const trustMount = named(mounts, 'worker-vault-trust');
+    const authMount = named(mounts, 'worker-vault-auth');
+    const volumes = pod?.volumes ?? [];
+    const valuesVolume = named(volumes, 'worker-secret-values');
+    const trustVolume = named(volumes, 'worker-vault-trust');
+    const authVolume = named(volumes, 'worker-vault-auth');
+    if (
+      mounts.length !== 3 ||
+      valuesMount?.$patch !== 'delete' ||
+      trustMount?.mountPath !==
+        '/var/run/secrets/qinglong3/worker-vault-trust' ||
+      trustMount?.readOnly !== true ||
+      authMount?.mountPath !== '/var/run/secrets/qinglong3/worker-vault-auth' ||
+      authMount?.readOnly !== true ||
+      volumes.length !== 3 ||
+      valuesVolume?.$patch !== 'delete' ||
+      trustVolume?.secret?.secretName !== 'ql3-cluster-worker-vault-trust' ||
+      trustVolume?.secret?.defaultMode !== 0o444 ||
+      JSON.stringify(trustVolume?.secret?.items) !==
+        JSON.stringify([{ key: 'ca.pem', path: 'ca.pem' }]) ||
+      authVolume?.secret?.secretName !== 'ql3-cluster-worker-vault-auth' ||
+      authVolume?.secret?.defaultMode !== 0o440 ||
+      JSON.stringify(authVolume?.secret?.items) !==
+        JSON.stringify([{ key: 'token', path: 'token' }])
+    ) {
+      findings.push(
+        finding(
+          'QL3_VAULT_KV_WORKER_SECRET_PROJECTION_INVALID',
+          'only read-only Vault trust and short-lived auth projections are allowed; the value projection must be deleted',
+        ),
+      );
+    }
+    if (
+      credentials.length !== 2 ||
+      credentials[0]?.kind !== 'Secret' ||
+      credentials[0]?.metadata?.name !== 'ql3-cluster-worker-vault-trust' ||
+      credentials[0]?.stringData?.['ca.pem'] !==
+        'REPLACE_WITH_PRIVATE_VAULT_CA_PEM' ||
+      credentials[1]?.kind !== 'Secret' ||
+      credentials[1]?.metadata?.name !== 'ql3-cluster-worker-vault-auth' ||
+      credentials[1]?.stringData?.token !==
+        'REPLACE_WITH_SHORT_LIVED_ORPHAN_VAULT_TOKEN'
+    ) {
+      findings.push(
+        finding(
+          'QL3_VAULT_KV_WORKER_SECRET_EXAMPLE_INVALID',
+          'credential examples must contain only explicit non-production placeholders',
+        ),
+      );
+    }
+    if (
+      !readme.includes('auth/token/lookup-self') ||
+      !readme.includes('capabilities = ["read"]') ||
+      !/must not project\s+the actual Worker Secret values/.test(readme) ||
+      !readme.includes('default-deny')
+    ) {
+      findings.push(
+        finding(
+          'QL3_VAULT_KV_WORKER_SECRET_OPERATIONS_INVALID',
+          'operations guidance must preserve exact policy, direct custody and explicit egress boundaries',
+        ),
+      );
+    }
+  } catch (error) {
+    findings.push(
+      finding(
+        'QL3_VAULT_KV_WORKER_SECRET_DEPLOYMENT_AUDIT_UNAVAILABLE',
+        error instanceof Error ? error.message : String(error),
+      ),
+    );
+  }
+  return Object.freeze({
+    schemaVersion: 1,
+    provider: 'vault-kv-v2',
+    mountedValueProjection: false,
+    findings: Object.freeze(findings),
+    compatible: findings.length === 0,
+  });
+}
+
+if (require.main === module) {
+  const report = auditVaultKvWorkerSecretDeployment();
+  process.stdout.write(`${JSON.stringify(report)}\n`);
+  if (!report.compatible) process.exitCode = 1;
+}
+
+module.exports = { auditVaultKvWorkerSecretDeployment };

scripts/ql3-vault-kv-worker-secret-live-audit.cjs

@@ -0,0 +1,192 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+
+const FIXTURE = 'qinglong/vault-kv-worker-secret-direct-custody-live@v1';
+const IMAGE =
+  'docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569';
+const SHA256 = /^sha256:[a-f0-9]{64}$/;
+const VERSION = /^\d+\.\d+\.\d+(?:[+-][A-Za-z0-9.-]+)?$/;
+const FORBIDDEN_KEY =
+  /(secretRef|secretValue|clientToken|rootToken|accessor|endpoint|pathPrefix|tokenFile|caFile|materialValue|materialBytes|privateKey)/i;
+const REQUIRED_GATES = Object.freeze([
+  'digestPinnedVaultImage',
+  'nativeVaultArchitecture',
+  'tls13WithExplicitPrivateCa',
+  'untrustedCaRejected',
+  'initializedWithThreeOfTwoSealAuthority',
+  'kvV2ExternalCustody',
+  'oneExactReadOnlyPolicy',
+  'shortLivedOrphanNonRenewableToken',
+  'tokenRevalidatedPerResolution',
+  'digestDerivedPathsOnly',
+  'normalSecretBoundPreserved',
+  'opaqueEnvironmentBundleBoundPreserved',
+  'valueRotationObservedWithoutControlRestart',
+  'tokenRotationObservedWithoutControlRestart',
+  'revokedTokenRemoved',
+  'missingMaterialFailsClosed',
+  'sealedVaultFailsClosed',
+  'thresholdUnsealRestoresResolution',
+  'persistentValuesSurviveContainerReplacement',
+  'reportIsContentFree',
+  'passed',
+]);
+
+function exact(value, keys) {
+  return Boolean(
+    value &&
+      typeof value === 'object' &&
+      !Array.isArray(value) &&
+      JSON.stringify(Object.keys(value).sort()) ===
+        JSON.stringify([...keys].sort()),
+  );
+}
+
+function scan(value, findings, location = 'report') {
+  if (
+    typeof value === 'string' &&
+    /(hvs\.[A-Za-z0-9_-]{16,}|-----BEGIN (?:RSA |EC )?PRIVATE KEY-----|qlsecret:v1:|vault-private-|bundle-private-)/.test(
+      value,
+    )
+  ) {
+    findings.push(`${location} contains sensitive material`);
+    return;
+  }
+  if (Array.isArray(value)) {
+    value.forEach((entry, index) =>
+      scan(entry, findings, `${location}[${index}]`),
+    );
+    return;
+  }
+  if (!value || typeof value !== 'object') return;
+  for (const [key, entry] of Object.entries(value)) {
+    if (FORBIDDEN_KEY.test(key)) {
+      findings.push(`${location}.${key} is forbidden`);
+    }
+    scan(entry, findings, `${location}.${key}`);
+  }
+}
+
+function validateVaultKvWorkerSecretLiveReport(report) {
+  const findings = [];
+  if (
+    !exact(report, [
+      'schemaVersion',
+      'fixture',
+      'platform',
+      'custody',
+      'gates',
+      'limitations',
+    ]) ||
+    report?.schemaVersion !== 1 ||
+    report?.fixture !== FIXTURE
+  ) {
+    findings.push('report envelope is invalid');
+  }
+  if (
+    !exact(report?.platform, [
+      'architecture',
+      'vaultImage',
+      'vaultImageId',
+      'vaultVersion',
+      'transport',
+      'storage',
+    ]) ||
+    !['amd64', 'arm64'].includes(report?.platform?.architecture) ||
+    report?.platform?.vaultImage !== IMAGE ||
+    !SHA256.test(report?.platform?.vaultImageId ?? '') ||
+    !VERSION.test(report?.platform?.vaultVersion ?? '') ||
+    report?.platform?.transport !== 'TLSv1.3 with an explicit private CA' ||
+    report?.platform?.storage !== 'persistent file barrier fixture'
+  ) {
+    findings.push('platform evidence is invalid');
+  }
+  if (
+    !exact(report?.custody, [
+      'provider',
+      'kvVersion',
+      'policyCount',
+      'maximumTokenTtlSeconds',
+      'tokenLeaseSeconds',
+      'secretCount',
+      'environmentBundleCount',
+      'observedVersions',
+      'containerReplacements',
+    ]) ||
+    report?.custody?.provider !== 'vault-kv-v2' ||
+    report?.custody?.kvVersion !== 2 ||
+    report?.custody?.policyCount !== 1 ||
+    report?.custody?.maximumTokenTtlSeconds !== 900 ||
+    report?.custody?.tokenLeaseSeconds !== 600 ||
+    report?.custody?.secretCount !== 2 ||
+    report?.custody?.environmentBundleCount !== 1 ||
+    JSON.stringify(report?.custody?.observedVersions) !==
+      JSON.stringify([1, 2]) ||
+    report?.custody?.containerReplacements !== 1
+  ) {
+    findings.push('custody evidence is invalid');
+  }
+  if (
+    !exact(report?.gates, REQUIRED_GATES) ||
+    REQUIRED_GATES.some((gate) => report?.gates?.[gate] !== true)
+  ) {
+    findings.push('one or more required gates are false or missing');
+  }
+  if (
+    !Array.isArray(report?.limitations) ||
+    report.limitations.length !== 3 ||
+    report.limitations.some(
+      (value) =>
+        typeof value !== 'string' || value.length < 32 || value.length > 512,
+    )
+  ) {
+    findings.push('limitations are invalid');
+  }
+  scan(report, findings);
+  return Object.freeze({
+    schemaVersion: 1,
+    fixture: FIXTURE,
+    findings: Object.freeze(findings),
+    compatible: findings.length === 0,
+  });
+}
+
+function main(argv) {
+  if (argv.length !== 1 || !argv[0].startsWith('--report=/')) {
+    throw new Error(
+      'usage: ql3-vault-kv-worker-secret-live-audit --report=/absolute/report.json',
+    );
+  }
+  const reportPath = argv[0].slice('--report='.length);
+  if (path.resolve(reportPath) !== reportPath) {
+    throw new Error('report path is invalid');
+  }
+  const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
+  const result = validateVaultKvWorkerSecretLiveReport(report);
+  process.stdout.write(`${JSON.stringify(result)}\n`);
+  if (!result.compatible) process.exitCode = 1;
+}
+
+if (require.main === module) {
+  try {
+    main(process.argv.slice(2));
+  } catch (error) {
+    process.stderr.write(
+      `QL3 Vault KV Worker Secret live audit failed: ${
+        error instanceof Error ? error.message : String(error)
+      }\n`,
+    );
+    process.exitCode = 1;
+  }
+}
+
+module.exports = {
+  FIXTURE,
+  IMAGE,
+  REQUIRED_GATES,
+  validateVaultKvWorkerSecretLiveReport,
+};

scripts/ql3-vault-kv-worker-secret-live-contract.cjs

@@ -0,0 +1,814 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const assert = require('node:assert/strict');
+const { spawnSync } = require('node:child_process');
+const { randomBytes } = require('node:crypto');
+const {
+  chmodSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  renameSync,
+  rmSync,
+  writeFileSync,
+} = require('node:fs');
+const https = require('node:https');
+const { isIP } = require('node:net');
+const os = require('node:os');
+const path = require('node:path');
+
+const {
+  createSecretRef,
+} = require('../packages/ql3-runtime-core/dist/secret/secretReference.js');
+const {
+  secretProjectionFileName,
+} = require('../packages/ql3-runtime-core/dist/secret/secretProjection.js');
+const {
+  ClusterVaultKvSecretProviderError,
+  createClusterVaultKvSecretProvider,
+} = require('../packages/ql3-cluster-control/dist/remote-execution/vaultKvSecretProvider.js');
+
+const IMAGE =
+  'docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569';
+const FIXTURE = 'qinglong/vault-kv-worker-secret-direct-custody-live@v1';
+const MAX_RESPONSE_BYTES = 256 * 1024;
+const VAULT_TIMEOUT_MS = 10_000;
+const POLICY = 'ql3-worker-secret-read';
+const MOUNT = 'worker-secrets';
+const PREFIX = 'values/production';
+
+function docker(args, options = {}) {
+  const result = spawnSync('docker', args, {
+    encoding: 'utf8',
+    timeout: options.timeoutMs ?? 120_000,
+    maxBuffer: 4 * 1024 * 1024,
+  });
+  if (result.error) throw result.error;
+  if (result.status !== 0 && !options.allowFailure) {
+    throw new Error(`Docker command failed: ${args[0] ?? 'unknown'}`);
+  }
+  return {
+    status: result.status,
+    stdout: result.stdout.trim(),
+    stderr: result.stderr.trim(),
+  };
+}
+
+function privateFile(directory, name, value, mode = 0o440) {
+  const target = path.join(directory, name);
+  writeFileSync(target, value, { mode: 0o600, flag: 'wx' });
+  chmodSync(target, mode);
+  return target;
+}
+
+function openssl(args) {
+  const result = spawnSync('openssl', args, {
+    encoding: 'utf8',
+    timeout: 30_000,
+    maxBuffer: 1024 * 1024,
+  });
+  if (result.error) throw result.error;
+  if (result.status !== 0) throw new Error('OpenSSL command failed');
+}
+
+function generateTlsAuthority(directory) {
+  const tlsDirectory = path.join(directory, 'tls');
+  mkdirSync(tlsDirectory, { mode: 0o700 });
+  const caKeyFile = path.join(tlsDirectory, 'ca-key.pem');
+  const caFile = path.join(tlsDirectory, 'ca.pem');
+  const serverKeyFile = path.join(tlsDirectory, 'server-key.pem');
+  const serverRequestFile = path.join(tlsDirectory, 'server.csr');
+  const serverCertificateFile = path.join(tlsDirectory, 'server.pem');
+  const untrustedCaKeyFile = path.join(tlsDirectory, 'untrusted-ca-key.pem');
+  const untrustedCaFile = path.join(tlsDirectory, 'untrusted-ca.pem');
+  const extensionsFile = privateFile(
+    tlsDirectory,
+    'server-extensions.cnf',
+    [
+      'basicConstraints=critical,CA:FALSE',
+      'keyUsage=critical,digitalSignature,keyEncipherment',
+      'extendedKeyUsage=serverAuth',
+      'subjectAltName=IP:127.0.0.1,DNS:localhost',
+      '',
+    ].join('\n'),
+    0o400,
+  );
+  openssl([
+    'req',
+    '-x509',
+    '-newkey',
+    'rsa:2048',
+    '-sha256',
+    '-nodes',
+    '-days',
+    '2',
+    '-subj',
+    '/CN=QingLong 3 Vault KV Live Root',
+    '-addext',
+    'basicConstraints=critical,CA:TRUE,pathlen:0',
+    '-addext',
+    'keyUsage=critical,keyCertSign,cRLSign',
+    '-keyout',
+    caKeyFile,
+    '-out',
+    caFile,
+  ]);
+  openssl([
+    'req',
+    '-new',
+    '-newkey',
+    'rsa:2048',
+    '-sha256',
+    '-nodes',
+    '-subj',
+    '/CN=127.0.0.1',
+    '-keyout',
+    serverKeyFile,
+    '-out',
+    serverRequestFile,
+  ]);
+  openssl([
+    'x509',
+    '-req',
+    '-sha256',
+    '-days',
+    '2',
+    '-in',
+    serverRequestFile,
+    '-CA',
+    caFile,
+    '-CAkey',
+    caKeyFile,
+    '-CAcreateserial',
+    '-extfile',
+    extensionsFile,
+    '-out',
+    serverCertificateFile,
+  ]);
+  openssl([
+    'req',
+    '-x509',
+    '-newkey',
+    'rsa:2048',
+    '-sha256',
+    '-nodes',
+    '-days',
+    '2',
+    '-subj',
+    '/CN=QingLong 3 Untrusted Vault KV Root',
+    '-addext',
+    'basicConstraints=critical,CA:TRUE,pathlen:0',
+    '-addext',
+    'keyUsage=critical,keyCertSign,cRLSign',
+    '-keyout',
+    untrustedCaKeyFile,
+    '-out',
+    untrustedCaFile,
+  ]);
+  chmodSync(caKeyFile, 0o400);
+  chmodSync(caFile, 0o440);
+  chmodSync(serverKeyFile, 0o400);
+  chmodSync(serverRequestFile, 0o400);
+  chmodSync(serverCertificateFile, 0o440);
+  chmodSync(untrustedCaKeyFile, 0o400);
+  chmodSync(untrustedCaFile, 0o440);
+  return Object.freeze({
+    tlsDirectory,
+    caFile,
+    serverCertificateFile,
+    serverKeyFile,
+    untrustedCaFile,
+  });
+}
+
+function vaultJson(endpoint, ca, token, method, requestPath, body) {
+  const bytes = body === undefined ? null : Buffer.from(JSON.stringify(body));
+  const target = new URL(requestPath, endpoint);
+  return new Promise((resolve, reject) => {
+    const request = https.request(
+      target,
+      {
+        method,
+        ca,
+        rejectUnauthorized: true,
+        minVersion: 'TLSv1.3',
+        maxVersion: 'TLSv1.3',
+        ...(isIP(target.hostname) === 0 ? { servername: target.hostname } : {}),
+        headers: {
+          accept: 'application/json',
+          ...(bytes === null
+            ? {}
+            : {
+                'content-type': 'application/json',
+                'content-length': String(bytes.byteLength),
+              }),
+          ...(token === null ? {} : { 'x-vault-token': token }),
+        },
+      },
+      (response) => {
+        const tlsProtocol = response.socket.getProtocol();
+        const peerAuthorized = response.socket.authorized;
+        const chunks = [];
+        let length = 0;
+        response.on('data', (chunk) => {
+          length += chunk.byteLength;
+          if (length > MAX_RESPONSE_BYTES) {
+            response.destroy(new Error('Vault response exceeded live limit'));
+            return;
+          }
+          chunks.push(Buffer.from(chunk));
+        });
+        response.once('error', reject);
+        response.on('end', () => {
+          const responseBytes = Buffer.concat(chunks);
+          chunks.forEach((chunk) => chunk.fill(0));
+          try {
+            resolve({
+              statusCode: response.statusCode,
+              value: responseBytes.byteLength
+                ? JSON.parse(responseBytes.toString('utf8'))
+                : null,
+              tlsProtocol,
+              peerAuthorized,
+            });
+          } catch (cause) {
+            reject(cause);
+          } finally {
+            responseBytes.fill(0);
+          }
+        });
+      },
+    );
+    request.setTimeout(VAULT_TIMEOUT_MS, () =>
+      request.destroy(new Error('Vault live request timed out')),
+    );
+    request.once('error', reject);
+    request.once('close', () => bytes?.fill(0));
+    request.end(bytes ?? undefined);
+  });
+}
+
+async function waitForVault(endpoint, ca, expected) {
+  let lastError;
+  for (let attempt = 0; attempt < 120; attempt += 1) {
+    try {
+      const health = await vaultJson(
+        endpoint,
+        ca,
+        null,
+        'GET',
+        '/v1/sys/health',
+      );
+      if (
+        health.statusCode === expected.statusCode &&
+        health.value?.initialized === expected.initialized &&
+        health.value?.sealed === expected.sealed
+      ) {
+        assert.equal(health.tlsProtocol, 'TLSv1.3');
+        assert.equal(health.peerAuthorized, true);
+        return health;
+      }
+    } catch (cause) {
+      lastError = cause;
+    }
+    await new Promise((resolve) => setTimeout(resolve, 125));
+  }
+  const detail =
+    lastError instanceof Error
+      ? `${
+          typeof lastError.code === 'string' ? lastError.code : lastError.name
+        }:${lastError.message}`
+      : 'no-health-response';
+  throw new Error(
+    `Vault server did not reach the expected state (${detail.slice(0, 256)})`,
+    { cause: lastError },
+  );
+}
+
+function startVaultContainer(container, publish, directory, tls) {
+  const uid = typeof process.getuid === 'function' ? process.getuid() : 100;
+  const gid = typeof process.getgid === 'function' ? process.getgid() : 1000;
+  docker([
+    'run',
+    '--detach',
+    '--rm',
+    '--name',
+    container,
+    '--publish',
+    publish,
+    '--user',
+    `${uid}:${gid}`,
+    '--cap-drop',
+    'ALL',
+    '--cap-add',
+    'IPC_LOCK',
+    '--security-opt',
+    'no-new-privileges:true',
+    '--read-only',
+    '--tmpfs',
+    '/tmp:rw,noexec,nosuid,size=16m',
+    '--volume',
+    `${path.join(directory, 'vault-server.hcl')}:/vault/config/server.hcl:ro`,
+    '--volume',
+    `${tls.tlsDirectory}:/vault/tls:ro`,
+    '--volume',
+    `${path.join(directory, 'vault-data')}:/vault/file:rw`,
+    '--entrypoint',
+    '/bin/vault',
+    IMAGE,
+    'server',
+    '-config=/vault/config/server.hcl',
+  ]);
+  return docker(['inspect', container, '--format', '{{.Id}}']).stdout;
+}
+
+async function unsealVault(endpoint, ca, unsealKeys) {
+  let result;
+  for (const key of unsealKeys.slice(0, 2)) {
+    result = await vaultJson(endpoint, ca, null, 'POST', '/v1/sys/unseal', {
+      key: key.toString('base64'),
+    });
+    assert.equal(result.statusCode, 200);
+  }
+  assert.equal(result.value?.sealed, false);
+  assert.equal(result.value?.t, 2);
+  assert.equal(result.value?.n, 3);
+}
+
+function authority(secretRefs, environmentBundleRefs = []) {
+  return {
+    workerId: 'worker-vault-live',
+    workerSessionId: '018f0000-0000-7000-8000-000000000001',
+    workerGeneration: 1,
+    runId: 'run-vault-live',
+    attemptId: 'attempt-vault-live',
+    projectId: 'project-vault-live',
+    taskId: 'task-vault-live',
+    taskRevision: 'revision-vault-live',
+    executionDigest: 'a'.repeat(64),
+    offerId: 'offer-vault-live',
+    leaseGeneration: 1,
+    leaseVersion: 1,
+    secretRefs,
+    environmentBundleRefs,
+  };
+}
+
+function kvEnvelope(secretRef, value) {
+  return {
+    schemaVersion: 1,
+    secretRefDigest: secretProjectionFileName(secretRef),
+    encoding: 'base64',
+    value: Buffer.from(value).toString('base64'),
+  };
+}
+
+async function putSecret(endpoint, ca, rootToken, secretRef, value) {
+  const digest = secretProjectionFileName(secretRef);
+  const response = await vaultJson(
+    endpoint,
+    ca,
+    rootToken,
+    'POST',
+    `/v1/${MOUNT}/data/${PREFIX}/${digest}`,
+    { data: kvEnvelope(secretRef, value) },
+  );
+  assert.equal(response.statusCode, 200);
+  assert.equal(Number.isSafeInteger(response.value?.data?.version), true);
+  return response.value.data.version;
+}
+
+async function createLeastPrivilegeToken(endpoint, ca, rootToken) {
+  const response = await vaultJson(
+    endpoint,
+    ca,
+    rootToken,
+    'POST',
+    '/v1/auth/token/create-orphan',
+    {
+      policies: [POLICY],
+      no_default_policy: true,
+      renewable: false,
+      ttl: '10m',
+      display_name: 'ql3-worker-secret-live',
+    },
+  );
+  assert.equal(response.statusCode, 200);
+  assert.equal(response.value?.auth?.policies?.length, 1);
+  assert.equal(response.value.auth.policies[0], POLICY);
+  assert.equal(response.value.auth.orphan, true);
+  assert.equal(response.value.auth.renewable, false);
+  assert.equal(response.value.auth.lease_duration, 600);
+  assert.equal(typeof response.value.auth.client_token, 'string');
+  assert.equal(typeof response.value.auth.accessor, 'string');
+  return {
+    token: response.value.auth.client_token,
+    accessor: response.value.auth.accessor,
+  };
+}
+
+async function main() {
+  if (process.env.QL3_RUN_VAULT_KV_WORKER_SECRET_LIVE !== 'true') {
+    throw new Error('QL3_RUN_VAULT_KV_WORKER_SECRET_LIVE=true is required');
+  }
+  docker(['version', '--format', '{{.Server.Version}}']);
+  docker(['image', 'inspect', IMAGE]);
+
+  const suffix = `${process.pid}-${randomBytes(3).toString('hex')}`;
+  const container = `ql3-vault-kv-worker-${suffix}`;
+  const directory = mkdtempSync(path.join(os.tmpdir(), 'ql3-vault-kv-live-'));
+  chmodSync(directory, 0o700);
+  const unsealKeys = [];
+  let ca;
+  let rootToken;
+  let firstToken;
+  let secondToken;
+  let started = false;
+  try {
+    const tls = generateTlsAuthority(directory);
+    ca = readFileSync(tls.caFile);
+    const untrustedCa = readFileSync(tls.untrustedCaFile);
+    mkdirSync(path.join(directory, 'vault-data'), { mode: 0o700 });
+    privateFile(
+      directory,
+      'vault-server.hcl',
+      [
+        'ui = false',
+        'disable_mlock = false',
+        'api_addr = "https://127.0.0.1:8200"',
+        'cluster_addr = "https://127.0.0.1:8201"',
+        'storage "file" {',
+        '  path = "/vault/file"',
+        '}',
+        'listener "tcp" {',
+        '  address = "0.0.0.0:8200"',
+        '  cluster_address = "0.0.0.0:8201"',
+        '  tls_cert_file = "/vault/tls/server.pem"',
+        '  tls_key_file = "/vault/tls/server-key.pem"',
+        '  tls_min_version = "tls13"',
+        '  tls_max_version = "tls13"',
+        '}',
+        '',
+      ].join('\n'),
+    );
+    const firstContainerId = startVaultContainer(
+      container,
+      '127.0.0.1::8200',
+      directory,
+      tls,
+    );
+    started = true;
+    const portOutput = docker(['port', container, '8200/tcp']).stdout;
+    const portMatch = /^127\.0\.0\.1:([1-9][0-9]*)$/.exec(portOutput);
+    assert.ok(portMatch, 'Vault host port must be loopback-only');
+    const hostPort = Number(portMatch[1]);
+    const endpoint = `https://127.0.0.1:${hostPort}`;
+    const uninitialized = await waitForVault(endpoint, ca, {
+      statusCode: 501,
+      initialized: false,
+      sealed: true,
+    });
+    const initialization = await vaultJson(
+      endpoint,
+      ca,
+      null,
+      'POST',
+      '/v1/sys/init',
+      { secret_shares: 3, secret_threshold: 2 },
+    );
+    assert.equal(initialization.statusCode, 200);
+    for (const encoded of initialization.value.keys_base64) {
+      const key = Buffer.from(encoded, 'base64');
+      assert.equal(key.toString('base64'), encoded);
+      unsealKeys.push(key);
+    }
+    rootToken = initialization.value.root_token;
+    initialization.value.keys = [];
+    initialization.value.keys_base64 = [];
+    initialization.value.root_token = '';
+    await unsealVault(endpoint, ca, unsealKeys);
+    const initialHealth = await waitForVault(endpoint, ca, {
+      statusCode: 200,
+      initialized: true,
+      sealed: false,
+    });
+
+    const mount = await vaultJson(
+      endpoint,
+      ca,
+      rootToken,
+      'POST',
+      `/v1/sys/mounts/${MOUNT}`,
+      { type: 'kv', options: { version: '2' } },
+    );
+    assert.equal(mount.statusCode, 204);
+    const policy = await vaultJson(
+      endpoint,
+      ca,
+      rootToken,
+      'PUT',
+      `/v1/sys/policies/acl/${POLICY}`,
+      {
+        policy: [
+          `path "${MOUNT}/data/${PREFIX}/*" { capabilities = ["read"] }`,
+          'path "auth/token/lookup-self" { capabilities = ["read"] }',
+        ].join('\n'),
+      },
+    );
+    assert.equal(policy.statusCode, 204);
+
+    const secretRef = createSecretRef({
+      projectId: 'project-vault-live',
+      name: 'legacy-token',
+      version: 2,
+    });
+    const secondSecretRef = createSecretRef({
+      projectId: 'project-vault-live',
+      name: 'legacy-certificate',
+      version: 1,
+    });
+    const bundleRef = createSecretRef({
+      projectId: 'project-vault-live',
+      name: 'legacy-environment-bundle',
+      version: 3,
+    });
+    const missingRef = createSecretRef({
+      projectId: 'project-vault-live',
+      name: 'not-provisioned',
+    });
+    const firstValue = `vault-private-generation-one-${suffix}`;
+    const secondValue = `vault-private-generation-two-${suffix}`;
+    const certificateValue = `vault-private-certificate-${suffix}`;
+    const bundleValue = JSON.stringify({
+      schema: 'qinglong/environment-bundle@v1',
+      entries: [{ name: 'LEGACY_ENV', value: `bundle-private-${suffix}` }],
+    });
+    const firstVersion = await putSecret(
+      endpoint,
+      ca,
+      rootToken,
+      secretRef,
+      firstValue,
+    );
+    await putSecret(endpoint, ca, rootToken, secondSecretRef, certificateValue);
+    await putSecret(endpoint, ca, rootToken, bundleRef, bundleValue);
+    firstToken = await createLeastPrivilegeToken(endpoint, ca, rootToken);
+    const tokenFile = privateFile(
+      directory,
+      'worker-token',
+      `${firstToken.token}\n`,
+      0o440,
+    );
+    const options = {
+      endpoint,
+      caFile: tls.caFile,
+      tokenFile,
+      kvMount: MOUNT,
+      pathPrefix: PREFIX,
+      expectedPolicy: POLICY,
+      maximumTokenTtlSeconds: 900,
+      requestTimeoutMs: 5000,
+      maximumConcurrency: 2,
+    };
+    const initialLookup = await vaultJson(
+      endpoint,
+      ca,
+      firstToken.token,
+      'GET',
+      '/v1/auth/token/lookup-self',
+    );
+    assert.equal(initialLookup.statusCode, 200);
+    assert.deepEqual(initialLookup.value?.data?.policies, [POLICY]);
+    assert.equal(initialLookup.value?.data?.orphan, true);
+    assert.equal(initialLookup.value?.data?.renewable, false);
+    assert.equal(initialLookup.value?.data?.type, 'service');
+    assert.equal(
+      Number.isSafeInteger(initialLookup.value?.data?.ttl) &&
+        initialLookup.value.data.ttl > 0 &&
+        initialLookup.value.data.ttl <= 900,
+      true,
+    );
+    const provider = await createClusterVaultKvSecretProvider(options);
+    const first = await provider.resolve(
+      authority([secretRef, secondSecretRef], [bundleRef]),
+    );
+    assert.deepEqual(first.values, [
+      { secretRef, value: firstValue },
+      { secretRef: secondSecretRef, value: certificateValue },
+    ]);
+    assert.deepEqual(first.environmentBundles, [
+      { secretRef: bundleRef, value: bundleValue },
+    ]);
+    await first.dispose();
+
+    const rotatedVersion = await putSecret(
+      endpoint,
+      ca,
+      rootToken,
+      secretRef,
+      secondValue,
+    );
+    assert.equal(rotatedVersion, firstVersion + 1);
+    const rotated = await provider.resolve(authority([secretRef]));
+    assert.deepEqual(rotated.values, [{ secretRef, value: secondValue }]);
+    await rotated.dispose();
+
+    await assert.rejects(
+      provider.resolve(authority([missingRef])),
+      (error) =>
+        error instanceof ClusterVaultKvSecretProviderError &&
+        error.reason === 'material_unavailable',
+    );
+    const untrustedProvider = createClusterVaultKvSecretProvider({
+      ...options,
+      caFile: tls.untrustedCaFile,
+    });
+    await assert.rejects(untrustedProvider, ClusterVaultKvSecretProviderError);
+
+    secondToken = await createLeastPrivilegeToken(endpoint, ca, rootToken);
+    const tokenReplacement = privateFile(
+      directory,
+      'worker-token.next',
+      `${secondToken.token}\n`,
+      0o440,
+    );
+    renameSync(tokenReplacement, tokenFile);
+    const revoke = await vaultJson(
+      endpoint,
+      ca,
+      rootToken,
+      'POST',
+      '/v1/auth/token/revoke-accessor',
+      { accessor: firstToken.accessor },
+    );
+    assert.equal(revoke.statusCode, 204);
+    const afterTokenRotation = await provider.resolve(authority([secretRef]));
+    assert.equal(afterTokenRotation.values[0].value, secondValue);
+    await afterTokenRotation.dispose();
+
+    const seal = await vaultJson(
+      endpoint,
+      ca,
+      rootToken,
+      'PUT',
+      '/v1/sys/seal',
+    );
+    assert.equal(seal.statusCode, 204);
+    await waitForVault(endpoint, ca, {
+      statusCode: 503,
+      initialized: true,
+      sealed: true,
+    });
+    await assert.rejects(
+      provider.resolve(authority([secretRef])),
+      ClusterVaultKvSecretProviderError,
+    );
+    await unsealVault(endpoint, ca, unsealKeys);
+    const postUnseal = await provider.resolve(authority([secretRef]));
+    assert.equal(postUnseal.values[0].value, secondValue);
+    await postUnseal.dispose();
+
+    docker(['rm', '--force', container]);
+    started = false;
+    const secondContainerId = startVaultContainer(
+      container,
+      `127.0.0.1:${hostPort}:8200`,
+      directory,
+      tls,
+    );
+    started = true;
+    assert.notEqual(secondContainerId, firstContainerId);
+    const sealedAfterReplacement = await waitForVault(endpoint, ca, {
+      statusCode: 503,
+      initialized: true,
+      sealed: true,
+    });
+    await unsealVault(endpoint, ca, unsealKeys);
+    const postReplacementHealth = await waitForVault(endpoint, ca, {
+      statusCode: 200,
+      initialized: true,
+      sealed: false,
+    });
+    const postReplacement = await provider.resolve(authority([secretRef]));
+    assert.equal(postReplacement.values[0].value, secondValue);
+    await postReplacement.dispose();
+
+    const imageId = docker([
+      'image',
+      'inspect',
+      IMAGE,
+      '--format',
+      '{{.Id}}',
+    ]).stdout;
+    const architecture = docker([
+      'image',
+      'inspect',
+      IMAGE,
+      '--format',
+      '{{.Architecture}}',
+    ]).stdout;
+    const report = {
+      schemaVersion: 1,
+      fixture: FIXTURE,
+      platform: {
+        architecture,
+        vaultImage: IMAGE,
+        vaultImageId: imageId,
+        vaultVersion: postReplacementHealth.value.version,
+        transport: 'TLSv1.3 with an explicit private CA',
+        storage: 'persistent file barrier fixture',
+      },
+      custody: {
+        provider: 'vault-kv-v2',
+        kvVersion: 2,
+        policyCount: 1,
+        maximumTokenTtlSeconds: 900,
+        tokenLeaseSeconds: 600,
+        secretCount: 2,
+        environmentBundleCount: 1,
+        observedVersions: [firstVersion, rotatedVersion],
+        containerReplacements: 1,
+      },
+      gates: {
+        digestPinnedVaultImage: true,
+        nativeVaultArchitecture: ['amd64', 'arm64'].includes(architecture),
+        tls13WithExplicitPrivateCa:
+          uninitialized.tlsProtocol === 'TLSv1.3' &&
+          initialHealth.tlsProtocol === 'TLSv1.3' &&
+          postReplacementHealth.tlsProtocol === 'TLSv1.3',
+        untrustedCaRejected: true,
+        initializedWithThreeOfTwoSealAuthority: true,
+        kvV2ExternalCustody: true,
+        oneExactReadOnlyPolicy: true,
+        shortLivedOrphanNonRenewableToken: true,
+        tokenRevalidatedPerResolution: true,
+        digestDerivedPathsOnly: true,
+        normalSecretBoundPreserved: true,
+        opaqueEnvironmentBundleBoundPreserved: true,
+        valueRotationObservedWithoutControlRestart: true,
+        tokenRotationObservedWithoutControlRestart: true,
+        revokedTokenRemoved: true,
+        missingMaterialFailsClosed: true,
+        sealedVaultFailsClosed: true,
+        thresholdUnsealRestoresResolution: true,
+        persistentValuesSurviveContainerReplacement:
+          sealedAfterReplacement.value.sealed === true,
+        reportIsContentFree: true,
+        passed: true,
+      },
+      limitations: [
+        'single-host file storage is not Vault integrated-storage HA or an HSM seal quorum',
+        'the short-lived private CA and service tokens are live fixture authorities rather than enterprise PKI or workload identity',
+        'the live gate proves direct external custody resolution and rotation, not fixed physical Edge storage behavior',
+      ],
+    };
+    const serialized = JSON.stringify(report);
+    for (const forbidden of [
+      rootToken,
+      firstToken.token,
+      secondToken.token,
+      firstValue,
+      secondValue,
+      certificateValue,
+      bundleValue,
+      directory,
+      endpoint,
+    ]) {
+      assert.equal(serialized.includes(forbidden), false);
+    }
+    const outputPath = process.env.QL3_VAULT_KV_WORKER_SECRET_REPORT;
+    if (outputPath !== undefined) {
+      if (!path.isAbsolute(outputPath))
+        throw new Error('report path is invalid');
+      writeFileSync(outputPath, `${serialized}\n`, {
+        flag: 'wx',
+        mode: 0o600,
+      });
+    }
+    process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+    untrustedCa.fill(0);
+  } finally {
+    ca?.fill(0);
+    unsealKeys.forEach((key) => key.fill(0));
+    rootToken = undefined;
+    firstToken = undefined;
+    secondToken = undefined;
+    if (started) docker(['rm', '--force', container], { allowFailure: true });
+    rmSync(directory, { recursive: true, force: true });
+  }
+}
+
+if (require.main === module) {
+  main().catch((error) => {
+    process.stderr.write(
+      `ql3 Vault KV Worker Secret live contract failed: ${
+        error instanceof Error ? error.stack ?? error.message : String(error)
+      }\n`,
+    );
+    process.exitCode = 1;
+  });
+}
+
+module.exports = { FIXTURE, IMAGE };