#!/usr/bin/env node

'use strict';

const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { createHash, randomBytes } = require('node:crypto');
const { spawnSync } = require('node:child_process');
const yaml = require('js-yaml');

const {
  REGISTRY_EVENT_SCHEMA,
  createFixture,
} = require('./ql3-plugin-package-recovery-e2e-fixture.cjs');
const {
  CONTRACT_VERSION,
  MIGRATION_COUNT,
} = require('./ql3-plugin-package-recovery-e2e-live-audit.cjs');

const ROOT = path.resolve(__dirname, '..');
const FIXTURE_SCRIPT = path.join(
  __dirname,
  'ql3-plugin-package-recovery-e2e-fixture.cjs',
);
const NAMESPACE = 'ql3-plugin-recovery-e2e';
const CONTROL_SERVICE_ACCOUNT = 'ql3-cluster-control';
const RECOVERY_SERVICE_ACCOUNT = 'ql3-plugin-package-recovery';
const POSTGRES_NAME = 'ql3-postgres-live';
const REGISTRY_NAME = 'ql3-registry-live';
const REGISTRY_HOST = `${REGISTRY_NAME}.${NAMESPACE}.svc`;
const KIND_NODE_IMAGE =
  'kindest/node:v1.32.8@sha256:abd489f042d2b644e2d033f5c2d900bc707798d075e8186cb65e3f1367a9d5a1';
const POSTGRES_IMAGE = 'postgres:18.4-bookworm';
const POSTGRES_INDEX_DIGEST =
  'sha256:1961f96e6029a02c3812d7cb329a3b03a3ac2bb067058dec17b0f5596aca9296';
const POSTGRES_IMAGE_REFERENCE = `${POSTGRES_IMAGE}@${POSTGRES_INDEX_DIGEST}`;
const POSTGRES_REPOSITORY_DIGEST = `postgres@${POSTGRES_INDEX_DIGEST}`;
const POSTGRES_RUNTIME_IMAGE =
  'postgres:18.4-bookworm-ql3-plugin-recovery-e2e';
const DEFAULT_ADMIN_IMAGE = 'qinglong3-cluster-admin:ql3-plugin-recovery-e2e';
const DEFAULT_CONTROL_IMAGE =
  'qinglong3-cluster-control:ql3-plugin-recovery-e2e';
const REPORT_SCHEMA = 'qinglong/plugin-package-recovery-e2e-live-contract@v3';
const INITIAL_SEED_JOB = 'ql3-plugin-package-e2e-seed-initial';
const INITIAL_RECOVERY_JOB = 'ql3-plugin-package-recovery-initial';
const UPGRADE_SEED_JOB = 'ql3-plugin-package-e2e-seed-upgrade';
const UPGRADE_RECOVERY_JOB = 'ql3-plugin-package-recovery-reject-upgrade';
const SAFE_CLUSTER =
  /^ql3-plugin-recovery-e2e(?:-[a-z0-9](?:[-a-z0-9]{0,24}[a-z0-9])?)?$/;

const KIND = process.env.QL3_KIND_BIN || 'kind';
const KUBECTL = process.env.QL3_KUBECTL_BIN || 'kubectl';
const DOCKER = process.env.QL3_DOCKER_BIN || 'docker';
const OPENSSL = process.env.QL3_OPENSSL_BIN || 'openssl';
const ADMIN_IMAGE = process.env.QL3_ADMIN_IMAGE || DEFAULT_ADMIN_IMAGE;
const CONTROL_IMAGE = process.env.QL3_CONTROL_IMAGE || DEFAULT_CONTROL_IMAGE;

function fail(message) {
  throw new Error(message);
}

function commandLabel(binary, args) {
  return [path.basename(binary), ...args].join(' ');
}

function run(binary, args, options = {}) {
  if (!options.quiet) {
    process.stderr.write(`+ ${options.label || commandLabel(binary, args)}\n`);
  }
  const capture = options.capture === true;
  const result = spawnSync(binary, args, {
    cwd: options.cwd || ROOT,
    env: options.env || process.env,
    input: options.input,
    encoding: 'utf8',
    maxBuffer: 64 * 1024 * 1024,
    stdio: capture
      ? ['pipe', 'pipe', 'pipe']
      : [
          options.input === undefined ? 'inherit' : 'pipe',
          'inherit',
          'inherit',
        ],
  });
  if (result.error) throw result.error;
  if (result.status !== 0 && !options.allowFailure) {
    const detail = capture ? `\n${result.stderr || result.stdout || ''}` : '';
    fail(
      `${path.basename(binary)} exited with status ${String(
        result.status,
      )}${detail}`,
    );
  }
  return Object.freeze({
    status: result.status,
    stdout: capture ? result.stdout.trim() : '',
    stderr: capture ? result.stderr.trim() : '',
  });
}

function kind(args, options = {}) {
  return run(KIND, args, options);
}

let kubeconfig = '';

function kubectl(args, options = {}) {
  return run(KUBECTL, ['--kubeconfig', kubeconfig, ...args], options);
}

function kubectlJson(args) {
  const output = kubectl([...args, '-o', 'json'], {
    capture: true,
    quiet: true,
  }).stdout;
  return JSON.parse(output);
}

function apply(value, label) {
  kubectl(['apply', '-f', '-'], {
    input: `${JSON.stringify(value)}\n`,
    label,
  });
}

function exactClusterName() {
  const value =
    process.env.QL3_KIND_CLUSTER ??
    `ql3-plugin-recovery-e2e-${process.pid.toString(36)}`;
  if (!SAFE_CLUSTER.test(value)) {
    fail(
      'QL3_KIND_CLUSTER must be an exact ql3-plugin-recovery-e2e[-suffix] name',
    );
  }
  return value;
}

function kindCreateEnvironment(clusterName) {
  const required = [
    '127.0.0.1',
    'localhost',
    `${clusterName}-control-plane`,
    `${clusterName}-worker`,
    `${clusterName}-worker2`,
    '.svc',
    '.cluster.local',
    '10.96.0.0/12',
    '10.244.0.0/16',
    '172.16.0.0/12',
  ];
  const values = [process.env.NO_PROXY, process.env.no_proxy, ...required]
    .flatMap((value) => (value ?? '').split(','))
    .map((value) => value.trim())
    .filter(Boolean);
  const noProxy = [...new Set(values)].join(',');
  return { ...process.env, NO_PROXY: noProxy, no_proxy: noProxy };
}

function randomSecret() {
  return randomBytes(24).toString('base64url');
}

function architecture() {
  if (process.arch === 'x64') return 'amd64';
  if (process.arch === 'arm64') return 'arm64';
  fail(`unsupported live-gate architecture ${process.arch}`);
}

function readYamlDocuments(filePath) {
  const documents = [];
  yaml.loadAll(fs.readFileSync(filePath, 'utf8'), (value) => {
    if (value) documents.push(value);
  });
  return documents;
}

function imageExists(image) {
  return (
    run(DOCKER, ['image', 'inspect', image], {
      capture: true,
      quiet: true,
      allowFailure: true,
    }).status === 0
  );
}

function sourceRevision() {
  const value = process.env.QL3_SOURCE_REVISION ?? '';
  if (!/^[a-f0-9]{40}$/.test(value)) {
    fail(
      'QL3_SOURCE_REVISION must be the exact lowercase 40-hex source revision',
    );
  }
  return value;
}

function privateReportPath(argv) {
  if (
    argv.length !== 1 ||
    !argv[0].startsWith('--report=') ||
    !path.isAbsolute(argv[0].slice('--report='.length))
  ) {
    fail(
      'usage: ql3-plugin-package-recovery-e2e-live-contract ' +
        '--report=/absolute/private-report.json',
    );
  }
  const reportFile = argv[0].slice('--report='.length);
  if (path.resolve(reportFile) !== reportFile) {
    fail('Plugin Package recovery E2E report path must be canonical');
  }
  if (fs.existsSync(reportFile)) {
    fail('refusing to overwrite the Plugin Package recovery E2E report');
  }
  const parent = fs.lstatSync(path.dirname(reportFile));
  if (!parent.isDirectory() || parent.isSymbolicLink()) {
    fail('Plugin Package recovery E2E report parent must be a real directory');
  }
  return reportFile;
}

function writePrivateReport(reportFile, report) {
  const temporaryReport = path.join(
    path.dirname(reportFile),
    `.${path.basename(reportFile)}.${process.pid}.` +
      `${randomBytes(6).toString('hex')}.tmp`,
  );
  let descriptor;
  try {
    descriptor = fs.openSync(temporaryReport, 'wx', 0o600);
    fs.writeFileSync(descriptor, `${JSON.stringify(report, null, 2)}\n`);
    fs.fsyncSync(descriptor);
    fs.closeSync(descriptor);
    descriptor = undefined;
    fs.linkSync(temporaryReport, reportFile);
  } finally {
    if (descriptor !== undefined) fs.closeSync(descriptor);
    fs.rmSync(temporaryReport, { force: true });
  }
}

function buildImages(revision) {
  if (process.env.QL3_SKIP_IMAGE_BUILD === '1') {
    assert.equal(imageExists(ADMIN_IMAGE), true, `${ADMIN_IMAGE} is absent`);
    assert.equal(
      imageExists(CONTROL_IMAGE),
      true,
      `${CONTROL_IMAGE} is absent`,
    );
    return;
  }
  run(
    DOCKER,
    [
      'build',
      '--file',
      'deploy/containers/ql3-cluster-admin/Dockerfile',
      '--tag',
      ADMIN_IMAGE,
      '--build-arg',
      `SOURCE_REVISION=${revision}`,
      '.',
    ],
    { label: 'build current QingLong 3.0 cluster-admin image' },
  );
  run(
    DOCKER,
    [
      'build',
      '--file',
      'deploy/containers/ql3-cluster-control/Dockerfile',
      '--tag',
      CONTROL_IMAGE,
      '--build-arg',
      `SOURCE_REVISION=${revision}`,
      '.',
    ],
    { label: 'build current QingLong 3.0 cluster-control image' },
  );
}

function ensurePostgresImage() {
  if (!imageExists(POSTGRES_IMAGE_REFERENCE)) {
    run(DOCKER, ['pull', POSTGRES_IMAGE_REFERENCE], {
      label: 'pull digest-pinned PostgreSQL 18.4 fixture image',
    });
  }
  const inspection = JSON.parse(
    run(DOCKER, ['image', 'inspect', POSTGRES_IMAGE_REFERENCE], {
      capture: true,
      quiet: true,
    }).stdout,
  )[0];
  assert.ok(
    inspection.RepoDigests.includes(POSTGRES_REPOSITORY_DIGEST),
    `${POSTGRES_IMAGE} does not match ${POSTGRES_REPOSITORY_DIGEST}`,
  );
  run(DOCKER, ['image', 'tag', POSTGRES_IMAGE_REFERENCE, POSTGRES_RUNTIME_IMAGE], {
    label: 'bind verified PostgreSQL fixture digest to its Kind-local tag',
  });
  assert.equal(imageId(POSTGRES_RUNTIME_IMAGE), inspection.Id);
}

function imageId(image) {
  return JSON.parse(
    run(DOCKER, ['image', 'inspect', image], {
      capture: true,
      quiet: true,
    }).stdout,
  )[0].Id;
}

function imageSourceRevision(image) {
  return JSON.parse(
    run(DOCKER, ['image', 'inspect', image], {
      capture: true,
      quiet: true,
    }).stdout,
  )[0].Config?.Labels?.['org.opencontainers.image.revision'];
}

function createRegistryCertificate(root) {
  const caKey = path.join(root, 'registry-ca.key');
  const caCert = path.join(root, 'registry-ca.crt');
  const serverKey = path.join(root, 'registry-tls.key');
  const request = path.join(root, 'registry.csr');
  const serverCert = path.join(root, 'registry-tls.crt');
  const extensions = path.join(root, 'registry.ext');
  fs.writeFileSync(
    extensions,
    [
      'basicConstraints=critical,CA:FALSE',
      'keyUsage=critical,digitalSignature,keyEncipherment',
      'extendedKeyUsage=serverAuth',
      `subjectAltName=DNS:${REGISTRY_NAME},DNS:${REGISTRY_HOST},DNS:${REGISTRY_HOST}.cluster.local`,
      '',
    ].join('\n'),
    { mode: 0o600 },
  );
  run(
    OPENSSL,
    [
      'req',
      '-x509',
      '-newkey',
      'rsa:2048',
      '-sha256',
      '-nodes',
      '-days',
      '2',
      '-subj',
      '/CN=QingLong Plugin Recovery E2E Registry CA',
      '-keyout',
      caKey,
      '-out',
      caCert,
    ],
    { quiet: true, capture: true },
  );
  run(
    OPENSSL,
    [
      'req',
      '-newkey',
      'rsa:2048',
      '-sha256',
      '-nodes',
      '-subj',
      `/CN=${REGISTRY_HOST}`,
      '-keyout',
      serverKey,
      '-out',
      request,
    ],
    { quiet: true, capture: true },
  );
  run(
    OPENSSL,
    [
      'x509',
      '-req',
      '-sha256',
      '-days',
      '2',
      '-in',
      request,
      '-CA',
      caCert,
      '-CAkey',
      caKey,
      '-CAcreateserial',
      '-extfile',
      extensions,
      '-out',
      serverCert,
    ],
    { quiet: true, capture: true },
  );
  return Object.freeze({
    ca: fs.readFileSync(caCert, 'utf8'),
    cert: fs.readFileSync(serverCert, 'utf8'),
    key: fs.readFileSync(serverKey, 'utf8'),
  });
}

function commonSecurityContext() {
  return {
    runAsNonRoot: true,
    runAsUser: 10001,
    runAsGroup: 10001,
    fsGroup: 10001,
    seccompProfile: { type: 'RuntimeDefault' },
  };
}

function containerSecurityContext() {
  return {
    allowPrivilegeEscalation: false,
    readOnlyRootFilesystem: true,
    capabilities: { drop: ['ALL'] },
  };
}

function postgresResources(secrets) {
  const initScript = `#!/bin/sh
set -eu
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname postgres \\
  --set=migration_password="$QL3_MIGRATION_PASSWORD" \\
  --set=runtime_password="$QL3_RUNTIME_PASSWORD" \\
  --set=admin_password="$QL3_ADMIN_PASSWORD" \\
  --set=automation_manager_password="$QL3_AUTOMATION_MANAGER_PASSWORD" \\
  --set=approval_manager_password="$QL3_APPROVAL_MANAGER_PASSWORD" \\
  --set=run_manager_password="$QL3_RUN_MANAGER_PASSWORD" \\
  --set=package_manager_password="$QL3_PACKAGE_MANAGER_PASSWORD" \\
  --set=package_executor_password="$QL3_PACKAGE_EXECUTOR_PASSWORD" \\
  --set=worker_credential_manager_password="$QL3_WORKER_CREDENTIAL_MANAGER_PASSWORD" \\
  --set=worker_credential_executor_password="$QL3_WORKER_CREDENTIAL_EXECUTOR_PASSWORD" \\
  --set=worker_password="$QL3_WORKER_INGRESS_PASSWORD" <<'SQL'
CREATE ROLE ql3_migration LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD :'migration_password';
CREATE ROLE ql3_runtime LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD :'runtime_password';
CREATE ROLE ql3_admin LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD :'admin_password';
CREATE ROLE ql3_automation_manager LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD :'automation_manager_password';
CREATE ROLE ql3_approval_manager LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD :'approval_manager_password';
CREATE ROLE ql3_run_manager LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD :'run_manager_password';
CREATE ROLE ql3_package_manager LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD :'package_manager_password';
CREATE ROLE ql3_package_executor LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD :'package_executor_password';
CREATE ROLE ql3_worker_credential_manager LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD :'worker_credential_manager_password';
CREATE ROLE ql3_worker_credential_executor LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD :'worker_credential_executor_password';
CREATE ROLE ql3_worker_ingress LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD :'worker_password';
SQL
createdb --username "$POSTGRES_USER" --owner ql3_migration qinglong
`;
  return [
    {
      apiVersion: 'v1',
      kind: 'Secret',
      metadata: { name: 'ql3-e2e-postgres-auth', namespace: NAMESPACE },
      type: 'Opaque',
      stringData: {
        'postgres-password': secrets.postgres,
        'migration-password': secrets.migration,
        'runtime-password': secrets.runtime,
        'admin-password': secrets.admin,
        'automation-manager-password': secrets.automationManager,
        'approval-manager-password': secrets.approvalManager,
        'run-manager-password': secrets.runManager,
        'package-manager-password': secrets.packageManager,
        'package-executor-password': secrets.packageExecutor,
        'worker-credential-manager-password': secrets.workerCredentialManager,
        'worker-credential-executor-password': secrets.workerCredentialExecutor,
        'worker-ingress-password': secrets.worker,
      },
    },
    {
      apiVersion: 'v1',
      kind: 'ConfigMap',
      metadata: { name: 'ql3-e2e-postgres-init', namespace: NAMESPACE },
      data: { '001-roles.sh': initScript },
    },
    {
      apiVersion: 'v1',
      kind: 'Service',
      metadata: { name: POSTGRES_NAME, namespace: NAMESPACE },
      spec: {
        selector: { 'app.kubernetes.io/name': POSTGRES_NAME },
        ports: [{ name: 'postgres', port: 5432, targetPort: 5432 }],
      },
    },
    {
      apiVersion: 'v1',
      kind: 'Pod',
      metadata: {
        name: POSTGRES_NAME,
        namespace: NAMESPACE,
        labels: { 'app.kubernetes.io/name': POSTGRES_NAME },
      },
      spec: {
        automountServiceAccountToken: false,
        restartPolicy: 'Never',
        securityContext: { seccompProfile: { type: 'RuntimeDefault' } },
        containers: [
          {
            name: 'postgres',
            image: POSTGRES_RUNTIME_IMAGE,
            imagePullPolicy: 'Never',
            env: [
              { name: 'POSTGRES_USER', value: 'postgres' },
              {
                name: 'POSTGRES_PASSWORD',
                valueFrom: {
                  secretKeyRef: {
                    name: 'ql3-e2e-postgres-auth',
                    key: 'postgres-password',
                  },
                },
              },
              { name: 'POSTGRES_DB', value: 'postgres' },
              {
                name: 'QL3_MIGRATION_PASSWORD',
                valueFrom: {
                  secretKeyRef: {
                    name: 'ql3-e2e-postgres-auth',
                    key: 'migration-password',
                  },
                },
              },
              {
                name: 'QL3_RUNTIME_PASSWORD',
                valueFrom: {
                  secretKeyRef: {
                    name: 'ql3-e2e-postgres-auth',
                    key: 'runtime-password',
                  },
                },
              },
              {
                name: 'QL3_ADMIN_PASSWORD',
                valueFrom: {
                  secretKeyRef: {
                    name: 'ql3-e2e-postgres-auth',
                    key: 'admin-password',
                  },
                },
              },
              {
                name: 'QL3_AUTOMATION_MANAGER_PASSWORD',
                valueFrom: {
                  secretKeyRef: {
                    name: 'ql3-e2e-postgres-auth',
                    key: 'automation-manager-password',
                  },
                },
              },
              {
                name: 'QL3_APPROVAL_MANAGER_PASSWORD',
                valueFrom: {
                  secretKeyRef: {
                    name: 'ql3-e2e-postgres-auth',
                    key: 'approval-manager-password',
                  },
                },
              },
              {
                name: 'QL3_RUN_MANAGER_PASSWORD',
                valueFrom: {
                  secretKeyRef: {
                    name: 'ql3-e2e-postgres-auth',
                    key: 'run-manager-password',
                  },
                },
              },
              {
                name: 'QL3_PACKAGE_MANAGER_PASSWORD',
                valueFrom: {
                  secretKeyRef: {
                    name: 'ql3-e2e-postgres-auth',
                    key: 'package-manager-password',
                  },
                },
              },
              {
                name: 'QL3_PACKAGE_EXECUTOR_PASSWORD',
                valueFrom: {
                  secretKeyRef: {
                    name: 'ql3-e2e-postgres-auth',
                    key: 'package-executor-password',
                  },
                },
              },
              {
                name: 'QL3_WORKER_CREDENTIAL_MANAGER_PASSWORD',
                valueFrom: {
                  secretKeyRef: {
                    name: 'ql3-e2e-postgres-auth',
                    key: 'worker-credential-manager-password',
                  },
                },
              },
              {
                name: 'QL3_WORKER_CREDENTIAL_EXECUTOR_PASSWORD',
                valueFrom: {
                  secretKeyRef: {
                    name: 'ql3-e2e-postgres-auth',
                    key: 'worker-credential-executor-password',
                  },
                },
              },
              {
                name: 'QL3_WORKER_INGRESS_PASSWORD',
                valueFrom: {
                  secretKeyRef: {
                    name: 'ql3-e2e-postgres-auth',
                    key: 'worker-ingress-password',
                  },
                },
              },
            ],
            securityContext: {
              allowPrivilegeEscalation: false,
            },
            ports: [{ name: 'postgres', containerPort: 5432 }],
            readinessProbe: {
              exec: {
                command: [
                  'pg_isready',
                  '--username',
                  'postgres',
                  '--dbname',
                  'postgres',
                ],
              },
              periodSeconds: 2,
              failureThreshold: 60,
            },
            resources: {
              requests: { cpu: '100m', memory: '128Mi' },
              limits: { cpu: '1', memory: '512Mi' },
            },
            volumeMounts: [
              { name: 'data', mountPath: '/var/lib/postgresql' },
              {
                name: 'init',
                mountPath: '/docker-entrypoint-initdb.d',
                readOnly: true,
              },
            ],
          },
        ],
        volumes: [
          { name: 'data', emptyDir: { sizeLimit: '2Gi' } },
          {
            name: 'init',
            configMap: {
              name: 'ql3-e2e-postgres-init',
              defaultMode: 365,
            },
          },
        ],
      },
    },
  ];
}

function registryResources(fixture, certificate, registryCredential) {
  return [
    {
      apiVersion: 'v1',
      kind: 'Secret',
      metadata: { name: 'ql3-e2e-registry-tls', namespace: NAMESPACE },
      type: 'Opaque',
      stringData: {
        'ca.crt': certificate.ca,
        'tls.crt': certificate.cert,
        'tls.key': certificate.key,
      },
    },
    {
      apiVersion: 'v1',
      kind: 'Secret',
      metadata: { name: 'ql3-e2e-registry-auth', namespace: NAMESPACE },
      type: 'Opaque',
      stringData: {
        authorization: registryCredential.authorization,
        'credentials.json': registryCredential.file,
      },
    },
    {
      apiVersion: 'v1',
      kind: 'ConfigMap',
      metadata: { name: 'ql3-e2e-fixture', namespace: NAMESPACE },
      data: {
        'fixture.cjs': fs.readFileSync(FIXTURE_SCRIPT, 'utf8'),
        'fixture.json': JSON.stringify(fixture),
      },
    },
    {
      apiVersion: 'v1',
      kind: 'Service',
      metadata: { name: REGISTRY_NAME, namespace: NAMESPACE },
      spec: {
        selector: { 'app.kubernetes.io/name': REGISTRY_NAME },
        ports: [{ name: 'https', port: 443, targetPort: 8443 }],
      },
    },
    {
      apiVersion: 'v1',
      kind: 'Pod',
      metadata: {
        name: REGISTRY_NAME,
        namespace: NAMESPACE,
        labels: { 'app.kubernetes.io/name': REGISTRY_NAME },
      },
      spec: {
        automountServiceAccountToken: false,
        restartPolicy: 'Never',
        securityContext: commonSecurityContext(),
        containers: [
          {
            name: 'registry',
            image: ADMIN_IMAGE,
            imagePullPolicy: 'Never',
            command: ['node', '/opt/ql3-e2e/fixture.cjs'],
            env: [
              { name: 'QL3_E2E_MODE', value: 'registry' },
              {
                name: 'QL3_E2E_FIXTURE_FILE',
                value: '/opt/ql3-e2e/fixture.json',
              },
              {
                name: 'QL3_E2E_REGISTRY_CERT_FILE',
                value: '/var/run/ql3-e2e-registry/tls.crt',
              },
              {
                name: 'QL3_E2E_REGISTRY_KEY_FILE',
                value: '/var/run/ql3-e2e-registry/tls.key',
              },
              { name: 'QL3_E2E_REGISTRY_PORT', value: '8443' },
              {
                name: 'QL3_E2E_REGISTRY_AUTHORIZATION',
                valueFrom: {
                  secretKeyRef: {
                    name: 'ql3-e2e-registry-auth',
                    key: 'authorization',
                  },
                },
              },
            ],
            securityContext: containerSecurityContext(),
            ports: [{ name: 'https', containerPort: 8443 }],
            readinessProbe: {
              tcpSocket: { port: 'https' },
              periodSeconds: 2,
              failureThreshold: 30,
            },
            resources: {
              requests: { cpu: '25m', memory: '32Mi' },
              limits: { cpu: '250m', memory: '128Mi' },
            },
            volumeMounts: [
              {
                name: 'fixture',
                mountPath: '/opt/ql3-e2e',
                readOnly: true,
              },
              {
                name: 'tls',
                mountPath: '/var/run/ql3-e2e-registry',
                readOnly: true,
              },
            ],
          },
        ],
        volumes: [
          {
            name: 'fixture',
            configMap: { name: 'ql3-e2e-fixture', defaultMode: 292 },
          },
          {
            name: 'tls',
            secret: {
              secretName: 'ql3-e2e-registry-tls',
              defaultMode: 292,
            },
          },
        ],
      },
    },
  ];
}

function postgresEnvironment(role, passwordKey, applicationName) {
  return [
    { name: 'QL3_POSTGRES_TLS_MODE', value: 'disable' },
    { name: 'QL3_POSTGRES_ALLOW_INSECURE', value: 'true' },
    { name: 'QL3_POSTGRES_APPLICATION_NAME', value: applicationName },
    { name: `QL3_POSTGRES_${role}_HOST`, value: POSTGRES_NAME },
    { name: `QL3_POSTGRES_${role}_PORT`, value: '5432' },
    { name: `QL3_POSTGRES_${role}_DATABASE`, value: 'qinglong' },
    {
      name: `QL3_POSTGRES_${role}_USER`,
      value: `ql3_${role.toLowerCase().replaceAll('-', '_')}`,
    },
    {
      name: `QL3_POSTGRES_${role}_PASSWORD`,
      valueFrom: {
        secretKeyRef: {
          name: 'ql3-e2e-postgres-auth',
          key: passwordKey,
        },
      },
    },
  ];
}

function migrationJob() {
  const [job] = readYamlDocuments(
    path.join(
      ROOT,
      'deploy/kubernetes/ql3-cluster/operations/base/migrate-job.yaml',
    ),
  );
  job.metadata.namespace = NAMESPACE;
  job.spec.activeDeadlineSeconds = 300;
  job.spec.template.spec.serviceAccountName = CONTROL_SERVICE_ACCOUNT;
  const container = job.spec.template.spec.containers[0];
  container.image = CONTROL_IMAGE;
  container.imagePullPolicy = 'Never';
  container.env = postgresEnvironment(
    'MIGRATION',
    'migration-password',
    'qinglong3-plugin-e2e-migration',
  );
  container.volumeMounts = [{ name: 'tmp', mountPath: '/tmp' }];
  job.spec.template.spec.volumes = [
    {
      name: 'tmp',
      emptyDir: { medium: 'Memory', sizeLimit: '16Mi' },
    },
  ];
  return job;
}

function seedJob(name, mode) {
  return {
    apiVersion: 'batch/v1',
    kind: 'Job',
    metadata: { name, namespace: NAMESPACE },
    spec: {
      backoffLimit: 0,
      activeDeadlineSeconds: 300,
      ttlSecondsAfterFinished: 600,
      template: {
        metadata: {
          labels: {
            'app.kubernetes.io/name': 'ql3-plugin-package-e2e-seed',
            'app.kubernetes.io/part-of': 'qinglong3',
          },
        },
        spec: {
          automountServiceAccountToken: false,
          restartPolicy: 'Never',
          securityContext: commonSecurityContext(),
          containers: [
            {
              name: 'seed',
              image: ADMIN_IMAGE,
              imagePullPolicy: 'Never',
              command: ['node', '/opt/ql3-e2e/fixture.cjs'],
              env: [
                { name: 'QL3_E2E_MODE', value: mode },
                {
                  name: 'QL3_E2E_FIXTURE_FILE',
                  value: '/opt/ql3-e2e/fixture.json',
                },
                {
                  name: 'NODE_PATH',
                  value: '/opt/qinglong/node_modules',
                },
                { name: 'QL3_E2E_POSTGRES_HOST', value: POSTGRES_NAME },
                { name: 'QL3_E2E_POSTGRES_PORT', value: '5432' },
                { name: 'QL3_E2E_POSTGRES_DATABASE', value: 'qinglong' },
                {
                  name: 'QL3_E2E_POSTGRES_RUNTIME_USER',
                  value: 'ql3_runtime',
                },
                {
                  name: 'QL3_E2E_POSTGRES_RUNTIME_PASSWORD',
                  valueFrom: {
                    secretKeyRef: {
                      name: 'ql3-e2e-postgres-auth',
                      key: 'runtime-password',
                    },
                  },
                },
                {
                  name: 'QL3_E2E_POSTGRES_PACKAGE_MANAGER_USER',
                  value: 'ql3_package_manager',
                },
                {
                  name: 'QL3_E2E_POSTGRES_PACKAGE_MANAGER_PASSWORD',
                  valueFrom: {
                    secretKeyRef: {
                      name: 'ql3-e2e-postgres-auth',
                      key: 'package-manager-password',
                    },
                  },
                },
                {
                  name: 'QL3_E2E_POSTGRES_PACKAGE_EXECUTOR_USER',
                  value: 'ql3_package_executor',
                },
                {
                  name: 'QL3_E2E_POSTGRES_PACKAGE_EXECUTOR_PASSWORD',
                  valueFrom: {
                    secretKeyRef: {
                      name: 'ql3-e2e-postgres-auth',
                      key: 'package-executor-password',
                    },
                  },
                },
              ],
              securityContext: containerSecurityContext(),
              resources: {
                requests: { cpu: '25m', memory: '64Mi' },
                limits: { cpu: '500m', memory: '256Mi' },
              },
              volumeMounts: [
                {
                  name: 'fixture',
                  mountPath: '/opt/ql3-e2e',
                  readOnly: true,
                },
                { name: 'tmp', mountPath: '/tmp' },
              ],
            },
          ],
          volumes: [
            {
              name: 'fixture',
              configMap: { name: 'ql3-e2e-fixture', defaultMode: 292 },
            },
            {
              name: 'tmp',
              emptyDir: { medium: 'Memory', sizeLimit: '16Mi' },
            },
          ],
        },
      },
    },
  };
}

function recoveryResources(fixture, jobName) {
  const rbac = readYamlDocuments(
    path.join(
      ROOT,
      'deploy/kubernetes/ql3-cluster/operations/plugin-package-recovery/base/rbac.yaml',
    ),
  );
  for (const value of rbac) {
    value.metadata.namespace = NAMESPACE;
    if (value.kind === 'RoleBinding') {
      for (const subject of value.subjects) subject.namespace = NAMESPACE;
    }
  }
  const [job] = readYamlDocuments(
    path.join(
      ROOT,
      'deploy/kubernetes/ql3-cluster/operations/plugin-package-recovery/base/recover-job.yaml',
    ),
  );
  job.metadata.name = jobName;
  job.metadata.namespace = NAMESPACE;
  job.spec.activeDeadlineSeconds = 300;
  const container = job.spec.template.spec.containers[0];
  container.image = ADMIN_IMAGE;
  container.imagePullPolicy = 'Never';
  container.env = [
    {
      name: 'QL3_CLUSTER_IDENTITY',
      valueFrom: {
        configMapKeyRef: {
          name: 'ql3-plugin-package-recovery-config',
          key: 'cluster-identity',
        },
      },
    },
    {
      name: 'QL3_KUBERNETES_NAMESPACE',
      valueFrom: {
        fieldRef: { apiVersion: 'v1', fieldPath: 'metadata.namespace' },
      },
    },
    {
      name: 'QL3_PLUGIN_PACKAGE_OCI_REGISTRIES',
      valueFrom: {
        configMapKeyRef: {
          name: 'ql3-plugin-package-recovery-config',
          key: 'oci-registries',
        },
      },
    },
    {
      name: 'QL3_PLUGIN_PACKAGE_PUBLISHER_TRUST_FILE',
      value: '/var/run/qinglong3/plugin-package-trust/publishers.json',
    },
    {
      name: 'NODE_EXTRA_CA_CERTS',
      value: '/var/run/secrets/ql3-e2e-registry/ca.crt',
    },
    {
      name: 'QL3_PLUGIN_PACKAGE_REGISTRY_CREDENTIAL_FILE',
      value: '/var/run/secrets/ql3-e2e-registry-auth/credentials.json',
    },
    { name: 'QL3_PLUGIN_PACKAGE_OCI_TIMEOUT_MS', value: '15000' },
    { name: 'QL3_PLUGIN_PACKAGE_RECOVERY_PAGE_SIZE', value: '8' },
    { name: 'QL3_PLUGIN_PACKAGE_RECOVERY_MAX_PAGES', value: '8' },
    ...postgresEnvironment(
      'PACKAGE_EXECUTOR',
      'package-executor-password',
      'qinglong3-plugin-package-recovery-e2e',
    ),
  ];
  container.volumeMounts = [
    { name: 'tmp', mountPath: '/tmp' },
    {
      name: 'plugin-package-trust',
      mountPath: '/var/run/qinglong3/plugin-package-trust',
      readOnly: true,
    },
    {
      name: 'registry-ca',
      mountPath: '/var/run/secrets/ql3-e2e-registry',
      readOnly: true,
    },
    {
      name: 'registry-credentials',
      mountPath: '/var/run/secrets/ql3-e2e-registry-auth',
      readOnly: true,
    },
  ];
  job.spec.template.spec.volumes = [
    {
      name: 'tmp',
      emptyDir: { medium: 'Memory', sizeLimit: '16Mi' },
    },
    {
      name: 'plugin-package-trust',
      configMap: {
        name: 'ql3-plugin-publisher-trust',
        defaultMode: 292,
        items: [{ key: 'publishers.json', path: 'publishers.json' }],
      },
    },
    {
      name: 'registry-ca',
      secret: {
        secretName: 'ql3-e2e-registry-tls',
        defaultMode: 292,
        items: [{ key: 'ca.crt', path: 'ca.crt' }],
      },
    },
    {
      name: 'registry-credentials',
      secret: {
        secretName: 'ql3-e2e-registry-auth',
        defaultMode: 288,
        items: [{ key: 'credentials.json', path: 'credentials.json' }],
      },
    },
  ];
  return [
    ...rbac,
    {
      apiVersion: 'v1',
      kind: 'ConfigMap',
      metadata: {
        name: 'ql3-plugin-package-recovery-config',
        namespace: NAMESPACE,
      },
      data: {
        'cluster-identity': 'ql3-plugin-package-recovery-e2e-cluster',
        'oci-registries': fixture.registry,
      },
    },
    {
      apiVersion: 'v1',
      kind: 'ConfigMap',
      metadata: {
        name: 'ql3-plugin-publisher-trust',
        namespace: NAMESPACE,
      },
      data: { 'publishers.json': JSON.stringify(fixture.trust) },
    },
    job,
  ];
}

function waitForPostgresService(timeoutMs = 60_000) {
  const deadline = Date.now() + timeoutMs;
  let lastFact = 'probe not started';
  while (Date.now() < deadline) {
    const probe = kubectl(
      [
        '-n',
        NAMESPACE,
        'exec',
        `pod/${POSTGRES_NAME}`,
        '--',
        'pg_isready',
        '--host',
        POSTGRES_NAME,
        '--port',
        '5432',
        '--username',
        'postgres',
        '--dbname',
        'qinglong',
      ],
      { capture: true, quiet: true, allowFailure: true },
    );
    if (probe.status === 0) return;
    lastFact = (probe.stderr || probe.stdout || `status ${probe.status}`).trim();
    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1_000);
  }
  fail(`PostgreSQL Service was not ready after ${timeoutMs}ms: ${lastFact}`);
}

function waitForJob(
  name,
  expectedStatus = 'complete',
  timeoutMs = 5 * 60 * 1000,
) {
  if (!['complete', 'failed'].includes(expectedStatus)) {
    fail('expected Job status is invalid');
  }
  const deadline = Date.now() + timeoutMs;
  while (Date.now() < deadline) {
    const job = kubectlJson(['-n', NAMESPACE, 'get', 'job', name]);
    const complete = job.status?.conditions?.find(
      (condition) =>
        condition.type === 'Complete' && condition.status === 'True',
    );
    if (complete) {
      if (expectedStatus !== 'complete') {
        fail(`${name} completed but failure was required`);
      }
      return job;
    }
    const failed = job.status?.conditions?.find(
      (condition) => condition.type === 'Failed' && condition.status === 'True',
    );
    if (failed) {
      if (expectedStatus === 'failed') return job;
      const logs = kubectl(
        ['-n', NAMESPACE, 'logs', `job/${name}`, '--all-containers=true'],
        { capture: true, quiet: true, allowFailure: true },
      );
      fail(
        `${name} failed: ${failed.reason ?? 'unknown'}\n${logs.stdout}\n${
          logs.stderr
        }`,
      );
    }
    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1_000);
  }
  const facts = kubectl(['-n', NAMESPACE, 'get', 'job,pod', '-o', 'wide'], {
    capture: true,
    quiet: true,
    allowFailure: true,
  });
  const logs = kubectl(
    ['-n', NAMESPACE, 'logs', `job/${name}`, '--all-containers=true'],
    { capture: true, quiet: true, allowFailure: true },
  );
  fail(
    `${name} timed out after ${timeoutMs}ms\n${facts.stdout}\n${logs.stdout}\n${logs.stderr}`,
  );
}

function jobLog(name) {
  return kubectl(
    ['-n', NAMESPACE, 'logs', `job/${name}`, '--all-containers=true'],
    { capture: true, quiet: true },
  ).stdout;
}

function lastJsonLine(output, predicate) {
  const values = output
    .split('\n')
    .map((line) => line.trim())
    .filter(Boolean)
    .map((line) => {
      try {
        return JSON.parse(line);
      } catch {
        return null;
      }
    })
    .filter(Boolean);
  const value = values.findLast(predicate);
  assert.ok(value, `expected JSON evidence was absent from:\n${output}`);
  return value;
}

function canI(verb, resource) {
  const result = kubectl(
    [
      'auth',
      'can-i',
      verb,
      resource,
      '--namespace',
      NAMESPACE,
      '--as',
      `system:serviceaccount:${NAMESPACE}:${RECOVERY_SERVICE_ACCOUNT}`,
    ],
    { capture: true, quiet: true, allowFailure: true },
  );
  assert.ok(result.status === 0 || result.status === 1);
  assert.ok(result.stdout === 'yes' || result.stdout === 'no');
  return result.stdout === 'yes';
}

function databaseEvidence(fixture) {
  const initialGenerationDigest = fixture.initial.generation.generationDigest;
  const upgradeGenerationDigest = fixture.upgrade.generation.generationDigest;
  assert.match(initialGenerationDigest, /^[0-9a-f]{64}$/);
  assert.match(upgradeGenerationDigest, /^[0-9a-f]{64}$/);
  const sql = `
SELECT json_build_object(
  'migrationCount', (SELECT count(*) FROM ql3.schema_migrations),
  'capabilityVersion', (
    SELECT contract_version
    FROM ql3.schema_capabilities
    WHERE contract_name = 'control-core'
  ),
  'initialState', (
    SELECT state
    FROM ql3.plugin_package_installs
    WHERE installation_id = '${fixture.initial.installationId}'
  ),
  'initialActiveLockDigest', (
    SELECT active_lock_digest
    FROM ql3.plugin_package_installs
    WHERE installation_id = '${fixture.initial.installationId}'
  ),
  'upgradeState', (
    SELECT state
    FROM ql3.plugin_package_installs
    WHERE installation_id = '${fixture.upgrade.installationId}'
  ),
  'upgradePreviousActiveLockDigest', (
    SELECT previous_active_lock_digest
    FROM ql3.plugin_package_installs
    WHERE installation_id = '${fixture.upgrade.installationId}'
  ),
  'upgradeActiveLockDigest', (
    SELECT active_lock_digest
    FROM ql3.plugin_package_installs
    WHERE installation_id = '${fixture.upgrade.installationId}'
  ),
  'upgradeFailureReason', (
    SELECT record_json #>> '{failure,reason}'
    FROM ql3.plugin_package_installs
    WHERE installation_id = '${fixture.upgrade.installationId}'
  ),
  'initialMutationCount', (
    SELECT count(*)
    FROM ql3.plugin_package_install_mutations
    WHERE installation_id = '${fixture.initial.installationId}'
  ),
  'upgradeMutationCount', (
    SELECT count(*)
    FROM ql3.plugin_package_install_mutations
    WHERE installation_id = '${fixture.upgrade.installationId}'
  ),
  'headInstallationId', (
    SELECT installation_id
    FROM ql3.plugin_package_install_heads
    WHERE project_id = 'default' AND package_name = 'e2e-monitor'
  ),
  'initialRevisionCount', (
    SELECT count(*) FROM ql3.plugin_package_materialized_revisions
    WHERE generation_digest = '${initialGenerationDigest}'
  ),
  'upgradeRevisionCount', (
    SELECT count(*) FROM ql3.plugin_package_materialized_revisions
    WHERE generation_digest = '${upgradeGenerationDigest}'
  ),
  'recoverableCount', (
    SELECT count(*)
    FROM ql3.plugin_package_installs
    WHERE state IN ('queued', 'staged', 'activating')
  )
)::text;
`.trim();
  const output = kubectl(
    [
      '-n',
      NAMESPACE,
      'exec',
      POSTGRES_NAME,
      '--',
      'psql',
      '--username',
      'postgres',
      '--dbname',
      'qinglong',
      '--tuples-only',
      '--no-align',
      '--command',
      sql,
    ],
    { capture: true, quiet: true },
  ).stdout;
  const value = JSON.parse(output);
  assert.equal(value.migrationCount, MIGRATION_COUNT);
  assert.equal(value.capabilityVersion, CONTRACT_VERSION);
  assert.equal(value.initialState, 'active');
  assert.equal(value.initialActiveLockDigest, fixture.initial.lock.lockDigest);
  assert.equal(value.upgradeState, 'failed');
  assert.equal(
    value.upgradePreviousActiveLockDigest,
    fixture.initial.lock.lockDigest,
  );
  assert.equal(value.upgradeActiveLockDigest, fixture.initial.lock.lockDigest);
  assert.equal(value.upgradeFailureReason, 'activation_fact_conflict');
  assert.equal(value.initialMutationCount, 4);
  assert.equal(value.upgradeMutationCount, 3);
  assert.equal(value.headInstallationId, fixture.upgrade.installationId);
  assert.equal(value.initialRevisionCount, 1);
  assert.equal(value.upgradeRevisionCount, 0);
  assert.equal(value.recoverableCount, 0);
  return value;
}

function activePointerEvidence(fixture) {
  const values = kubectlJson([
    '-n',
    NAMESPACE,
    'get',
    'configmaps',
    '-l',
    'qinglong.io/plugin-package-active=v2',
  ]);
  assert.equal(values.items.length, 1);
  const configMap = values.items[0];
  const pointer = JSON.parse(configMap.data['active.json']);
  assert.equal(pointer.intent.installationId, fixture.initial.installationId);
  assert.equal(pointer.intent.lockDigest, fixture.initial.lock.lockDigest);
  assert.equal(pointer.receipt.generation, 1);
  return Object.freeze({
    name: configMap.metadata.name,
    uid: configMap.metadata.uid,
    resourceVersion: configMap.metadata.resourceVersion,
    activeJson: configMap.data['active.json'],
    intentDigest: pointer.intent.intentDigest,
    activationRef: pointer.receipt.activationRef,
  });
}

function reportActivePointer(pointer) {
  return Object.freeze({
    name: pointer.name,
    uid: pointer.uid,
    resourceVersion: pointer.resourceVersion,
    activeJsonDigest: createHash('sha256')
      .update(pointer.activeJson, 'utf8')
      .digest('hex'),
    intentDigest: pointer.intentDigest,
    activationRef: pointer.activationRef,
  });
}

function registryEvidence(fixture) {
  const output = kubectl(
    ['-n', NAMESPACE, 'logs', REGISTRY_NAME, '-c', 'registry'],
    { capture: true, quiet: true },
  ).stdout;
  const events = output
    .split('\n')
    .map((line) => line.trim())
    .filter(Boolean)
    .map((line) => JSON.parse(line))
    .filter((value) => value.schema === REGISTRY_EVENT_SCHEMA);
  const packageRequests = events.filter((event) => event.path !== '/v2/');
  const expectedPaths = [
    ...fixture.initial.routes.flatMap((routeValue) => [
      routeValue.path,
      routeValue.path,
    ]),
    ...fixture.upgrade.routes.flatMap((routeValue) => [
      routeValue.path,
      routeValue.path,
    ]),
  ].sort();
  assert.equal(packageRequests.length, expectedPaths.length);
  assert.deepEqual(
    packageRequests.map((event) => event.path).sort(),
    expectedPaths,
  );
  assert.ok(packageRequests.every((event) => event.status === 200));
  assert.ok(packageRequests.every((event) => event.authenticated === true));
  return Object.freeze({
    https: true,
    authentication: 'exact-registry-basic',
    authenticatedRequestCount: packageRequests.length,
    requestCount: packageRequests.length,
    uniquePaths: new Set(packageRequests.map((event) => event.path)).size,
    initialRequestCount: fixture.initial.routes.length * 2,
    upgradeRequestCount: fixture.upgrade.routes.length * 2,
    redirects: 0,
  });
}

function recoveryRbacEvidence() {
  const role = kubectlJson([
    '-n',
    NAMESPACE,
    'get',
    'role',
    RECOVERY_SERVICE_ACCOUNT,
  ]);
  assert.deepEqual(role.rules, [
    {
      apiGroups: [''],
      resources: ['configmaps'],
      verbs: ['get', 'create', 'update'],
    },
  ]);
  const decisions = {
    getConfigMaps: canI('get', 'configmaps'),
    createConfigMaps: canI('create', 'configmaps'),
    updateConfigMaps: canI('update', 'configmaps'),
    listConfigMaps: canI('list', 'configmaps'),
    deleteConfigMaps: canI('delete', 'configmaps'),
    getSecrets: canI('get', 'secrets'),
  };
  assert.deepEqual(decisions, {
    getConfigMaps: true,
    createConfigMaps: true,
    updateConfigMaps: true,
    listConfigMaps: false,
    deleteConfigMaps: false,
    getSecrets: false,
  });
  return Object.freeze(decisions);
}

function applyRuntimeAfterRecovery(recoveryJob, migrationJobValue, secrets) {
  const recoveryComplete = recoveryJob.status.conditions.find(
    (condition) => condition.type === 'Complete' && condition.status === 'True',
  );
  const migrationComplete = migrationJobValue.status.conditions.find(
    (condition) => condition.type === 'Complete' && condition.status === 'True',
  );
  assert.ok(recoveryComplete?.lastTransitionTime);
  assert.ok(migrationComplete?.lastTransitionTime);
  assert.ok(
    Date.parse(recoveryComplete.lastTransitionTime) >=
      Date.parse(migrationComplete.lastTransitionTime),
  );
  const annotation = {
    'qinglong.io/plugin-recovery-job-uid': recoveryJob.metadata.uid,
    'qinglong.io/plugin-recovery-resource-version':
      recoveryJob.metadata.resourceVersion,
    'qinglong.io/plugin-recovery-completed-at':
      recoveryComplete.lastTransitionTime,
    'qinglong.io/migration-job-uid': migrationJobValue.metadata.uid,
    'qinglong.io/migration-completed-at': migrationComplete.lastTransitionTime,
  };
  apply(
    {
      apiVersion: 'v1',
      kind: 'Secret',
      metadata: {
        name: 'ql3-cluster-control-runtime',
        namespace: NAMESPACE,
      },
      type: 'Opaque',
      stringData: {
        'postgres-runtime-url': `postgresql://ql3_runtime:${secrets.runtime}@${POSTGRES_NAME}:5432/qinglong`,
        'api-credential-pepper': randomBytes(32).toString('base64url'),
      },
    },
    'create runtime-only credential after recovery success',
  );
  const names = [
    'service-account.yaml',
    'service.yaml',
    'pod-disruption-budget.yaml',
    'deployment.yaml',
  ];
  const resources = names.flatMap((name) =>
    readYamlDocuments(
      path.join(ROOT, 'deploy/kubernetes/ql3-cluster/base', name),
    ),
  );
  for (const resource of resources) {
    resource.metadata.namespace = NAMESPACE;
    if (resource.kind !== 'Deployment') continue;
    resource.metadata.annotations = annotation;
    const container = resource.spec.template.spec.containers[0];
    container.image = CONTROL_IMAGE;
    container.imagePullPolicy = 'Never';
    container.env = [
      { name: 'QL_DEPLOYMENT_PROFILE', value: 'cluster-control' },
      { name: 'QL3_CLUSTER_CONTROL_ENABLED', value: 'true' },
      { name: 'QL3_CLUSTER_HTTP_HOST', value: '0.0.0.0' },
      { name: 'QL3_CLUSTER_HTTP_PORT', value: '5800' },
      { name: 'QL3_CLUSTER_HTTP_DRAIN_TIMEOUT_MS', value: '10000' },
      { name: 'QL3_POSTGRES_TLS_MODE', value: 'disable' },
      { name: 'QL3_POSTGRES_ALLOW_INSECURE', value: 'true' },
      { name: 'QL3_POSTGRES_MAX_CONNECTIONS', value: '4' },
      {
        name: 'QL3_POSTGRES_APPLICATION_NAME',
        value: 'qinglong3-plugin-recovery-e2e-runtime',
      },
      {
        name: 'QL3_CLUSTER_REPLICA_ID',
        valueFrom: {
          fieldRef: { apiVersion: 'v1', fieldPath: 'metadata.name' },
        },
      },
      {
        name: 'QL3_POSTGRES_RUNTIME_URL',
        valueFrom: {
          secretKeyRef: {
            name: 'ql3-cluster-control-runtime',
            key: 'postgres-runtime-url',
          },
        },
      },
      {
        name: 'QL3_API_CREDENTIAL_PEPPER',
        valueFrom: {
          secretKeyRef: {
            name: 'ql3-cluster-control-runtime',
            key: 'api-credential-pepper',
          },
        },
      },
    ];
    container.volumeMounts = [{ name: 'tmp', mountPath: '/tmp' }];
    resource.spec.template.spec.volumes = [
      {
        name: 'tmp',
        emptyDir: { medium: 'Memory', sizeLimit: '16Mi' },
      },
    ];
  }
  for (const resource of resources) {
    apply(resource, `deployment controller apply ${resource.kind}`);
  }
  kubectl(
    [
      '-n',
      NAMESPACE,
      'rollout',
      'status',
      'deployment/ql3-cluster-control',
      '--timeout=5m',
    ],
    { capture: true, quiet: true },
  );
  const deployment = kubectlJson([
    '-n',
    NAMESPACE,
    'get',
    'deployment',
    'ql3-cluster-control',
  ]);
  assert.equal(deployment.status.availableReplicas, 2);
  assert.deepEqual(deployment.metadata.annotations, {
    ...deployment.metadata.annotations,
    ...annotation,
  });
  for (const [key, value] of Object.entries(annotation)) {
    assert.equal(deployment.metadata.annotations[key], value);
  }
  assert.ok(
    Date.parse(deployment.metadata.creationTimestamp) >=
      Date.parse(recoveryComplete.lastTransitionTime),
  );
  const pods = kubectlJson([
    '-n',
    NAMESPACE,
    'get',
    'pods',
    '-l',
    'app.kubernetes.io/name=ql3-cluster-control',
  ]);
  assert.equal(pods.items.length, 2);
  assert.equal(new Set(pods.items.map((pod) => pod.spec.nodeName)).size, 2);
  assert.ok(
    pods.items.every(
      (pod) =>
        pod.status.containerStatuses?.[0]?.ready === true &&
        pod.spec.automountServiceAccountToken === false,
    ),
  );
  return Object.freeze({
    replicas: deployment.status.availableReplicas,
    creationTimestamp: deployment.metadata.creationTimestamp,
    recoveryJobUid: annotation['qinglong.io/plugin-recovery-job-uid'],
    recoveryCompletedAt: annotation['qinglong.io/plugin-recovery-completed-at'],
    nodes: pods.items.map((pod) => pod.spec.nodeName).sort(),
    imageIds: [
      ...new Set(
        pods.items.map((pod) => pod.status.containerStatuses[0].imageID),
      ),
    ],
  });
}

function assertRuntimeCannotReadPluginAuthority() {
  const result = kubectl(
    [
      '-n',
      NAMESPACE,
      'exec',
      POSTGRES_NAME,
      '--',
      'sh',
      '-c',
      'PGPASSWORD="$QL3_RUNTIME_PASSWORD" psql --host=ql3-postgres-live --username=ql3_runtime --dbname=qinglong --command="SELECT count(*) FROM ql3.plugin_package_installs"',
    ],
    { capture: true, quiet: true, allowFailure: true },
  );
  assert.notEqual(result.status, 0);
  assert.match(result.stderr, /permission denied/i);
}

function jobImageId(name) {
  const pods = kubectlJson([
    '-n',
    NAMESPACE,
    'get',
    'pods',
    '-l',
    `job-name=${name}`,
  ]);
  assert.equal(pods.items.length, 1);
  return pods.items[0].status.containerStatuses[0].imageID;
}

function buildOrderingEvidence(
  migrated,
  initialRecovered,
  rejectedUpgrade,
  runtime,
) {
  return Object.freeze({
    migrationJobUid: migrated.metadata.uid,
    migrationCompletedAt: migrated.status.completionTime,
    initialRecoveryJobUid: initialRecovered.metadata.uid,
    initialRecoveryCompletedAt: initialRecovered.status.completionTime,
    upgradeRecoveryJobUid: rejectedUpgrade.metadata.uid,
    upgradeRecoveryCompletedAt: runtime.recoveryCompletedAt,
    runtimeCreatedAt: runtime.creationTimestamp,
    runtimeBoundRecoveryJobUid: runtime.recoveryJobUid,
  });
}

function diagnostics() {
  if (!kubeconfig || !fs.existsSync(kubeconfig)) return;
  const snapshot = kubectl(
    ['-n', NAMESPACE, 'get', 'pod,job,deployment,configmap', '-o', 'wide'],
    { capture: true, quiet: true, allowFailure: true },
  );
  if (snapshot.stdout) process.stderr.write(`${snapshot.stdout}\n`);
}

async function main(argv = process.argv.slice(2)) {
  const reportFile = privateReportPath(argv);
  if (process.env.QL3_PLUGIN_PACKAGE_RECOVERY_E2E_LIVE !== '1') {
    fail(
      'Refusing to create a live cluster without QL3_PLUGIN_PACKAGE_RECOVERY_E2E_LIVE=1',
    );
  }
  const revision = sourceRevision();
  const clusterName = exactClusterName();
  const existing = kind(['get', 'clusters'], {
    capture: true,
    quiet: true,
    allowFailure: true,
  })
    .stdout.split('\n')
    .map((value) => value.trim())
    .filter(Boolean);
  if (existing.includes(clusterName)) {
    fail(
      `Refusing to reuse or delete pre-existing Kind cluster ${clusterName}`,
    );
  }
  const root = fs.mkdtempSync(
    path.join(os.tmpdir(), 'ql3-plugin-recovery-e2e-'),
  );
  kubeconfig = path.join(root, 'kubeconfig');
  const kindConfig = path.join(root, 'kind.yaml');
  const fixtureArchitecture = architecture();
  const fixture = createFixture({
    registry: REGISTRY_HOST,
    architecture: fixtureArchitecture,
  });
  const certificate = createRegistryCertificate(root);
  const secrets = {
    postgres: randomSecret(),
    migration: randomSecret(),
    runtime: randomSecret(),
    admin: randomSecret(),
    automationManager: randomSecret(),
    approvalManager: randomSecret(),
    runManager: randomSecret(),
    packageManager: randomSecret(),
    packageExecutor: randomSecret(),
    workerCredentialManager: randomSecret(),
    workerCredentialExecutor: randomSecret(),
    worker: randomSecret(),
    registry: randomSecret(),
  };
  const registryCredential = {
    authorization: `Basic ${Buffer.from(
      `ql3-e2e:${secrets.registry}`,
      'utf8',
    ).toString('base64')}`,
    file: JSON.stringify({
      schema: 'qinglong/plugin-package-registry-credentials@v1',
      credentials: [
        {
          registry: fixture.registry,
          scheme: 'basic',
          username: 'ql3-e2e',
          password: secrets.registry,
        },
      ],
    }),
  };
  let created = false;
  const startedAt = Date.now();
  try {
    buildImages(revision);
    const adminSourceRevision = imageSourceRevision(ADMIN_IMAGE);
    const controlSourceRevision = imageSourceRevision(CONTROL_IMAGE);
    assert.equal(adminSourceRevision, revision);
    assert.equal(controlSourceRevision, revision);
    const adminBuildId = imageId(ADMIN_IMAGE);
    const controlBuildId = imageId(CONTROL_IMAGE);
    ensurePostgresImage();
    fs.writeFileSync(
      kindConfig,
      `${yaml.dump({
        kind: 'Cluster',
        apiVersion: 'kind.x-k8s.io/v1alpha4',
        nodes: [
          { role: 'control-plane' },
          { role: 'worker' },
          { role: 'worker' },
        ],
      })}`,
      { mode: 0o600 },
    );
    kind(
      [
        'create',
        'cluster',
        '--name',
        clusterName,
        '--image',
        KIND_NODE_IMAGE,
        '--config',
        kindConfig,
        '--kubeconfig',
        kubeconfig,
        '--wait',
        '5m',
      ],
      {
        env: kindCreateEnvironment(clusterName),
        label: `create isolated Kind cluster ${clusterName}`,
      },
    );
    created = true;
    for (const image of [
      ADMIN_IMAGE,
      CONTROL_IMAGE,
      POSTGRES_RUNTIME_IMAGE,
    ]) {
      kind(
        [
          'load',
          'docker-image',
          '--name',
          clusterName,
          '--nodes',
          `${clusterName}-control-plane,${clusterName}-worker,${clusterName}-worker2`,
          image,
        ],
        { label: `load ${image} into ${clusterName}` },
      );
    }
    apply(
      {
        apiVersion: 'v1',
        kind: 'Namespace',
        metadata: { name: NAMESPACE },
      },
      'create Plugin Package recovery E2E namespace',
    );
    apply(
      {
        apiVersion: 'v1',
        kind: 'ServiceAccount',
        metadata: {
          name: CONTROL_SERVICE_ACCOUNT,
          namespace: NAMESPACE,
        },
        automountServiceAccountToken: false,
      },
      'create migration/runtime ServiceAccount',
    );
    for (const resource of postgresResources(secrets)) {
      apply(resource, `apply PostgreSQL fixture ${resource.kind}`);
    }
    for (const resource of registryResources(
      fixture,
      certificate,
      registryCredential,
    )) {
      apply(resource, `apply OCI fixture ${resource.kind}`);
    }
    kubectl(
      [
        '-n',
        NAMESPACE,
        'wait',
        '--for=condition=Ready',
        `pod/${POSTGRES_NAME}`,
        '--timeout=5m',
      ],
      { capture: true, quiet: true },
    );
    waitForPostgresService();
    kubectl(
      [
        '-n',
        NAMESPACE,
        'wait',
        '--for=condition=Ready',
        `pod/${REGISTRY_NAME}`,
        '--timeout=2m',
      ],
      { capture: true, quiet: true },
    );

    apply(migrationJob(), 'create reviewed migration Job');
    const migrated = waitForJob('ql3-cluster-migration');
    const migrationLog = jobLog('ql3-cluster-migration');
    lastJsonLine(
      migrationLog,
      (value) => value.event === 'migration_completed',
    );

    apply(
      seedJob(INITIAL_SEED_JOB, 'seed-initial'),
      'persist initial durable queued Plugin Package installation',
    );
    waitForJob(INITIAL_SEED_JOB);
    const initialSeed = lastJsonLine(
      jobLog(INITIAL_SEED_JOB),
      (value) => value.event === 'seed_completed',
    );
    assert.equal(initialSeed.phase, 'initial');
    assert.equal(initialSeed.status, 'admitted');
    assert.equal(initialSeed.state, 'queued');
    assert.equal(initialSeed.lockDigest, fixture.initial.lock.lockDigest);

    for (const resource of recoveryResources(fixture, INITIAL_RECOVERY_JOB)) {
      apply(resource, `apply initial recovery ${resource.kind}`);
    }
    const initialRecovered = waitForJob(INITIAL_RECOVERY_JOB);
    const initialCompleted = lastJsonLine(
      jobLog(INITIAL_RECOVERY_JOB),
      (value) => value.event === 'recovery_completed',
    );
    assert.equal(initialCompleted.recovery.safeToAdmit, true);
    assert.equal(initialCompleted.recovery.remaining, false);
    assert.equal(initialCompleted.recovery.manualRequired, 0);
    const pointerBeforeUpgrade = activePointerEvidence(fixture);

    apply(
      seedJob(UPGRADE_SEED_JOB, 'seed-upgrade'),
      'persist invalid durable queued Plugin Package upgrade',
    );
    waitForJob(UPGRADE_SEED_JOB);
    const upgradeSeed = lastJsonLine(
      jobLog(UPGRADE_SEED_JOB),
      (value) => value.event === 'seed_completed',
    );
    assert.equal(upgradeSeed.phase, 'upgrade');
    assert.equal(upgradeSeed.status, 'admitted');
    assert.equal(upgradeSeed.state, 'queued');
    assert.equal(upgradeSeed.lockDigest, fixture.upgrade.lock.lockDigest);

    for (const resource of recoveryResources(fixture, UPGRADE_RECOVERY_JOB)) {
      apply(resource, `apply upgrade rejection recovery ${resource.kind}`);
    }
    const rejectedUpgrade = waitForJob(UPGRADE_RECOVERY_JOB);
    const rejectionCompleted = lastJsonLine(
      jobLog(UPGRADE_RECOVERY_JOB),
      (value) => value.event === 'recovery_completed',
    );
    assert.equal(rejectionCompleted.recovery.safeToAdmit, true);
    assert.equal(rejectionCompleted.recovery.remaining, false);
    assert.equal(rejectionCompleted.recovery.manualRequired, 0);

    const database = databaseEvidence(fixture);
    const pointerAfterRejection = activePointerEvidence(fixture);
    assert.deepEqual(pointerAfterRejection, pointerBeforeUpgrade);
    const oci = registryEvidence(fixture);
    const rbac = recoveryRbacEvidence();
    assertRuntimeCannotReadPluginAuthority();
    const runtime = applyRuntimeAfterRecovery(
      rejectedUpgrade,
      migrated,
      secrets,
    );
    const initialRecoveryImageId = jobImageId(INITIAL_RECOVERY_JOB);
    const upgradeRecoveryImageId = jobImageId(UPGRADE_RECOVERY_JOB);
    const migrationImageId = jobImageId('ql3-cluster-migration');
    const postgresPod = kubectlJson([
      '-n',
      NAMESPACE,
      'get',
      'pod',
      POSTGRES_NAME,
    ]);
    const report = Object.freeze({
      schema: REPORT_SCHEMA,
      observedAt: new Date().toISOString(),
      sourceRevision: revision,
      passed: true,
      cluster: clusterName,
      architecture: fixtureArchitecture,
      elapsedMs: Date.now() - startedAt,
      images: Object.freeze({
        adminBuildId,
        adminSourceRevision,
        controlBuildId,
        controlSourceRevision,
        postgresRepositoryDigest: POSTGRES_REPOSITORY_DIGEST,
        migrationImageId,
        initialRecoveryImageId,
        upgradeRecoveryImageId,
        postgresImageId: postgresPod.status.containerStatuses[0].imageID,
      }),
      ordering: buildOrderingEvidence(
        migrated,
        initialRecovered,
        rejectedUpgrade,
        runtime,
      ),
      failedUpgrade: Object.freeze({
        recoveryJobUid: rejectedUpgrade.metadata.uid,
        rejectionReason: database.upgradeFailureReason,
        candidateRevisionCount: database.upgradeRevisionCount,
        activePointerUnchanged: true,
      }),
      database,
      oci,
      kubernetes: Object.freeze({
        activePointer: reportActivePointer(pointerAfterRejection),
        rbac,
      }),
      runtime,
      gates: Object.freeze({
        healthyInitialActivation: true,
        invalidUpgradeRejectedBeforeActivation: true,
        activePointerUidUnchanged: true,
        activePointerResourceVersionUnchanged: true,
        activePointerJsonUnchanged: true,
        candidateRevisionAbsent: true,
        exactAuthenticatedOciRequests: true,
        recoveryRbacLeastPrivilege: true,
        runtimeRolledOutAfterRecovery: true,
        passed: true,
      }),
      limitations: Object.freeze([
        'isolated PostgreSQL uses explicit TLS disable; production manifests remain verify-full',
        'the authenticated HTTPS OCI Distribution fixture implements the immutable GET/referrers surface used by the resolver, not a production registry storage implementation',
        'the disposable Kind control plane is single-replica; this gate proves workload ordering, not Kubernetes control-plane HA',
      ]),
    });
    writePrivateReport(reportFile, report);
    process.stdout.write(
      `${JSON.stringify({
        schemaVersion: 1,
        fixture: REPORT_SCHEMA,
        reportWritten: true,
        passed: true,
      })}\n`,
    );
  } catch (error) {
    diagnostics();
    throw error;
  } finally {
    for (const value of Object.keys(secrets)) secrets[value] = '';
    registryCredential.authorization = '';
    registryCredential.file = '';
    if (created && process.env.QL3_KEEP_KIND_CLUSTER !== '1') {
      kind(['delete', 'cluster', '--name', clusterName], {
        label: `delete exact Kind cluster ${clusterName}`,
      });
    } else if (created) {
      process.stderr.write(
        `Kept isolated Kind cluster ${clusterName} by explicit request\n`,
      );
    }
    fs.rmSync(root, { recursive: true, force: true });
  }
}

if (require.main === module) {
  main().catch((error) => {
    process.stderr.write(
      `${JSON.stringify({
        schema: 'qinglong/plugin-package-recovery-e2e-live-failure@v1',
        name: error?.name ?? 'Error',
        message: error?.message ?? 'unknown failure',
      })}\n`,
    );
    process.exitCode = 1;
  });
}

module.exports = {
  REPORT_SCHEMA,
  buildOrderingEvidence,
  privateReportPath,
  writePrivateReport,
};