qinglong/scripts/ql3-approval-management-kubernetes-live-contract.cjs

#!/usr/bin/env node

'use strict';

const assert = require('node:assert/strict');
const crypto = require('node:crypto');
const fs = require('node:fs');
const path = require('node:path');

const {
  K3sDockerLiveFixture,
  run,
  waitFor,
} = require('./lib/ql3-k3s-docker-live.cjs');
const { createMutualTlsPki } = require('./lib/ql3-live-pki.cjs');
const {
  clientTcpProbe,
  createManagementClientExecutor,
  managementHealthStatus,
  patchManagementGeneration,
  podReady,
  podTcpProbe,
  readyManagementPods,
  waitForTwoPreserved,
  waitManagementRollout,
} = require('./lib/ql3-management-kubernetes-live.cjs');
const {
  createManagementIdentityCeremony,
} = require('./lib/ql3-management-live-identity.cjs');
const {
  imageDigest,
  imageTag,
  reviewedOperatorManifest,
} = require('./ql3-cloudnativepg-live-contract.cjs');
const {
  FIXTURE,
  LIMITATIONS,
  validateApprovalManagementKubernetesLiveReport,
} = require('./ql3-approval-management-kubernetes-live-audit.cjs');

const ROOT = path.resolve(__dirname, '..');
const NAMESPACE = 'qinglong3-system';
const DEPLOYMENT = 'ql3-approval-management';
const SERVICE = DEPLOYMENT;
const SERVERNAME = SERVICE + '.' + NAMESPACE + '.svc';
const MANAGEMENT_PATH = '/api/v3/approvals/management';
const POSTGRES_CLUSTER = 'ql3-postgres';
const ISSUER = 'https://identity.qinglong.test/';
const AUDIENCE = 'qinglong3-approval-management';
const LOCK = JSON.parse(
  fs.readFileSync(
    path.join(
      ROOT,
      'deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/operator-lock.json',
    ),
    'utf8',
  ),
);
const OPERATOR_IMAGE = LOCK.operator.image;
const POSTGRES_IMAGE = LOCK.operand.image;
const OPERATOR_VERSION = LOCK.operator.version;
const ADMIN_IMAGE_BASE = 'ql3-approval-manager-live';
const CONTROL_IMAGE_BASE = 'ql3-approval-migration-live';
const ZERO_DIGEST = 'sha256:' + '0'.repeat(64);
const ROLE_NAMES = Object.freeze([
  'ql3_migration',
  'ql3_runtime',
  'ql3_admin',
  'ql3_package_manager',
  'ql3_package_executor',
  'ql3_automation_manager',
  'ql3_approval_manager',
  'ql3_worker_credential_manager',
  'ql3_worker_credential_executor',
  'ql3_worker_ingress',
]);
const ACTION = Object.freeze({
  permission: 'run.start',
  actionType: 'tool.invoke',
  actionRef: 'tool:approval-live',
  actionDigest: 'a'.repeat(64),
  previewDigest: 'b'.repeat(64),
});
const identity = createManagementIdentityCeremony({
  issuer: ISSUER,
  audience: AUDIENCE,
  purpose: 'approval-management',
  tokenType: 'ql3-approval-management+jwt',
  subject: 'approval-operator',
  jtiPrefix: 'ql3-approval-live',
});

function sha256(value) {
  return 'sha256:' + crypto.createHash('sha256').update(value).digest('hex');
}

function randomSecret() {
  return crypto.randomBytes(32).toString('base64url');
}

function eventId(ordinal) {
  assert.ok(Number.isSafeInteger(ordinal) && ordinal >= 1 && ordinal < 1e12);
  return '40000000-0000-4000-8000-' + String(ordinal).padStart(12, '0');
}

function reviewedKey(kid) {
  return identity.reviewedKey(kid);
}

function keyset(generation, keys, revokedKids = []) {
  return identity.keyset(generation, keys, revokedKids);
}

function assertion(key, suffix) {
  return identity.assertion(key, suffix);
}

function assertionForSubject(key, subject, suffix = crypto.randomUUID()) {
  return identity.assertionForSubject(key, subject, 'subject-' + suffix);
}

function weakAssertion(key, suffix = crypto.randomUUID()) {
  return identity.weakAssertion(key, suffix);
}

function commandBase(projectId, approvalRequestId, requestId, ordinal) {
  return Object.freeze({
    projectId,
    approvalRequestId,
    requestId,
    auditEventId: eventId(ordinal),
    failureAuditEventId: eventId(ordinal + 500_000),
  });
}

function inspectCommand(projectId, approvalRequestId, requestId, ordinal) {
  return Object.freeze({
    schemaVersion: 1,
    operation: 'approval.inspect',
    request: commandBase(projectId, approvalRequestId, requestId, ordinal),
  });
}

function decisionCommand(
  projectId,
  approvalRequestId,
  requestId,
  decisionId,
  ordinal,
) {
  return Object.freeze({
    schemaVersion: 1,
    operation: 'approval.decide',
    request: Object.freeze({
      ...commandBase(projectId, approvalRequestId, requestId, ordinal),
      expectedVersion: 1,
      expectedAction: ACTION,
      decisionId,
      decision: 'approved',
      reasonCode: 'reviewed',
    }),
  });
}

function imageIdDigest(image) {
  assert.match(image.Id, /^sha256:[a-f0-9]{64}$/);
  return image.Id;
}

function localManifest(rendered, imageName, localImage) {
  const placeholder = imageName + '@' + ZERO_DIGEST;
  assert.equal(rendered.split(placeholder).length - 1, 1);
  return rendered.replace(placeholder, localImage);
}

function applySecret(fixture, name, type, stringData) {
  fixture.apply({
    apiVersion: 'v1',
    kind: 'Secret',
    metadata: { name, namespace: NAMESPACE },
    type,
    stringData,
  });
}

function psql(fixture, podName, sql) {
  return fixture.kubectl(
    [
      '-n',
      NAMESPACE,
      'exec',
      podName,
      '-c',
      'postgres',
      '--',
      'psql',
      '-U',
      'postgres',
      '-d',
      'qinglong',
      '--no-psqlrc',
      '--tuples-only',
      '--no-align',
      '--set',
      'ON_ERROR_STOP=1',
      '--command',
      sql,
    ],
    { capture: true, quiet: true },
  ).stdout;
}

function currentPrimaryPod(fixture) {
  const primaryName = fixture.kubectlJson([
    '-n',
    NAMESPACE,
    'get',
    'cluster',
    POSTGRES_CLUSTER,
  ]).status.currentPrimary;
  assert.match(primaryName || '', /^ql3-postgres-[1-9][0-9]*$/);
  const pods = fixture.kubectlJson([
    '-n',
    NAMESPACE,
    'get',
    'pods',
    '-l',
    'cnpg.io/cluster=' + POSTGRES_CLUSTER,
  ]).items;
  const primary = pods.find((pod) => pod.metadata.name === primaryName);
  assert.ok(primary, 'CloudNativePG primary Pod not found');
  return primary;
}

function sqlString(value) {
  assert.equal(typeof value, 'string');
  return "'" + value.replaceAll("'", "''") + "'";
}

function loadApprovalContract() {
  const file = path.join(
    ROOT,
    'packages/ql3-runtime-core/dist/approved-action/approvedAction.js',
  );
  if (!fs.existsSync(file)) {
    throw new Error(
      'runtime-core must be built before the Approval Kubernetes live contract',
    );
  }
  return require(file);
}

function seedApproval(fixture, primaryPod, projectId, approvalRequestId) {
  const { approvalRequestDigest, createApprovalRequest } =
    loadApprovalContract();
  const requestedAtMs = Date.now() - 1_000;
  const request = createApprovalRequest({
    id: approvalRequestId,
    projectId,
    action: ACTION,
    risk: 'high',
    decisionMode: 'human_confirmation',
    requestedBy: { type: 'agent', id: 'approval-requester' },
    requestedAtMs,
    expiresAtMs: requestedAtMs + 60 * 60 * 1000,
    requestFence: { projectVersion: 1, bindingVersion: 1 },
  });
  const requestJson = JSON.stringify(request);
  assert.equal(requestJson.includes('$ql3json$'), false);
  psql(
    fixture,
    primaryPod.metadata.name,
    [
      'INSERT INTO "ql3"."projects" (',
      '  id, name, slug, status, version, created_at_ms, updated_at_ms',
      ') VALUES (' +
        sqlString(projectId) +
        ", 'Approval Live', " +
        sqlString(projectId) +
        ", 'active', 1, " +
        String(requestedAtMs) +
        ', ' +
        String(requestedAtMs) +
        ');',
      'INSERT INTO "ql3"."project_role_bindings" (',
      '  project_id, subject_type, subject_id, version, state, role,',
      '  mutation_id, changed_by_type, changed_by_id, created_at_ms',
      ') VALUES (' +
        sqlString(projectId) +
        ", 'user', 'approval-operator', 1, 'active', 'owner'," +
        " 'approval-live-owner-v1', 'system', 'live-contract', " +
        String(requestedAtMs) +
        ');',
      'INSERT INTO "ql3"."approval_requests" (',
      '  request_id, project_id, version, state, action_type, action_ref,',
      '  action_digest, preview_digest, requested_by_type, requested_by_id,',
      '  decision_id, consumption_id, dispatch_id, expires_at_ms,',
      '  request_json, request_digest, updated_at_ms',
      ') VALUES (' +
        sqlString(approvalRequestId) +
        ', ' +
        sqlString(projectId) +
        ", 1, 'pending', " +
        sqlString(ACTION.actionType) +
        ', ' +
        sqlString(ACTION.actionRef) +
        ', ' +
        sqlString(ACTION.actionDigest) +
        ', ' +
        sqlString(ACTION.previewDigest) +
        ", 'agent', 'approval-requester', NULL, NULL, NULL, " +
        String(request.expiresAtMs) +
        ', $ql3json$' +
        requestJson +
        '$ql3json$::jsonb, ' +
        sqlString(approvalRequestDigest(request)) +
        ', ' +
        String(requestedAtMs) +
        ');',
    ].join('\n'),
  );
  return request;
}

function managerOptions(fixture) {
  return {
    fixture,
    namespace: NAMESPACE,
    deployment: DEPLOYMENT,
    description: 'two Ready approval manager Pods on distinct nodes',
  };
}

function patchGeneration(fixture, generation, annotations = {}) {
  patchManagementGeneration({
    ...managerOptions(fixture),
    generation,
    annotations,
  });
}

function healthStatus(fixture, pod, route) {
  return managementHealthStatus({
    fixture,
    namespace: NAMESPACE,
    podName: pod.metadata.name,
    port: 8447,
    route,
    servername: SERVERNAME,
    caFile: '/var/run/secrets/qinglong3/approval-management-tls/ca.crt',
  });
}

function privateReportPath(argv) {
  if (
    argv.length !== 1 ||
    !argv[0].startsWith('--report=') ||
    !path.isAbsolute(argv[0].slice('--report='.length))
  ) {
    throw new Error(
      'usage: ql3-approval-management-kubernetes-live-contract ' +
        '--report=/absolute/private-report.json',
    );
  }
  const reportFile = argv[0].slice('--report='.length);
  if (fs.existsSync(reportFile)) {
    throw new Error('refusing to overwrite the Approval live report');
  }
  const parent = fs.lstatSync(path.dirname(reportFile));
  if (!parent.isDirectory() || parent.isSymbolicLink()) {
    throw new Error('Approval live report parent must be a real directory');
  }
  return reportFile;
}

async function main(argv = process.argv.slice(2)) {
  const reportFile = privateReportPath(argv);
  if (process.env.QL3_APPROVAL_MANAGEMENT_KUBERNETES_LIVE !== '1') {
    throw new Error(
      'Refusing to mutate Docker/Kubernetes without ' +
        'QL3_APPROVAL_MANAGEMENT_KUBERNETES_LIVE=1',
    );
  }
  const operatorManifestFile = process.env.QL3_CNPG_OPERATOR_MANIFEST_FILE;
  if (!operatorManifestFile) {
    throw new Error('QL3_CNPG_OPERATOR_MANIFEST_FILE is required');
  }
  const reviewedManifest = reviewedOperatorManifest(operatorManifestFile);
  const fixture = new K3sDockerLiveFixture({ prefix: 'ql3-approval-live' });
  const suffix =
    process.pid.toString(36) + '-' + crypto.randomBytes(3).toString('hex');
  const adminImage = ADMIN_IMAGE_BASE + ':' + suffix;
  const controlImage = CONTROL_IMAGE_BASE + ':' + suffix;
  let adminImageBuilt = false;
  let controlImageBuilt = false;
  try {
    const nodes = await fixture.start();
    const architecture = fixture.inspectImage(fixture.k3sImage).Architecture;
    assert.ok(['amd64', 'arm64'].includes(architecture));

    for (const reviewedImage of [OPERATOR_IMAGE, POSTGRES_IMAGE]) {
      run(fixture.docker, ['pull', reviewedImage]);
      const inspected = fixture.inspectImage(reviewedImage);
      assert.ok(
        inspected.RepoDigests?.some((entry) =>
          entry.endsWith('@' + imageDigest(reviewedImage)),
        ),
        'Docker did not retain reviewed digest for ' + reviewedImage,
      );
      const preloadTag = imageTag(reviewedImage);
      run(fixture.docker, ['tag', reviewedImage, preloadTag]);
      fixture.loadImage(preloadTag, path.basename(preloadTag) + '.tar');
    }

    const sourceRevision = run('git', ['rev-parse', 'HEAD'], {
      capture: true,
      quiet: true,
    }).stdout;
    run(fixture.docker, [
      'build',
      '--file',
      'deploy/containers/ql3-cluster-admin/Dockerfile',
      '--tag',
      adminImage,
      '--build-arg',
      'SOURCE_REVISION=' + sourceRevision,
      '.',
    ]);
    adminImageBuilt = true;
    fixture.loadImage(adminImage, 'approval-admin.tar');
    run(fixture.docker, [
      'build',
      '--file',
      'deploy/containers/ql3-cluster-control/Dockerfile',
      '--tag',
      controlImage,
      '--build-arg',
      'SOURCE_REVISION=' + sourceRevision,
      '.',
    ]);
    controlImageBuilt = true;
    fixture.loadImage(controlImage, 'approval-migration.tar');
    const adminImageInfo = fixture.inspectImage(adminImage);
    const postgresImageInfo = fixture.inspectImage(POSTGRES_IMAGE);
    const k3sImageInfo = fixture.inspectImage(fixture.k3sImage);

    fixture.kubectl(['apply', '--server-side', '-f', reviewedManifest]);
    fixture.kubectl([
      '-n',
      'cnpg-system',
      'set',
      'image',
      'deployment/cnpg-controller-manager',
      'manager=' + imageTag(OPERATOR_IMAGE),
    ]);
    fixture.kubectl([
      'wait',
      '--for=condition=Established',
      'crd/clusters.postgresql.cnpg.io',
      'crd/databaseroles.postgresql.cnpg.io',
      'crd/databases.postgresql.cnpg.io',
      '--timeout=5m',
    ]);
    fixture.kubectl([
      '-n',
      'cnpg-system',
      'rollout',
      'status',
      'deployment/cnpg-controller-manager',
      '--timeout=5m',
    ]);

    fixture.kubectl([
      'apply',
      '-f',
      'deploy/kubernetes/ql3-cluster/base/namespace.yaml',
    ]);
    fixture.kubectl([
      '-n',
      NAMESPACE,
      'apply',
      '-f',
      'deploy/kubernetes/ql3-cluster/base/service-account.yaml',
    ]);
    const passwords = Object.fromEntries(
      ROLE_NAMES.map((role) => [role, randomSecret()]),
    );
    for (const role of ROLE_NAMES) {
      applySecret(
        fixture,
        'ql3-postgres-' +
          role.replace(/^ql3_/, '').replaceAll('_', '-') +
          '-auth',
        'kubernetes.io/basic-auth',
        { username: role, password: passwords[role] },
      );
    }
    const databaseManifest = fixture
      .kubectl(
        ['kustomize', 'deploy/kubernetes/ql3-cluster/operators/cloudnative-pg'],
        { capture: true, quiet: true },
      )
      .stdout.replace(POSTGRES_IMAGE, imageTag(POSTGRES_IMAGE));
    assert.equal(databaseManifest.includes(POSTGRES_IMAGE), false);
    fixture.kubectl(['apply', '-f', '-'], {
      input: databaseManifest + '\n',
    });
    fixture.kubectl([
      '-n',
      NAMESPACE,
      'wait',
      '--for=condition=Ready',
      'cluster/' + POSTGRES_CLUSTER,
      '--timeout=20m',
    ]);
    const databasePods = (
      await waitFor('three ready CloudNativePG instances', 600_000, () => {
        const pods = fixture
          .kubectlJson([
            '-n',
            NAMESPACE,
            'get',
            'pods',
            '-l',
            'cnpg.io/cluster=' + POSTGRES_CLUSTER,
          ])
          .items.filter(podReady);
        return pods.length === 3
          ? { ready: true, value: pods }
          : {
              ready: false,
              fact: pods.length + '/3 ready database Pods',
            };
      })
    ).value;

    const migrationManifest = localManifest(
      fixture.kubectl(
        [
          'kustomize',
          'deploy/kubernetes/ql3-cluster/operations/cloudnative-pg',
        ],
        { capture: true, quiet: true },
      ).stdout,
      'registry.example.com/qinglong/qinglong3-cluster-control',
      controlImage,
    );
    fixture.kubectl(['create', '-f', '-'], {
      input: migrationManifest + '\n',
    });
    fixture.kubectl([
      '-n',
      NAMESPACE,
      'wait',
      '--for=condition=Complete',
      'job/ql3-cluster-migration',
      '--timeout=10m',
    ]);
    const migrationPrimary = currentPrimaryPod(fixture);
    const migrationState = JSON.parse(
      psql(
        fixture,
        migrationPrimary.metadata.name,
        [
          'SELECT json_build_object(',
          "  'migrationCount', (SELECT count(*)::integer",
          '    FROM "ql3"."schema_migrations"),',
          "  'controlCoreCapability', (SELECT contract_version::integer",
          '    FROM "ql3"."schema_capabilities"',
          "    WHERE contract_name = 'control-core'))",
        ].join('\n'),
      ),
    );
    assert.deepEqual(migrationState, {
      migrationCount: 54,
      controlCoreCapability: 53,
    });

    const projectId = 'approval-live-' + suffix;
    const approvalRequestId = 'approval-request-' + suffix;
    const decisionId = 'approval-decision-' + suffix;
    const primary = currentPrimaryPod(fixture);
    seedApproval(fixture, primary, projectId, approvalRequestId);

    const pki = createMutualTlsPki({
      directory: fixture.temporary,
      servername: SERVERNAME,
      label: 'QL3 Approval Management Live',
      run,
      crypto,
    });
    let pkiMaterial = pki.read();
    const oldKey = reviewedKey('approval-live-key-1');
    const newKey = reviewedKey('approval-live-key-2');
    const keysets = [
      keyset(1, [oldKey]),
      keyset(2, [oldKey, newKey]),
      keyset(3, [oldKey, newKey], [oldKey.kid]),
    ];
    const applyIdentity = (document) =>
      applySecret(fixture, DEPLOYMENT + '-identity', 'Opaque', {
        'keyset.json': JSON.stringify(document) + '\n',
      });
    const applyTls = () =>
      applySecret(fixture, DEPLOYMENT + '-tls', 'kubernetes.io/tls', {
        'tls.crt': pkiMaterial.serverCertificate,
        'tls.key': pkiMaterial.serverKey,
        'ca.crt': pkiMaterial.ca,
        'client.crl': pkiMaterial.clientCrl,
      });
    applyIdentity(keysets[0]);
    applyTls();

    const previousBundleSha256 = pki.bundleSha256();
    const caDigest = sha256(pkiMaterial.ca);
    const crlDigest = sha256(pkiMaterial.clientCrl);
    let managerManifest = localManifest(
      fixture.kubectl(
        [
          'kustomize',
          'deploy/kubernetes/ql3-cluster/operations/approval-management/cloudnative-pg',
        ],
        { capture: true, quiet: true },
      ).stdout,
      'registry.example.com/qinglong/qinglong3-cluster-admin',
      adminImage,
    );
    assert.equal(managerManifest.split(ZERO_DIGEST).length - 1, 2);
    managerManifest = managerManifest
      .replace(ZERO_DIGEST, caDigest)
      .replace(ZERO_DIGEST, crlDigest);
    fixture.kubectl(['apply', '-f', '-'], {
      input: managerManifest + '\n',
    });
    waitManagementRollout(managerOptions(fixture));
    let managerPods = await readyManagementPods(managerOptions(fixture));

    const deployment = fixture.kubectlJson([
      '-n',
      NAMESPACE,
      'get',
      'deployment',
      DEPLOYMENT,
    ]);
    assert.equal(deployment.spec.replicas, 2);
    assert.equal(deployment.spec.strategy.rollingUpdate.maxUnavailable, 0);
    assert.equal(
      deployment.spec.template.spec.automountServiceAccountToken,
      false,
    );
    assert.equal(
      deployment.spec.template.spec.affinity.podAntiAffinity
        .requiredDuringSchedulingIgnoredDuringExecution.length,
      1,
    );
    assert.equal(
      fixture.kubectlJson(['-n', NAMESPACE, 'get', 'pdb', DEPLOYMENT]).spec
        .minAvailable,
      1,
    );
    for (const pod of managerPods) {
      assert.equal(pod.spec.serviceAccountName, DEPLOYMENT);
      assert.equal(pod.spec.automountServiceAccountToken, false);
      assert.equal(
        pod.spec.volumes.some((volume) =>
          volume.projected?.sources?.some(
            (source) => source.serviceAccountToken !== undefined,
          ),
        ),
        false,
      );
    }

    const executeClient = createManagementClientExecutor({
      fixture,
      namespace: NAMESPACE,
      servername: SERVERNAME,
      port: 8447,
      managementPath: MANAGEMENT_PATH,
      adminImage,
      ca: pkiMaterial.ca,
      serviceAccount: 'ql3-approval-management-client',
      appName: 'ql3-approval-management-client',
      component: 'approval-management-client',
      networkPolicyLabel: 'qinglong.io/approval-management-client',
      clientCliPath:
        '/opt/qinglong/node_modules/@qinglong/cluster-admin/' +
        'dist/approval-management/approvalManagementClientCli.js',
      description: 'approval management',
    });
    const oldAssertion = () => assertion(oldKey);
    const newAssertion = () => assertion(newKey);
    const initialRequests = await Promise.all([
      executeClient(
        {
          name: 'ql3-approval-inspect-initial-a',
          target: managerPods[0],
          command: inspectCommand(
            projectId,
            approvalRequestId,
            'approval-inspect-initial-a',
            1,
          ),
          bearer: oldAssertion(),
          clientCertificate: pkiMaterial.oldClientCertificate,
          clientKey: pkiMaterial.oldClientKey,
        },
        { statusCode: 200, resultStatus: ['found'] },
      ),
      executeClient(
        {
          name: 'ql3-approval-inspect-initial-b',
          target: managerPods[1],
          command: inspectCommand(
            projectId,
            approvalRequestId,
            'approval-inspect-initial-b',
            2,
          ),
          bearer: oldAssertion(),
          clientCertificate: pkiMaterial.newClientCertificate,
          clientKey: pkiMaterial.newClientKey,
        },
        { statusCode: 200, resultStatus: ['found'] },
      ),
    ]);
    assert.deepEqual(
      initialRequests.map((entry) => entry.output.result.status),
      ['found', 'found'],
    );
    const weakUserRejected = await executeClient(
      {
        name: 'ql3-approval-inspect-weak-user',
        target: managerPods[0],
        command: inspectCommand(
          projectId,
          approvalRequestId,
          'approval-inspect-weak-user',
          3,
        ),
        bearer: weakAssertion(oldKey),
        clientCertificate: pkiMaterial.newClientCertificate,
        clientKey: pkiMaterial.newClientKey,
      },
      { statusCode: 401, responseCode: 'authentication_required' },
    );
    const outsiderDenied = await executeClient(
      {
        name: 'ql3-approval-inspect-outsider',
        target: managerPods[1],
        command: inspectCommand(
          projectId,
          approvalRequestId,
          'approval-inspect-outsider',
          10,
        ),
        bearer: assertionForSubject(oldKey, 'approval-outsider'),
        clientCertificate: pkiMaterial.newClientCertificate,
        clientKey: pkiMaterial.newClientKey,
      },
      { statusCode: 403, responseCode: 'forbidden' },
    );

    const generation1Uids = new Set(managerPods.map((pod) => pod.metadata.uid));
    applyIdentity(keysets[1]);
    patchGeneration(fixture, 2);
    const generation2 = await waitForTwoPreserved({
      ...managerOptions(fixture),
      excludedUids: generation1Uids,
      expectedGeneration: 2,
      description:
        'zero-unavailable approval manager identity generation 2 rollout',
    });
    managerPods = generation2.pods;
    const overlapOld = await executeClient(
      {
        name: 'ql3-approval-inspect-overlap-old',
        target: managerPods[0],
        command: inspectCommand(
          projectId,
          approvalRequestId,
          'approval-inspect-overlap-old',
          4,
        ),
        bearer: oldAssertion(),
        clientCertificate: pkiMaterial.newClientCertificate,
        clientKey: pkiMaterial.newClientKey,
      },
      { statusCode: 200, resultStatus: ['found'] },
    );
    const decided = await executeClient(
      {
        name: 'ql3-approval-decide-overlap-new',
        target: managerPods[1],
        command: decisionCommand(
          projectId,
          approvalRequestId,
          'approval-decide-overlap-new',
          decisionId,
          5,
        ),
        bearer: newAssertion(),
        clientCertificate: pkiMaterial.newClientCertificate,
        clientKey: pkiMaterial.newClientKey,
      },
      { statusCode: 200, resultStatus: ['decided'] },
    );
    assert.equal(decided.output.result.approval.state, 'approved');
    assert.equal(decided.output.result.approval.version, 2);

    const generation2Uids = new Set(managerPods.map((pod) => pod.metadata.uid));
    applyIdentity(keysets[2]);
    patchGeneration(fixture, 3);
    const generation3 = await waitForTwoPreserved({
      ...managerOptions(fixture),
      excludedUids: generation2Uids,
      expectedGeneration: 3,
      description:
        'zero-unavailable approval manager identity generation 3 rollout',
    });
    managerPods = generation3.pods;
    const rejectedOldKey = await executeClient(
      {
        name: 'ql3-approval-inspect-revoked-key',
        target: managerPods[0],
        command: inspectCommand(
          projectId,
          approvalRequestId,
          'approval-inspect-revoked-key',
          6,
        ),
        bearer: oldAssertion(),
        clientCertificate: pkiMaterial.newClientCertificate,
        clientKey: pkiMaterial.newClientKey,
      },
      { statusCode: 401, responseCode: 'authentication_required' },
    );
    const replayed = await executeClient(
      {
        name: 'ql3-approval-decide-active-key',
        target: managerPods[1],
        command: decisionCommand(
          projectId,
          approvalRequestId,
          'approval-decide-active-key',
          decisionId,
          7,
        ),
        bearer: newAssertion(),
        clientCertificate: pkiMaterial.newClientCertificate,
        clientKey: pkiMaterial.newClientKey,
      },
      { statusCode: 200, resultStatus: ['existing'] },
    );

    applyIdentity(keysets[1]);
    patchGeneration(fixture, 'rollback-2');
    const rollback = await waitFor(
      'approval identity ledger rollback surge failure',
      180_000,
      () => {
        const pods = fixture
          .kubectlJson([
            '-n',
            NAMESPACE,
            'get',
            'pods',
            '-l',
            'app.kubernetes.io/name=' + DEPLOYMENT,
          ])
          .items.filter((pod) => pod.metadata.deletionTimestamp === undefined);
        const ready = pods.filter(podReady);
        const candidate = pods.find(
          (pod) =>
            !managerPods.some(
              (current) => current.metadata.uid === pod.metadata.uid,
            ) &&
            pod.status.containerStatuses?.[0] &&
            !pod.status.containerStatuses[0].ready &&
            (pod.status.containerStatuses[0].restartCount > 0 ||
              pod.status.containerStatuses[0].state?.waiting?.reason ===
                'CrashLoopBackOff'),
        );
        return ready.length === 2 && candidate
          ? { ready: true, value: candidate }
          : {
              ready: false,
              fact:
                ready.length +
                ' ready Pods; rollback candidate=' +
                Boolean(candidate),
            };
      },
    );
    applyIdentity(keysets[2]);
    patchGeneration(fixture, '3-rollback-recovered');
    fixture.kubectl([
      '-n',
      NAMESPACE,
      'delete',
      'pod',
      rollback.value.metadata.name,
      '--grace-period=0',
      '--force',
      '--wait=true',
    ]);
    waitManagementRollout(managerOptions(fixture));
    managerPods = await readyManagementPods(managerOptions(fixture));

    const previousSerialSha256 = pki.oldSerialSha256();
    pki.revokeOldClient();
    pkiMaterial = pki.read();
    const currentBundleSha256 = pki.bundleSha256();
    assert.notEqual(currentBundleSha256, previousBundleSha256);
    applyTls();
    const preCertificateUids = new Set(
      managerPods.map((pod) => pod.metadata.uid),
    );
    patchGeneration(fixture, '3-client-crl-2', {
      'qinglong.io/approval-management-client-ca-sha256': sha256(
        pkiMaterial.ca,
      ),
      'qinglong.io/approval-management-client-crl-sha256': sha256(
        pkiMaterial.clientCrl,
      ),
    });
    const certificateRollout = await waitForTwoPreserved({
      ...managerOptions(fixture),
      excludedUids: preCertificateUids,
      expectedGeneration: '3-client-crl-2',
      description:
        'zero-unavailable approval manager client certificate rollout',
    });
    managerPods = certificateRollout.pods;
    const revokedCertificate = await executeClient(
      {
        name: 'ql3-approval-decide-revoked-cert',
        target: managerPods[0],
        command: decisionCommand(
          projectId,
          approvalRequestId,
          'approval-decide-revoked-cert',
          decisionId,
          8,
        ),
        bearer: newAssertion(),
        clientCertificate: pkiMaterial.oldClientCertificate,
        clientKey: pkiMaterial.oldClientKey,
      },
      {
        statusCode: 401,
        responseCode: 'client_certificate_required',
      },
    );
    const activeCertificate = await executeClient(
      {
        name: 'ql3-approval-decide-active-cert',
        target: managerPods[1],
        command: decisionCommand(
          projectId,
          approvalRequestId,
          'approval-decide-active-cert',
          decisionId,
          9,
        ),
        bearer: newAssertion(),
        clientCertificate: pkiMaterial.newClientCertificate,
        clientKey: pkiMaterial.newClientKey,
      },
      { statusCode: 200, resultStatus: ['existing'] },
    );

    const primaryBeforeFailover = currentPrimaryPod(fixture);
    fixture.kubectl([
      '-n',
      NAMESPACE,
      'delete',
      'pod',
      primaryBeforeFailover.metadata.name,
      '--grace-period=0',
      '--force',
      '--wait=false',
    ]);
    const promoted = await waitFor(
      'CloudNativePG primary promotion',
      600_000,
      () => {
        const status = fixture.kubectlJson([
          '-n',
          NAMESPACE,
          'get',
          'cluster',
          POSTGRES_CLUSTER,
        ]).status;
        return status.currentPrimary &&
          status.currentPrimary !== primaryBeforeFailover.metadata.name &&
          Number(status.readyInstances) >= 2
          ? { ready: true, value: status.currentPrimary }
          : {
              ready: false,
              fact:
                'primary=' +
                (status.currentPrimary || 'none') +
                ' ready=' +
                String(status.readyInstances ?? 0),
            };
      },
    );
    await waitFor('CloudNativePG recovery to three instances', 900_000, () => {
      const status = fixture.kubectlJson([
        '-n',
        NAMESPACE,
        'get',
        'cluster',
        POSTGRES_CLUSTER,
      ]).status;
      return Number(status.readyInstances) === 3
        ? { ready: true, value: status }
        : {
            ready: false,
            fact:
              String(status.readyInstances ?? 0) +
              '/3 ready database instances',
          };
    });

    const databaseService = fixture.kubectlJson([
      '-n',
      NAMESPACE,
      'get',
      'service',
      POSTGRES_CLUSTER + '-rw',
    ]);
    const databaseSelector = databaseService.spec.selector;
    fixture.kubectl([
      '-n',
      NAMESPACE,
      'patch',
      'service',
      POSTGRES_CLUSTER + '-rw',
      '--type=merge',
      '-p',
      JSON.stringify({
        spec: {
          selector: { ...databaseSelector, 'ql3.invalid': 'true' },
        },
      }),
    ]);
    const unavailable = await Promise.all(
      managerPods.map((pod, index) =>
        executeClient(
          {
            name: 'ql3-approval-database-unavailable-' + String(index + 1),
            target: pod,
            command: decisionCommand(
              projectId,
              approvalRequestId,
              'approval-database-unavailable-' + String(index + 1),
              decisionId,
              20 + index,
            ),
            bearer: newAssertion(),
            clientCertificate: pkiMaterial.newClientCertificate,
            clientKey: pkiMaterial.newClientKey,
          },
          { statusCode: 503, responseCode: 'unavailable' },
        ),
      ),
    );
    assert.deepEqual(
      unavailable.map((entry) => entry.statusCode),
      [503, 503],
    );
    await waitFor('approval manager readiness withdrawal', 60_000, () => {
      const current = fixture.kubectlJson([
        '-n',
        NAMESPACE,
        'get',
        'deployment',
        DEPLOYMENT,
      ]);
      return Number(current.status.readyReplicas ?? 0) === 0
        ? { ready: true, value: current }
        : {
            ready: false,
            fact: String(current.status.readyReplicas ?? 0) + ' ready replicas',
          };
    });
    assert.deepEqual(
      managerPods.map((pod) => healthStatus(fixture, pod, '/readyz')),
      [503, 503],
    );
    assert.deepEqual(
      managerPods.map((pod) => healthStatus(fixture, pod, '/livez')),
      [200, 200],
    );
    fixture.kubectl([
      '-n',
      NAMESPACE,
      'patch',
      'service',
      POSTGRES_CLUSTER + '-rw',
      '--type=json',
      '-p',
      JSON.stringify([
        {
          op: 'replace',
          path: '/spec/selector',
          value: databaseSelector,
        },
      ]),
    ]);
    await waitFor('restored CloudNativePG service endpoint', 120_000, () => {
      const endpoints = fixture.kubectlJson([
        '-n',
        NAMESPACE,
        'get',
        'endpoints',
        POSTGRES_CLUSTER + '-rw',
      ]);
      const count = endpoints.subsets?.flatMap(
        (subset) => subset.addresses ?? [],
      ).length;
      return count >= 1
        ? { ready: true, value: count }
        : {
            ready: false,
            fact: String(count ?? 0) + ' service endpoints',
          };
    });
    assert.deepEqual(
      managerPods.map((pod) => healthStatus(fixture, pod, '/readyz')),
      [503, 503],
    );
    const staleUids = new Set(managerPods.map((pod) => pod.metadata.uid));
    patchGeneration(fixture, '3-database-recovered');
    managerPods = await readyManagementPods({
      ...managerOptions(fixture),
      excludedUids: staleUids,
      expectedGeneration: '3-database-recovered',
    });
    const recoveredRequests = await Promise.all(
      managerPods.map((pod, index) =>
        executeClient(
          {
            name: 'ql3-approval-database-recovered-' + String(index + 1),
            target: pod,
            command: decisionCommand(
              projectId,
              approvalRequestId,
              'approval-database-recovered-' + String(index + 1),
              decisionId,
              30 + index,
            ),
            bearer: newAssertion(),
            clientCertificate: pkiMaterial.newClientCertificate,
            clientKey: pkiMaterial.newClientKey,
          },
          { statusCode: 200, resultStatus: ['existing'] },
        ),
      ),
    );
    assert.deepEqual(
      recoveredRequests.map((entry) => entry.output.result.status),
      ['existing', 'existing'],
    );

    const finalPrimary = currentPrimaryPod(fixture);
    const durable = JSON.parse(
      psql(
        fixture,
        finalPrimary.metadata.name,
        [
          'SELECT json_build_object(',
          "  'approvalVersion', (SELECT version::integer",
          '    FROM "ql3"."approval_requests"',
          '    WHERE request_id = ' + sqlString(approvalRequestId) + '),',
          "  'approvalState', (SELECT state",
          '    FROM "ql3"."approval_requests"',
          '    WHERE request_id = ' + sqlString(approvalRequestId) + '),',
          "  'decisionId', (SELECT decision_id",
          '    FROM "ql3"."approval_requests"',
          '    WHERE request_id = ' + sqlString(approvalRequestId) + '),',
          "  'allowedAuditCount', (SELECT count(*)::integer",
          '    FROM "ql3"."security_audit_events"',
          '    WHERE project_id = ' +
            sqlString(projectId) +
            " AND outcome = 'allowed'),",
          "  'deniedAuditCount', (SELECT count(*)::integer",
          '    FROM "ql3"."security_audit_events"',
          '    WHERE project_id = ' +
            sqlString(projectId) +
            " AND outcome = 'denied'),",
          "  'decisionAuditCount', (SELECT count(*)::integer",
          '    FROM "ql3"."security_audit_events"',
          '    WHERE project_id = ' +
            sqlString(projectId) +
            " AND operation_id = 'approval.decide'" +
            " AND outcome = 'allowed'),",
          "  'identityGeneration', (SELECT generation::integer",
          '    FROM "ql3"."plugin_package_identity_keyset_ledger"',
          "    WHERE authority = 'approval-management'),",
          "  'migrationCount', (SELECT count(*)::integer",
          '    FROM "ql3"."schema_migrations"),',
          "  'controlCoreCapability', (SELECT contract_version::integer",
          '    FROM "ql3"."schema_capabilities"',
          "    WHERE contract_name = 'control-core'),",
          "  'postgresVersionNumber',",
          "    current_setting('server_version_num')::integer,",
          "  'currentUser', current_user)",
        ].join('\n'),
      ),
    );
    assert.deepEqual(durable, {
      approvalVersion: 2,
      approvalState: 'approved',
      decisionId,
      allowedAuditCount: 4,
      deniedAuditCount: 1,
      decisionAuditCount: 1,
      identityGeneration: 3,
      migrationCount: 54,
      controlCoreCapability: 53,
      postgresVersionNumber: 180004,
      currentUser: 'postgres',
    });

    const roleList = ROLE_NAMES.map(sqlString).join(',');
    const roleRows = JSON.parse(
      psql(
        fixture,
        finalPrimary.metadata.name,
        [
          'SELECT json_agg(json_build_object(',
          "  'name', rolname,",
          "  'login', rolcanlogin,",
          "  'superuser', rolsuper,",
          "  'createDatabase', rolcreatedb,",
          "  'createRole', rolcreaterole,",
          "  'replication', rolreplication,",
          "  'bypassRls', rolbypassrls) ORDER BY rolname)",
          'FROM pg_roles WHERE rolname IN (' + roleList + ')',
        ].join('\n'),
      ),
    );
    assert.deepEqual(
      roleRows.map((role) => role.name),
      [...ROLE_NAMES].sort(),
    );
    const rolesLeastPrivilege = roleRows.every(
      (role) =>
        role.login === true &&
        role.superuser === false &&
        role.createDatabase === false &&
        role.createRole === false &&
        role.replication === false &&
        role.bypassRls === false,
    );
    assert.equal(rolesLeastPrivilege, true);

    const canI = (verb, resource) => {
      const result = fixture.kubectl(
        [
          'auth',
          'can-i',
          verb,
          resource,
          '-n',
          NAMESPACE,
          '--as=system:serviceaccount:' + NAMESPACE + ':' + DEPLOYMENT,
        ],
        { capture: true, quiet: true, allowFailure: true },
      );
      assert.equal(
        result.status,
        result.stdout === 'yes' ? 0 : 1,
        'unexpected kubectl auth can-i result: ' + result.stdout,
      );
      return result.stdout;
    };
    assert.equal(canI('get', 'secrets'), 'no');
    assert.equal(canI('patch', 'deployments.apps'), 'no');

    const managerServiceIp = fixture.kubectlJson([
      '-n',
      NAMESPACE,
      'get',
      'service',
      SERVICE,
    ]).spec.clusterIP;
    const networkProbe = {
      fixture,
      namespace: NAMESPACE,
      adminImage,
      appName: 'ql3-approval-network-probe',
      networkPolicyLabel: 'qinglong.io/approval-management-client',
    };
    const labelledClientAllowed = await clientTcpProbe({
      ...networkProbe,
      name: 'ql3-approval-network-labelled',
      targetHost: managerServiceIp,
      port: 8447,
      labelled: true,
      expectedConnected: true,
    });
    const unlabelledClientDenied = await clientTcpProbe({
      ...networkProbe,
      name: 'ql3-approval-network-unlabelled',
      targetHost: managerServiceIp,
      port: 8447,
      labelled: false,
      expectedConnected: false,
    });
    const wrongPortDenied = await clientTcpProbe({
      ...networkProbe,
      name: 'ql3-approval-network-wrong-port',
      targetHost: managerServiceIp,
      port: 8446,
      labelled: true,
      expectedConnected: false,
    });
    const kubernetesServiceIp = fixture.kubectlJson([
      'get',
      'service',
      'kubernetes',
      '-n',
      'default',
    ]).spec.clusterIP;
    const postgresServiceIp = fixture.kubectlJson([
      '-n',
      NAMESPACE,
      'get',
      'service',
      POSTGRES_CLUSTER + '-rw',
    ]).spec.clusterIP;
    const podProbe = {
      fixture,
      namespace: NAMESPACE,
      podName: managerPods[0].metadata.name,
    };
    const cloudNativePgEgressAllowed =
      podTcpProbe({
        ...podProbe,
        host: postgresServiceIp,
        port: 5432,
      }).status === 0;
    const kubernetesApiEgressDenied =
      podTcpProbe({
        ...podProbe,
        host: kubernetesServiceIp,
        port: 443,
      }).status !== 0;
    const publicInternetEgressDenied =
      podTcpProbe({
        ...podProbe,
        host: '1.1.1.1',
        port: 443,
      }).status !== 0;
    assert.equal(cloudNativePgEgressAllowed, true);
    assert.equal(kubernetesApiEgressDenied, true);
    assert.equal(publicInternetEgressDenied, true);

    const finalNodes = fixture.kubectlJson(['get', 'nodes']).items;
    const cniReadyNodes = finalNodes.filter(
      (node) =>
        podReady(node) &&
        Array.isArray(node.spec.podCIDRs) &&
        node.spec.podCIDRs.length === 1,
    );
    assert.equal(cniReadyNodes.length, 3);
    assert.equal(
      new Set(cniReadyNodes.map((node) => node.spec.podCIDRs[0])).size,
      3,
    );
    const serverNode = finalNodes.find(
      (node) => node.metadata.name === fixture.server,
    );
    assert.equal(
      serverNode?.metadata.annotations?.[
        'flannel.alpha.coreos.com/backend-type'
      ],
      'vxlan',
    );
    assert.equal(
      serverNode?.metadata.annotations?.[
        'flannel.alpha.coreos.com/kube-subnet-manager'
      ],
      'true',
    );

    const finalCluster = fixture.kubectlJson([
      '-n',
      NAMESPACE,
      'get',
      'cluster',
      POSTGRES_CLUSTER,
    ]);
    assert.equal(Number(finalCluster.status.readyInstances), 3);
    const baselineSuccesses = [
      ...initialRequests,
      overlapOld,
      decided,
      replayed,
    ];
    const report = {
      schemaVersion: 1,
      fixture: FIXTURE,
      observedAt: new Date().toISOString(),
      platform: {
        distribution: 'k3s',
        kubernetesVersion: nodes[0].status.nodeInfo.kubeletVersion,
        architecture,
        kubernetesImageId: imageIdDigest(k3sImageInfo),
        managementImageId: imageIdDigest(adminImageInfo),
        cniName: 'flannel',
        cniDistributionBinding: fixture.k3sImage,
        controlPlaneNodes: 1,
        workerNodes: 2,
        cniReadyNodes: cniReadyNodes.length,
      },
      database: {
        operator: 'cloudnative-pg',
        operatorVersion: OPERATOR_VERSION,
        postgresVersionNumber: durable.postgresVersionNumber,
        postgresImageId: imageIdDigest(postgresImageInfo),
        instances: Number(finalCluster.spec.instances),
        readyInstances: Number(finalCluster.status.readyInstances),
        managerRole: 'ql3_approval_manager',
        migrationCount: durable.migrationCount,
        controlCoreCapability: durable.controlCoreCapability,
        tlsVerified: true,
        primaryChangedDuringFailover:
          promoted.value !== primaryBeforeFailover.metadata.name,
      },
      deployment: {
        namespace: NAMESPACE,
        service: SERVICE,
        port: 8447,
        replicas: deployment.spec.replicas,
        readyReplicas: managerPods.length,
        podIdentitySha256: managerPods.map((pod) => sha256(pod.metadata.uid)),
        nodeIdentitySha256: managerPods.map((pod) => sha256(pod.spec.nodeName)),
        serviceAccount: DEPLOYMENT,
        automountServiceAccountToken: false,
        requiredPodAntiAffinity: true,
        podDisruptionBudgetMinAvailable: 1,
        maxUnavailable: 0,
        maxConnectionsPerPod: 2,
      },
      client: {
        binary: 'ql3-approval-client',
        operations: ['approval.inspect', 'approval.decide'],
        inputKind: 'Secret',
        inputImmutable: true,
        callerDrivenJob: true,
        backoffLimit: 0,
        serviceAccountTokenMounted: false,
        rbacGranted: false,
        transportProtocol: 'TLSv1.3',
        mutualTls: true,
        servernameVerified: true,
        exactPodRequests: baselineSuccesses.length,
        inspectStatuses: [
          initialRequests[0].output.result.status,
          initialRequests[1].output.result.status,
          overlapOld.output.result.status,
        ],
        decisionStatuses: [
          decided.output.result.status,
          replayed.output.result.status,
        ],
        responseRedacted: true,
      },
      identityRotation: {
        overlapOldAssertionAccepted: overlapOld.statusCode === 200,
        overlapNewAssertionAccepted: decided.statusCode === 200,
        revokedOldAssertionRejected: rejectedOldKey.statusCode === 401,
        activeNewAssertionAccepted: replayed.statusCode === 200,
        rollbackSurgeFailedClosed: Boolean(rollback.value),
        twoReadyReplicasPreserved:
          generation2.minimumReady >= 2 && generation3.minimumReady >= 2,
        durableGenerationReachedThree: durable.identityGeneration === 3,
      },
      certificateRotation: {
        previousSerialSha256,
        currentSerialSha256: pki.newSerialSha256(),
        previousBundleSha256,
        currentBundleSha256,
        oldClientAcceptedBefore: initialRequests[0].statusCode === 200,
        replacementClientAcceptedBefore: initialRequests[1].statusCode === 200,
        oldClientRejectedAfter: revokedCertificate.statusCode === 401,
        replacementClientAcceptedAfter: activeCertificate.statusCode === 200,
        fullPodReplacement: managerPods.every(
          (pod) => !preCertificateUids.has(pod.metadata.uid),
        ),
        allReplicasReadyThroughout: certificateRollout.minimumReady >= 2,
      },
      availability: {
        databaseFailureWithdrewReadiness: true,
        databaseFailurePreservedLiveness: true,
        stalePodsDidNotRecoverInPlace: true,
        freshPodsRecoveredAfterDatabase: managerPods.every(
          (pod) => !staleUids.has(pod.metadata.uid),
        ),
        bothReplicasServedAfterRecovery: recoveredRequests.every(
          (entry) => entry.statusCode === 200,
        ),
      },
      isolation: {
        labelledClientAllowed,
        unlabelledClientDenied,
        wrongPortDenied,
        kubernetesApiEgressDenied,
        publicInternetEgressDenied,
        cloudNativePgEgressAllowed,
        managerSecretReadDenied: canI('get', 'secrets') === 'no',
        managerMutationRbacDenied: canI('patch', 'deployments.apps') === 'no',
      },
      durability: {
        approvalVersion: durable.approvalVersion,
        approvalState: durable.approvalState,
        decisionIdSha256: sha256(durable.decisionId),
        allowedAuditCount: durable.allowedAuditCount,
        deniedAuditCount: durable.deniedAuditCount,
        duplicateDecisionCount: durable.decisionAuditCount - 1,
        identityGeneration: durable.identityGeneration,
        survivedCloudNativePgFailover: true,
      },
      gates: {
        realThreeNodeKubernetes: nodes.length === 3,
        realCniPolicy:
          labelledClientAllowed &&
          unlabelledClientDenied &&
          wrongPortDenied &&
          kubernetesApiEgressDenied &&
          publicInternetEgressDenied &&
          cloudNativePgEgressAllowed,
        threeInstanceCloudNativePg: databasePods.length === 3,
        twoManagerPodsOnDistinctNodes:
          new Set(managerPods.map((pod) => pod.spec.nodeName)).size === 2,
        tls13ProductClientAcrossBothPods:
          new Set(baselineSuccesses.map((entry) => entry.targetPod)).size >= 2,
        strongUserDecision:
          weakUserRejected.statusCode === 401 &&
          outsiderDenied.statusCode === 403 &&
          decided.statusCode === 200,
        identityProjectionRotation: durable.identityGeneration === 3,
        certificateRevocationRollout: revokedCertificate.statusCode === 401,
        databaseReadinessFence: true,
        durableFactsSurvivedFailover: true,
        leastPrivilege: rolesLeastPrivilege,
        passed: true,
      },
      limitations: [...LIMITATIONS],
    };
    const audit = validateApprovalManagementKubernetesLiveReport(report);
    assert.deepEqual(audit.findings, []);
    fs.writeFileSync(reportFile, JSON.stringify(report, null, 2) + '\n', {
      mode: 0o600,
      flag: 'wx',
    });
    process.stdout.write(
      JSON.stringify({
        schemaVersion: 1,
        fixture: FIXTURE,
        reportWritten: true,
        passed: true,
      }) + '\n',
    );
  } finally {
    await fixture.cleanup();
    if (adminImageBuilt) {
      run(fixture.docker, ['image', 'rm', '-f', adminImage], {
        capture: true,
        quiet: true,
        allowFailure: true,
      });
    }
    if (controlImageBuilt) {
      run(fixture.docker, ['image', 'rm', '-f', controlImage], {
        capture: true,
        quiet: true,
        allowFailure: true,
      });
    }
  }
}

if (require.main === module) {
  main().catch((error) => {
    process.stderr.write(
      'QL3 approval management Kubernetes live contract failed: ' +
        (error instanceof Error
          ? error.stack || error.message
          : String(error)) +
        '\n',
    );
    process.exitCode = 1;
  });
}

module.exports = {
  assertion,
  decisionCommand,
  inspectCommand,
  keyset,
  reviewedKey,
  assertionForSubject,
  weakAssertion,
};