feat(ql3): add cluster run management live gate

scripts/ql3-approval-management-kubernetes-live-contract.cjs

@@ -16,6 +16,7 @@ const { createMutualTlsPki } = require('./lib/ql3-live-pki.cjs');
 const {
   clientTcpProbe,
   createManagementClientExecutor,
+  managementHealthStatus,
   patchManagementGeneration,
   podReady,
   podTcpProbe,
@@ -90,10 +91,7 @@ const identity = createManagementIdentityCeremony({
 });
 
 function sha256(value) {
-  return (
-    'sha256:' +
-    crypto.createHash('sha256').update(value).digest('hex')
-  );
+  return 'sha256:' + crypto.createHash('sha256').update(value).digest('hex');
 }
 
 function randomSecret() {
@@ -102,9 +100,7 @@ function randomSecret() {
 
 function eventId(ordinal) {
   assert.ok(Number.isSafeInteger(ordinal) && ordinal >= 1 && ordinal < 1e12);
-  return (
-    '40000000-0000-4000-8000-' + String(ordinal).padStart(12, '0')
-  );
+  return '40000000-0000-4000-8000-' + String(ordinal).padStart(12, '0');
 }
 
 function reviewedKey(kid) {
@@ -119,75 +115,12 @@ function assertion(key, suffix) {
   return identity.assertion(key, suffix);
 }
 
-function assertionForSubject(
-  key,
-  subject,
-  suffix = crypto.randomUUID(),
-) {
-  assert.match(subject, /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/);
-  const now = Math.floor(Date.now() / 1_000);
-  const header = Buffer.from(
-    JSON.stringify({
-      alg: 'EdDSA',
-      kid: key.kid,
-      typ: 'ql3-approval-management+jwt',
-    }),
-  ).toString('base64url');
-  const payload = Buffer.from(
-    JSON.stringify({
-      acr: 'urn:ql3:mfa',
-      amr: ['pwd', 'otp'],
-      aud: AUDIENCE,
-      auth_time: now - 1,
-      exp: now + 290,
-      iat: now,
-      iss: ISSUER,
-      jti: 'ql3-approval-live-subject-' + suffix,
-      ql3_purpose: 'approval-management',
-      sub: subject,
-    }),
-  ).toString('base64url');
-  const signed = header + '.' + payload;
-  return (
-    signed +
-    '.' +
-    crypto
-      .sign(null, Buffer.from(signed, 'ascii'), key.privateKey)
-      .toString('base64url')
-  );
+function assertionForSubject(key, subject, suffix = crypto.randomUUID()) {
+  return identity.assertionForSubject(key, subject, 'subject-' + suffix);
 }
 
 function weakAssertion(key, suffix = crypto.randomUUID()) {
-  const now = Math.floor(Date.now() / 1_000);
-  const header = Buffer.from(
-    JSON.stringify({
-      alg: 'EdDSA',
-      kid: key.kid,
-      typ: 'ql3-approval-management+jwt',
-    }),
-  ).toString('base64url');
-  const payload = Buffer.from(
-    JSON.stringify({
-      acr: 'urn:ql3:password',
-      amr: ['pwd'],
-      aud: AUDIENCE,
-      auth_time: now - 1,
-      exp: now + 290,
-      iat: now,
-      iss: ISSUER,
-      jti: 'ql3-approval-live-weak-' + suffix,
-      ql3_purpose: 'approval-management',
-      sub: 'approval-operator',
-    }),
-  ).toString('base64url');
-  const signed = header + '.' + payload;
-  return (
-    signed +
-    '.' +
-    crypto
-      .sign(null, Buffer.from(signed, 'ascii'), key.privateKey)
-      .toString('base64url')
-  );
+  return identity.weakAssertion(key, suffix);
 }
 
 function commandBase(projectId, approvalRequestId, requestId, ordinal) {
@@ -204,12 +137,7 @@ function inspectCommand(projectId, approvalRequestId, requestId, ordinal) {
   return Object.freeze({
     schemaVersion: 1,
     operation: 'approval.inspect',
-    request: commandBase(
-      projectId,
-      approvalRequestId,
-      requestId,
-      ordinal,
-    ),
+    request: commandBase(projectId, approvalRequestId, requestId, ordinal),
   });
 }
 
@@ -224,12 +152,7 @@ function decisionCommand(
     schemaVersion: 1,
     operation: 'approval.decide',
     request: Object.freeze({
-      ...commandBase(
-        projectId,
-        approvalRequestId,
-        requestId,
-        ordinal,
-      ),
+      ...commandBase(projectId, approvalRequestId, requestId, ordinal),
       expectedVersion: 1,
       expectedAction: ACTION,
       decisionId,
@@ -327,12 +250,7 @@ function loadApprovalContract() {
   return require(file);
 }
 
-function seedApproval(
-  fixture,
-  primaryPod,
-  projectId,
-  approvalRequestId,
-) {
+function seedApproval(fixture, primaryPod, projectId, approvalRequestId) {
   const { approvalRequestDigest, createApprovalRequest } =
     loadApprovalContract();
   const requestedAtMs = Date.now() - 1_000;
@@ -422,31 +340,15 @@ function patchGeneration(fixture, generation, annotations = {}) {
 }
 
 function healthStatus(fixture, pod, route) {
-  const script = [
-    "const fs=require('node:fs');const https=require('node:https');",
-    "const request=https.request({host:'127.0.0.1',port:8447,path:process.argv[1],",
-    "servername:process.argv[2],ca:fs.readFileSync('/var/run/secrets/qinglong3/approval-management-tls/ca.crt'),",
-    "minVersion:'TLSv1.3',maxVersion:'TLSv1.3',rejectUnauthorized:true,agent:false},",
-    "(response)=>{response.resume();response.on('end',()=>process.stdout.write(String(response.statusCode)))});",
-    "request.on('error',(error)=>{process.stderr.write(error.message);process.exitCode=1});request.end();",
-  ].join('\n');
-  return Number(
-    fixture.kubectl(
-      [
-        '-n',
-        NAMESPACE,
-        'exec',
-        pod.metadata.name,
-        '--',
-        'node',
-        '-e',
-        script,
-        route,
-        SERVERNAME,
-      ],
-      { capture: true, quiet: true },
-    ).stdout,
-  );
+  return managementHealthStatus({
+    fixture,
+    namespace: NAMESPACE,
+    podName: pod.metadata.name,
+    port: 8447,
+    route,
+    servername: SERVERNAME,
+    caFile: '/var/run/secrets/qinglong3/approval-management-tls/ca.crt',
+  });
 }
 
 function privateReportPath(argv) {
@@ -507,10 +409,7 @@ async function main(argv = process.argv.slice(2)) {
       );
       const preloadTag = imageTag(reviewedImage);
       run(fixture.docker, ['tag', reviewedImage, preloadTag]);
-      fixture.loadImage(
-        preloadTag,
-        path.basename(preloadTag) + '.tar',
-      );
+      fixture.loadImage(preloadTag, path.basename(preloadTag) + '.tar');
     }
 
     const sourceRevision = run('git', ['rev-parse', 'HEAD'], {
@@ -615,28 +514,24 @@ async function main(argv = process.argv.slice(2)) {
       '--timeout=20m',
     ]);
     const databasePods = (
-      await waitFor(
-        'three ready CloudNativePG instances',
-        600_000,
-        () => {
-          const pods = fixture
-            .kubectlJson([
-              '-n',
-              NAMESPACE,
-              'get',
-              'pods',
-              '-l',
-              'cnpg.io/cluster=' + POSTGRES_CLUSTER,
-            ])
-            .items.filter(podReady);
-          return pods.length === 3
-            ? { ready: true, value: pods }
-            : {
-                ready: false,
-                fact: pods.length + '/3 ready database Pods',
-              };
-        },
-      )
+      await waitFor('three ready CloudNativePG instances', 600_000, () => {
+        const pods = fixture
+          .kubectlJson([
+            '-n',
+            NAMESPACE,
+            'get',
+            'pods',
+            '-l',
+            'cnpg.io/cluster=' + POSTGRES_CLUSTER,
+          ])
+          .items.filter(podReady);
+        return pods.length === 3
+          ? { ready: true, value: pods }
+          : {
+              ready: false,
+              fact: pods.length + '/3 ready database Pods',
+            };
+      })
     ).value;
 
     const migrationManifest = localManifest(
@@ -685,12 +580,7 @@ async function main(argv = process.argv.slice(2)) {
     const approvalRequestId = 'approval-request-' + suffix;
     const decisionId = 'approval-decision-' + suffix;
     const primary = currentPrimaryPod(fixture);
-    seedApproval(
-      fixture,
-      primary,
-      projectId,
-      approvalRequestId,
-    );
+    seedApproval(fixture, primary, projectId, approvalRequestId);
 
     const pki = createMutualTlsPki({
       directory: fixture.temporary,
@@ -764,13 +654,8 @@ async function main(argv = process.argv.slice(2)) {
       1,
     );
     assert.equal(
-      fixture.kubectlJson([
-        '-n',
-        NAMESPACE,
-        'get',
-        'pdb',
-        DEPLOYMENT,
-      ]).spec.minAvailable,
+      fixture.kubectlJson(['-n', NAMESPACE, 'get', 'pdb', DEPLOYMENT]).spec
+        .minAvailable,
       1,
     );
     for (const pod of managerPods) {
@@ -876,9 +761,7 @@ async function main(argv = process.argv.slice(2)) {
       { statusCode: 403, responseCode: 'forbidden' },
     );
 
-    const generation1Uids = new Set(
-      managerPods.map((pod) => pod.metadata.uid),
-    );
+    const generation1Uids = new Set(managerPods.map((pod) => pod.metadata.uid));
     applyIdentity(keysets[1]);
     patchGeneration(fixture, 2);
     const generation2 = await waitForTwoPreserved({
@@ -925,9 +808,7 @@ async function main(argv = process.argv.slice(2)) {
     assert.equal(decided.output.result.approval.state, 'approved');
     assert.equal(decided.output.result.approval.version, 2);
 
-    const generation2Uids = new Set(
-      managerPods.map((pod) => pod.metadata.uid),
-    );
+    const generation2Uids = new Set(managerPods.map((pod) => pod.metadata.uid));
     applyIdentity(keysets[2]);
     patchGeneration(fixture, 3);
     const generation3 = await waitForTwoPreserved({
@@ -987,9 +868,7 @@ async function main(argv = process.argv.slice(2)) {
             '-l',
             'app.kubernetes.io/name=' + DEPLOYMENT,
           ])
-          .items.filter(
-            (pod) => pod.metadata.deletionTimestamp === undefined,
-          );
+          .items.filter((pod) => pod.metadata.deletionTimestamp === undefined);
         const ready = pods.filter(podReady);
         const candidate = pods.find(
           (pod) =>
@@ -1127,27 +1006,23 @@ async function main(argv = process.argv.slice(2)) {
             };
       },
     );
-    await waitFor(
-      'CloudNativePG recovery to three instances',
-      900_000,
-      () => {
-        const status = fixture.kubectlJson([
-          '-n',
-          NAMESPACE,
-          'get',
-          'cluster',
-          POSTGRES_CLUSTER,
-        ]).status;
-        return Number(status.readyInstances) === 3
-          ? { ready: true, value: status }
-          : {
-              ready: false,
-              fact:
-                String(status.readyInstances ?? 0) +
-                '/3 ready database instances',
-            };
-      },
-    );
+    await waitFor('CloudNativePG recovery to three instances', 900_000, () => {
+      const status = fixture.kubectlJson([
+        '-n',
+        NAMESPACE,
+        'get',
+        'cluster',
+        POSTGRES_CLUSTER,
+      ]).status;
+      return Number(status.readyInstances) === 3
+        ? { ready: true, value: status }
+        : {
+            ready: false,
+            fact:
+              String(status.readyInstances ?? 0) +
+              '/3 ready database instances',
+          };
+    });
 
     const databaseService = fixture.kubectlJson([
       '-n',
@@ -1175,8 +1050,7 @@ async function main(argv = process.argv.slice(2)) {
       managerPods.map((pod, index) =>
         executeClient(
           {
-            name:
-              'ql3-approval-database-unavailable-' + String(index + 1),
+            name: 'ql3-approval-database-unavailable-' + String(index + 1),
             target: pod,
             command: decisionCommand(
               projectId,
@@ -1209,9 +1083,7 @@ async function main(argv = process.argv.slice(2)) {
         ? { ready: true, value: current }
         : {
             ready: false,
-            fact:
-              String(current.status.readyReplicas ?? 0) +
-              ' ready replicas',
+            fact: String(current.status.readyReplicas ?? 0) + ' ready replicas',
           };
     });
     assert.deepEqual(
@@ -1238,35 +1110,29 @@ async function main(argv = process.argv.slice(2)) {
         },
       ]),
     ]);
-    await waitFor(
-      'restored CloudNativePG service endpoint',
-      120_000,
-      () => {
-        const endpoints = fixture.kubectlJson([
-          '-n',
-          NAMESPACE,
-          'get',
-          'endpoints',
-          POSTGRES_CLUSTER + '-rw',
-        ]);
-        const count = endpoints.subsets?.flatMap(
-          (subset) => subset.addresses ?? [],
-        ).length;
-        return count >= 1
-          ? { ready: true, value: count }
-          : {
-              ready: false,
-              fact: String(count ?? 0) + ' service endpoints',
-            };
-      },
-    );
+    await waitFor('restored CloudNativePG service endpoint', 120_000, () => {
+      const endpoints = fixture.kubectlJson([
+        '-n',
+        NAMESPACE,
+        'get',
+        'endpoints',
+        POSTGRES_CLUSTER + '-rw',
+      ]);
+      const count = endpoints.subsets?.flatMap(
+        (subset) => subset.addresses ?? [],
+      ).length;
+      return count >= 1
+        ? { ready: true, value: count }
+        : {
+            ready: false,
+            fact: String(count ?? 0) + ' service endpoints',
+          };
+    });
     assert.deepEqual(
       managerPods.map((pod) => healthStatus(fixture, pod, '/readyz')),
       [503, 503],
     );
-    const staleUids = new Set(
-      managerPods.map((pod) => pod.metadata.uid),
-    );
+    const staleUids = new Set(managerPods.map((pod) => pod.metadata.uid));
     patchGeneration(fixture, '3-database-recovered');
     managerPods = await readyManagementPods({
       ...managerOptions(fixture),
@@ -1277,8 +1143,7 @@ async function main(argv = process.argv.slice(2)) {
       managerPods.map((pod, index) =>
         executeClient(
           {
-            name:
-              'ql3-approval-database-recovered-' + String(index + 1),
+            name: 'ql3-approval-database-recovered-' + String(index + 1),
             target: pod,
             command: decisionCommand(
               projectId,
@@ -1402,10 +1267,7 @@ async function main(argv = process.argv.slice(2)) {
           resource,
           '-n',
           NAMESPACE,
-          '--as=system:serviceaccount:' +
-            NAMESPACE +
-            ':' +
-            DEPLOYMENT,
+          '--as=system:serviceaccount:' + NAMESPACE + ':' + DEPLOYMENT,
         ],
         { capture: true, quiet: true, allowFailure: true },
       );
@@ -1576,12 +1438,8 @@ async function main(argv = process.argv.slice(2)) {
         port: 8447,
         replicas: deployment.spec.replicas,
         readyReplicas: managerPods.length,
-        podIdentitySha256: managerPods.map((pod) =>
-          sha256(pod.metadata.uid),
-        ),
-        nodeIdentitySha256: managerPods.map((pod) =>
-          sha256(pod.spec.nodeName),
-        ),
+        podIdentitySha256: managerPods.map((pod) => sha256(pod.metadata.uid)),
+        nodeIdentitySha256: managerPods.map((pod) => sha256(pod.spec.nodeName)),
         serviceAccount: DEPLOYMENT,
         automountServiceAccountToken: false,
         requiredPodAntiAffinity: true,
@@ -1620,8 +1478,7 @@ async function main(argv = process.argv.slice(2)) {
         activeNewAssertionAccepted: replayed.statusCode === 200,
         rollbackSurgeFailedClosed: Boolean(rollback.value),
         twoReadyReplicasPreserved:
-          generation2.minimumReady >= 2 &&
-          generation3.minimumReady >= 2,
+          generation2.minimumReady >= 2 && generation3.minimumReady >= 2,
         durableGenerationReachedThree: durable.identityGeneration === 3,
       },
       certificateRotation: {
@@ -1630,16 +1487,13 @@ async function main(argv = process.argv.slice(2)) {
         previousBundleSha256,
         currentBundleSha256,
         oldClientAcceptedBefore: initialRequests[0].statusCode === 200,
-        replacementClientAcceptedBefore:
-          initialRequests[1].statusCode === 200,
+        replacementClientAcceptedBefore: initialRequests[1].statusCode === 200,
         oldClientRejectedAfter: revokedCertificate.statusCode === 401,
-        replacementClientAcceptedAfter:
-          activeCertificate.statusCode === 200,
+        replacementClientAcceptedAfter: activeCertificate.statusCode === 200,
         fullPodReplacement: managerPods.every(
           (pod) => !preCertificateUids.has(pod.metadata.uid),
         ),
-        allReplicasReadyThroughout:
-          certificateRollout.minimumReady >= 2,
+        allReplicasReadyThroughout: certificateRollout.minimumReady >= 2,
       },
       availability: {
         databaseFailureWithdrewReadiness: true,
@@ -1660,8 +1514,7 @@ async function main(argv = process.argv.slice(2)) {
         publicInternetEgressDenied,
         cloudNativePgEgressAllowed,
         managerSecretReadDenied: canI('get', 'secrets') === 'no',
-        managerMutationRbacDenied:
-          canI('patch', 'deployments.apps') === 'no',
+        managerMutationRbacDenied: canI('patch', 'deployments.apps') === 'no',
       },
       durability: {
         approvalVersion: durable.approvalVersion,
@@ -1686,15 +1539,13 @@ async function main(argv = process.argv.slice(2)) {
         twoManagerPodsOnDistinctNodes:
           new Set(managerPods.map((pod) => pod.spec.nodeName)).size === 2,
         tls13ProductClientAcrossBothPods:
-          new Set(baselineSuccesses.map((entry) => entry.targetPod)).size >=
-          2,
+          new Set(baselineSuccesses.map((entry) => entry.targetPod)).size >= 2,
         strongUserDecision:
           weakUserRejected.statusCode === 401 &&
           outsiderDenied.statusCode === 403 &&
           decided.statusCode === 200,
         identityProjectionRotation: durable.identityGeneration === 3,
-        certificateRevocationRollout:
-          revokedCertificate.statusCode === 401,
+        certificateRevocationRollout: revokedCertificate.statusCode === 401,
         databaseReadinessFence: true,
         durableFactsSurvivedFailover: true,
         leastPrivilege: rolesLeastPrivilege,
@@ -1739,7 +1590,9 @@ if (require.main === module) {
   main().catch((error) => {
     process.stderr.write(
       'QL3 approval management Kubernetes live contract failed: ' +
-        (error instanceof Error ? error.stack || error.message : String(error)) +
+        (error instanceof Error
+          ? error.stack || error.message
+          : String(error)) +
         '\n',
     );
     process.exitCode = 1;