test(ql3): report control rollout failure facts

scripts/ql3-security-administration-kubernetes-live-contract.cjs

@@ -787,6 +787,7 @@ function clusterControlResources(controlImage) {
               name: 'cluster-control',
               image: controlImage,
               imagePullPolicy: 'Never',
+              terminationMessagePolicy: 'FallbackToLogsOnError',
               securityContext: {
                 allowPrivilegeEscalation: false,
                 readOnlyRootFilesystem: true,
@@ -906,6 +907,130 @@ function applyControlRuntimeSecret(fixture, runtimeDatabaseUrl, keyring) {
   });
 }
 
+function controlTerminationFact(message) {
+  if (typeof message !== 'string' || message.length < 1 || message.length > 4096) {
+    return 'rejected';
+  }
+  const lines = message.trim().split('\n');
+  try {
+    const fact = JSON.parse(lines.at(-1));
+    const keys = Object.keys(fact).sort();
+    const expected = [
+      'component',
+      'event',
+      'level',
+      'name',
+      'schemaVersion',
+      ...(fact.code === undefined ? [] : ['code']),
+    ].sort();
+    if (
+      JSON.stringify(keys) !== JSON.stringify(expected) ||
+      fact.schemaVersion !== 1 ||
+      fact.component !== 'qinglong3-cluster-control' ||
+      fact.level !== 'error' ||
+      fact.event !== 'process_failed' ||
+      typeof fact.name !== 'string' ||
+      !/^[A-Za-z][A-Za-z0-9]{0,127}$/.test(fact.name) ||
+      (fact.code !== undefined &&
+        (typeof fact.code !== 'string' ||
+          !/^[A-Z][A-Z0-9_]{0,127}$/.test(fact.code)))
+    ) {
+      return 'rejected';
+    }
+    return Object.freeze({
+      name: fact.name,
+      ...(fact.code === undefined ? {} : { code: fact.code }),
+    });
+  } catch {
+    return 'rejected';
+  }
+}
+
+function controlRolloutFailureEvidence(fixture) {
+  const deployment = fixture.kubectlJson([
+    '-n',
+    NAMESPACE,
+    'get',
+    'deployment',
+    CONTROL_NAME,
+  ]);
+  const pods = fixture.kubectlJson([
+    '-n',
+    NAMESPACE,
+    'get',
+    'pods',
+    '-l',
+    `app.kubernetes.io/name=${CONTROL_NAME}`,
+  ]).items;
+  return Object.freeze({
+    deployment: Object.freeze({
+      generation: deployment.metadata.generation ?? null,
+      observedGeneration: deployment.status?.observedGeneration ?? null,
+      replicas: deployment.status?.replicas ?? 0,
+      updatedReplicas: deployment.status?.updatedReplicas ?? 0,
+      availableReplicas: deployment.status?.availableReplicas ?? 0,
+      unavailableReplicas: deployment.status?.unavailableReplicas ?? 0,
+      conditions: Object.freeze(
+        (deployment.status?.conditions ?? []).map((condition) =>
+          Object.freeze({
+            type: condition.type,
+            status: condition.status,
+            reason: condition.reason ?? null,
+          }),
+        ),
+      ),
+    }),
+    pods: Object.freeze(
+      pods.map((pod) =>
+        Object.freeze({
+          name: pod.metadata.name,
+          node: pod.spec.nodeName ?? null,
+          phase: pod.status?.phase ?? null,
+          conditions: Object.freeze(
+            (pod.status?.conditions ?? []).map((condition) =>
+              Object.freeze({
+                type: condition.type,
+                status: condition.status,
+                reason: condition.reason ?? null,
+              }),
+            ),
+          ),
+          containers: Object.freeze(
+            (pod.status?.containerStatuses ?? []).map((container) => {
+              const terminated =
+                container.state?.terminated ??
+                container.lastState?.terminated;
+              return Object.freeze({
+                name: container.name,
+                ready: container.ready,
+                restartCount: container.restartCount,
+                state: container.state?.waiting
+                  ? Object.freeze({
+                      kind: 'waiting',
+                      reason: container.state.waiting.reason ?? null,
+                    })
+                  : container.state?.running
+                    ? Object.freeze({ kind: 'running' })
+                    : terminated
+                      ? Object.freeze({
+                          kind: 'terminated',
+                          reason: terminated.reason ?? null,
+                          exitCode: terminated.exitCode,
+                        })
+                      : Object.freeze({ kind: 'unknown' }),
+                failure:
+                  terminated?.message === undefined
+                    ? null
+                    : controlTerminationFact(terminated.message),
+              });
+            }),
+          ),
+        }),
+      ),
+    ),
+  });
+}
+
 async function waitForControlRollout(fixture, restart) {
   if (restart) {
     fixture.kubectl([
@@ -916,14 +1041,24 @@ async function waitForControlRollout(fixture, restart) {
       `deployment/${CONTROL_NAME}`,
     ]);
   }
-  fixture.kubectl([
+  const rollout = fixture.kubectl(
-    '-n',
+    [
-    NAMESPACE,
+      '-n',
-    'rollout',
+      NAMESPACE,
-    'status',
+      'rollout',
-    `deployment/${CONTROL_NAME}`,
+      'status',
-    '--timeout=5m',
+      `deployment/${CONTROL_NAME}`,
-  ]);
+      '--timeout=5m',
+    ],
+    { capture: true, quiet: true, allowFailure: true },
+  );
+  if (rollout.status !== 0) {
+    throw new Error(
+      `Cluster Control rollout unavailable: ${JSON.stringify(
+        controlRolloutFailureEvidence(fixture),
+      )}`,
+    );
+  }
   return (
     await waitFor('two ready Cluster Control replicas', 120_000, () => {
       const deployment = fixture.kubectlJson([
@@ -2218,6 +2353,8 @@ if (require.main === module) {
 module.exports = {
   auditListCommand,
   clusterControlResources,
+  controlRolloutFailureEvidence,
+  controlTerminationFact,
   credentialAuthenticationProbeSource,
   credentialIssueCommand,
   credentialRevokeCommand,

test/back/ql3SecurityAdministrationKubernetesLiveContract.test.cjs

@@ -7,6 +7,8 @@ const { test } = require('node:test');
 const {
   auditListCommand,
   clusterControlResources,
+  controlRolloutFailureEvidence,
+  controlTerminationFact,
   credentialAuthenticationProbeSource,
   credentialIssueCommand,
   credentialRevokeCommand,
@@ -199,6 +201,10 @@ test('runs the credential ceremony against two real anti-affine control replicas
   assert.equal(deployment.kind, 'Deployment');
   assert.equal(deployment.spec.replicas, 2);
   assert.equal(deployment.spec.strategy.rollingUpdate.maxUnavailable, 0);
+  assert.equal(
+    deployment.spec.template.spec.containers[0].terminationMessagePolicy,
+    'FallbackToLogsOnError',
+  );
   assert.equal(
     deployment.spec.template.spec.affinity.podAntiAffinity
       .requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey,
@@ -218,6 +224,79 @@ test('runs the credential ceremony against two real anti-affine control replicas
   );
 });
 
+test('keeps failed control rollout evidence bounded and content-free', () => {
+  const failure = JSON.stringify({
+    schemaVersion: 1,
+    component: 'qinglong3-cluster-control',
+    level: 'error',
+    event: 'process_failed',
+    name: 'ClusterControlDatabaseUnavailableError',
+    code: 'ECONNREFUSED',
+  });
+  assert.deepEqual(controlTerminationFact(`ignored\n${failure}\n`), {
+    name: 'ClusterControlDatabaseUnavailableError',
+    code: 'ECONNREFUSED',
+  });
+  assert.equal(
+    controlTerminationFact(
+      JSON.stringify({
+        schemaVersion: 1,
+        component: 'qinglong3-cluster-control',
+        level: 'error',
+        event: 'process_failed',
+        name: 'Error',
+        secret: 'must-not-escape',
+      }),
+    ),
+    'rejected',
+  );
+
+  const evidence = controlRolloutFailureEvidence({
+    kubectlJson(arguments_) {
+      if (arguments_.includes('deployment')) {
+        return {
+          metadata: { generation: 3 },
+          status: {
+            observedGeneration: 3,
+            replicas: 2,
+            updatedReplicas: 2,
+            unavailableReplicas: 2,
+            conditions: [
+              { type: 'Available', status: 'False', reason: 'MinimumReplicasUnavailable' },
+            ],
+          },
+        };
+      }
+      return {
+        items: [{
+          metadata: { name: 'control-1' },
+          spec: { nodeName: 'worker-1' },
+          status: {
+            phase: 'Running',
+            conditions: [
+              { type: 'Ready', status: 'False', reason: 'ContainersNotReady' },
+            ],
+            containerStatuses: [{
+              name: 'cluster-control',
+              ready: false,
+              restartCount: 2,
+              state: { waiting: { reason: 'CrashLoopBackOff' } },
+              lastState: { terminated: { exitCode: 1, reason: 'Error', message: failure } },
+            }],
+          },
+        }],
+      };
+    },
+  });
+  assert.equal(evidence.deployment.availableReplicas, 0);
+  assert.equal(evidence.pods[0].containers[0].state.reason, 'CrashLoopBackOff');
+  assert.deepEqual(evidence.pods[0].containers[0].failure, {
+    name: 'ClusterControlDatabaseUnavailableError',
+    code: 'ECONNREFUSED',
+  });
+  assert.doesNotMatch(JSON.stringify(evidence), /must-not-escape/);
+});
+
 test('keeps the real authentication probe content-free', () => {
   const source = credentialAuthenticationProbeSource();
   assert.match(source, /ql3-security-live-control/);