From 9c58b15d2bb2d8446ec47c35a5f8f46cab984988 Mon Sep 17 00:00:00 2001 From: whyour Date: Wed, 26 Aug 2026 03:48:12 +0800 Subject: [PATCH] test(ql3): report control rollout failure facts --- ...dministration-kubernetes-live-contract.cjs | 153 +++++++++++++++++- ...inistrationKubernetesLiveContract.test.cjs | 79 +++++++++ 2 files changed, 224 insertions(+), 8 deletions(-) diff --git a/scripts/ql3-security-administration-kubernetes-live-contract.cjs b/scripts/ql3-security-administration-kubernetes-live-contract.cjs index 6396a9f3..d7018474 100644 --- a/scripts/ql3-security-administration-kubernetes-live-contract.cjs +++ b/scripts/ql3-security-administration-kubernetes-live-contract.cjs @@ -787,6 +787,7 @@ function clusterControlResources(controlImage) { name: 'cluster-control', image: controlImage, imagePullPolicy: 'Never', + terminationMessagePolicy: 'FallbackToLogsOnError', securityContext: { allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, @@ -906,6 +907,130 @@ function applyControlRuntimeSecret(fixture, runtimeDatabaseUrl, keyring) { }); } +function controlTerminationFact(message) { + if (typeof message !== 'string' || message.length < 1 || message.length > 4096) { + return 'rejected'; + } + const lines = message.trim().split('\n'); + try { + const fact = JSON.parse(lines.at(-1)); + const keys = Object.keys(fact).sort(); + const expected = [ + 'component', + 'event', + 'level', + 'name', + 'schemaVersion', + ...(fact.code === undefined ? [] : ['code']), + ].sort(); + if ( + JSON.stringify(keys) !== JSON.stringify(expected) || + fact.schemaVersion !== 1 || + fact.component !== 'qinglong3-cluster-control' || + fact.level !== 'error' || + fact.event !== 'process_failed' || + typeof fact.name !== 'string' || + !/^[A-Za-z][A-Za-z0-9]{0,127}$/.test(fact.name) || + (fact.code !== undefined && + (typeof fact.code !== 'string' || + !/^[A-Z][A-Z0-9_]{0,127}$/.test(fact.code))) + ) { + return 'rejected'; + } + return Object.freeze({ + name: fact.name, + ...(fact.code === undefined ? {} : { code: fact.code }), + }); + } catch { + return 'rejected'; + } +} + +function controlRolloutFailureEvidence(fixture) { + const deployment = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'deployment', + CONTROL_NAME, + ]); + const pods = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'pods', + '-l', + `app.kubernetes.io/name=${CONTROL_NAME}`, + ]).items; + return Object.freeze({ + deployment: Object.freeze({ + generation: deployment.metadata.generation ?? null, + observedGeneration: deployment.status?.observedGeneration ?? null, + replicas: deployment.status?.replicas ?? 0, + updatedReplicas: deployment.status?.updatedReplicas ?? 0, + availableReplicas: deployment.status?.availableReplicas ?? 0, + unavailableReplicas: deployment.status?.unavailableReplicas ?? 0, + conditions: Object.freeze( + (deployment.status?.conditions ?? []).map((condition) => + Object.freeze({ + type: condition.type, + status: condition.status, + reason: condition.reason ?? null, + }), + ), + ), + }), + pods: Object.freeze( + pods.map((pod) => + Object.freeze({ + name: pod.metadata.name, + node: pod.spec.nodeName ?? null, + phase: pod.status?.phase ?? null, + conditions: Object.freeze( + (pod.status?.conditions ?? []).map((condition) => + Object.freeze({ + type: condition.type, + status: condition.status, + reason: condition.reason ?? null, + }), + ), + ), + containers: Object.freeze( + (pod.status?.containerStatuses ?? []).map((container) => { + const terminated = + container.state?.terminated ?? + container.lastState?.terminated; + return Object.freeze({ + name: container.name, + ready: container.ready, + restartCount: container.restartCount, + state: container.state?.waiting + ? Object.freeze({ + kind: 'waiting', + reason: container.state.waiting.reason ?? null, + }) + : container.state?.running + ? Object.freeze({ kind: 'running' }) + : terminated + ? Object.freeze({ + kind: 'terminated', + reason: terminated.reason ?? null, + exitCode: terminated.exitCode, + }) + : Object.freeze({ kind: 'unknown' }), + failure: + terminated?.message === undefined + ? null + : controlTerminationFact(terminated.message), + }); + }), + ), + }), + ), + ), + }); +} + async function waitForControlRollout(fixture, restart) { if (restart) { fixture.kubectl([ @@ -916,14 +1041,24 @@ async function waitForControlRollout(fixture, restart) { `deployment/${CONTROL_NAME}`, ]); } - fixture.kubectl([ - '-n', - NAMESPACE, - 'rollout', - 'status', - `deployment/${CONTROL_NAME}`, - '--timeout=5m', - ]); + const rollout = fixture.kubectl( + [ + '-n', + NAMESPACE, + 'rollout', + 'status', + `deployment/${CONTROL_NAME}`, + '--timeout=5m', + ], + { capture: true, quiet: true, allowFailure: true }, + ); + if (rollout.status !== 0) { + throw new Error( + `Cluster Control rollout unavailable: ${JSON.stringify( + controlRolloutFailureEvidence(fixture), + )}`, + ); + } return ( await waitFor('two ready Cluster Control replicas', 120_000, () => { const deployment = fixture.kubectlJson([ @@ -2218,6 +2353,8 @@ if (require.main === module) { module.exports = { auditListCommand, clusterControlResources, + controlRolloutFailureEvidence, + controlTerminationFact, credentialAuthenticationProbeSource, credentialIssueCommand, credentialRevokeCommand, diff --git a/test/back/ql3SecurityAdministrationKubernetesLiveContract.test.cjs b/test/back/ql3SecurityAdministrationKubernetesLiveContract.test.cjs index 6d6040af..f998c32b 100644 --- a/test/back/ql3SecurityAdministrationKubernetesLiveContract.test.cjs +++ b/test/back/ql3SecurityAdministrationKubernetesLiveContract.test.cjs @@ -7,6 +7,8 @@ const { test } = require('node:test'); const { auditListCommand, clusterControlResources, + controlRolloutFailureEvidence, + controlTerminationFact, credentialAuthenticationProbeSource, credentialIssueCommand, credentialRevokeCommand, @@ -199,6 +201,10 @@ test('runs the credential ceremony against two real anti-affine control replicas assert.equal(deployment.kind, 'Deployment'); assert.equal(deployment.spec.replicas, 2); assert.equal(deployment.spec.strategy.rollingUpdate.maxUnavailable, 0); + assert.equal( + deployment.spec.template.spec.containers[0].terminationMessagePolicy, + 'FallbackToLogsOnError', + ); assert.equal( deployment.spec.template.spec.affinity.podAntiAffinity .requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey, @@ -218,6 +224,79 @@ test('runs the credential ceremony against two real anti-affine control replicas ); }); +test('keeps failed control rollout evidence bounded and content-free', () => { + const failure = JSON.stringify({ + schemaVersion: 1, + component: 'qinglong3-cluster-control', + level: 'error', + event: 'process_failed', + name: 'ClusterControlDatabaseUnavailableError', + code: 'ECONNREFUSED', + }); + assert.deepEqual(controlTerminationFact(`ignored\n${failure}\n`), { + name: 'ClusterControlDatabaseUnavailableError', + code: 'ECONNREFUSED', + }); + assert.equal( + controlTerminationFact( + JSON.stringify({ + schemaVersion: 1, + component: 'qinglong3-cluster-control', + level: 'error', + event: 'process_failed', + name: 'Error', + secret: 'must-not-escape', + }), + ), + 'rejected', + ); + + const evidence = controlRolloutFailureEvidence({ + kubectlJson(arguments_) { + if (arguments_.includes('deployment')) { + return { + metadata: { generation: 3 }, + status: { + observedGeneration: 3, + replicas: 2, + updatedReplicas: 2, + unavailableReplicas: 2, + conditions: [ + { type: 'Available', status: 'False', reason: 'MinimumReplicasUnavailable' }, + ], + }, + }; + } + return { + items: [{ + metadata: { name: 'control-1' }, + spec: { nodeName: 'worker-1' }, + status: { + phase: 'Running', + conditions: [ + { type: 'Ready', status: 'False', reason: 'ContainersNotReady' }, + ], + containerStatuses: [{ + name: 'cluster-control', + ready: false, + restartCount: 2, + state: { waiting: { reason: 'CrashLoopBackOff' } }, + lastState: { terminated: { exitCode: 1, reason: 'Error', message: failure } }, + }], + }, + }], + }; + }, + }); + assert.equal(evidence.deployment.availableReplicas, 0); + assert.equal(evidence.pods[0].containers[0].state.reason, 'CrashLoopBackOff'); + assert.deepEqual(evidence.pods[0].containers[0].failure, { + name: 'ClusterControlDatabaseUnavailableError', + code: 'ECONNREFUSED', + }); + assert.doesNotMatch(JSON.stringify(evidence), /must-not-escape/); +}); + test('keeps the real authentication probe content-free', () => { const source = credentialAuthenticationProbeSource(); assert.match(source, /ql3-security-live-control/);