open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-20 16:07:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

test(ql3): report control rollout failure facts

Browse Source
This commit is contained in:
whyour
2026-08-26 03:48:12 +08:00
parent ac0992081e
commit 9c58b15d2b
2 changed files with 224 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ql3-security-administration-kubernetes-live-contract.cjs
+145 -8
View File
@@ -787,6 +787,7 @@ function clusterControlResources(controlImage) {
name: 'cluster-control',
image: controlImage,
imagePullPolicy: 'Never',
terminationMessagePolicy: 'FallbackToLogsOnError',
securityContext: {
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
@@ -906,6 +907,130 @@ function applyControlRuntimeSecret(fixture, runtimeDatabaseUrl, keyring) {
});
}
function controlTerminationFact(message) {
if (typeof message !== 'string' || message.length < 1 || message.length > 4096) {
return 'rejected';
}
const lines = message.trim().split('\n');
try {
const fact = JSON.parse(lines.at(-1));
const keys = Object.keys(fact).sort();
const expected = [
'component',
'event',
'level',
'name',
'schemaVersion',
...(fact.code === undefined ? [] : ['code']),
].sort();
if (
JSON.stringify(keys) !== JSON.stringify(expected) ||
fact.schemaVersion !== 1 ||
fact.component !== 'qinglong3-cluster-control' ||
fact.level !== 'error' ||
fact.event !== 'process_failed' ||
typeof fact.name !== 'string' ||
!/^[A-Za-z][A-Za-z0-9]{0,127}$/.test(fact.name) ||
(fact.code !== undefined &&
(typeof fact.code !== 'string' ||
!/^[A-Z][A-Z0-9_]{0,127}$/.test(fact.code)))
) {
return 'rejected';
}
return Object.freeze({
name: fact.name,
...(fact.code === undefined ? {} : { code: fact.code }),
});
} catch {
return 'rejected';
}
}
function controlRolloutFailureEvidence(fixture) {
const deployment = fixture.kubectlJson([
'-n',
NAMESPACE,
'get',
'deployment',
CONTROL_NAME,
]);
const pods = fixture.kubectlJson([
'-n',
NAMESPACE,
'get',
'pods',
'-l',
`app.kubernetes.io/name=${CONTROL_NAME}`,
]).items;
return Object.freeze({
deployment: Object.freeze({
generation: deployment.metadata.generation ?? null,
observedGeneration: deployment.status?.observedGeneration ?? null,
replicas: deployment.status?.replicas ?? 0,
updatedReplicas: deployment.status?.updatedReplicas ?? 0,
availableReplicas: deployment.status?.availableReplicas ?? 0,
unavailableReplicas: deployment.status?.unavailableReplicas ?? 0,
conditions: Object.freeze(
(deployment.status?.conditions ?? []).map((condition) =>
Object.freeze({
type: condition.type,
status: condition.status,
reason: condition.reason ?? null,
}),
),
),
}),
pods: Object.freeze(
pods.map((pod) =>
Object.freeze({
name: pod.metadata.name,
node: pod.spec.nodeName ?? null,
phase: pod.status?.phase ?? null,
conditions: Object.freeze(
(pod.status?.conditions ?? []).map((condition) =>
Object.freeze({
type: condition.type,
status: condition.status,
reason: condition.reason ?? null,
}),
),
),
containers: Object.freeze(
(pod.status?.containerStatuses ?? []).map((container) => {
const terminated =
container.state?.terminated ??
container.lastState?.terminated;
return Object.freeze({
name: container.name,
ready: container.ready,
restartCount: container.restartCount,
state: container.state?.waiting
? Object.freeze({
kind: 'waiting',
reason: container.state.waiting.reason ?? null,
})
: container.state?.running
? Object.freeze({ kind: 'running' })
: terminated
? Object.freeze({
kind: 'terminated',
reason: terminated.reason ?? null,
exitCode: terminated.exitCode,
})
: Object.freeze({ kind: 'unknown' }),
failure:
terminated?.message === undefined
? null
: controlTerminationFact(terminated.message),
});
}),
),
}),
),
),
});
}
async function waitForControlRollout(fixture, restart) {
if (restart) {
fixture.kubectl([
@@ -916,14 +1041,24 @@ async function waitForControlRollout(fixture, restart) {
`deployment/${CONTROL_NAME}`,
]);
}
fixture.kubectl([
'-n',
NAMESPACE,
'rollout',
'status',
`deployment/${CONTROL_NAME}`,
'--timeout=5m',
]);
const rollout = fixture.kubectl(
[
'-n',
NAMESPACE,
'rollout',
'status',
`deployment/${CONTROL_NAME}`,
'--timeout=5m',
],
{ capture: true, quiet: true, allowFailure: true },
);
if (rollout.status !== 0) {
throw new Error(
`Cluster Control rollout unavailable: ${JSON.stringify(
controlRolloutFailureEvidence(fixture),
)}`,
);
}
return (
await waitFor('two ready Cluster Control replicas', 120_000, () => {
const deployment = fixture.kubectlJson([
@@ -2218,6 +2353,8 @@ if (require.main === module) {
module.exports = {
auditListCommand,
clusterControlResources,
controlRolloutFailureEvidence,
controlTerminationFact,
credentialAuthenticationProbeSource,
credentialIssueCommand,
credentialRevokeCommand,
test/back/ql3SecurityAdministrationKubernetesLiveContract.test.cjs
+79
View File
@@ -7,6 +7,8 @@ const { test } = require('node:test');
const {
auditListCommand,
clusterControlResources,
controlRolloutFailureEvidence,
controlTerminationFact,
credentialAuthenticationProbeSource,
credentialIssueCommand,
credentialRevokeCommand,
@@ -199,6 +201,10 @@ test('runs the credential ceremony against two real anti-affine control replicas
assert.equal(deployment.kind, 'Deployment');
assert.equal(deployment.spec.replicas, 2);
assert.equal(deployment.spec.strategy.rollingUpdate.maxUnavailable, 0);
assert.equal(
deployment.spec.template.spec.containers[0].terminationMessagePolicy,
'FallbackToLogsOnError',
);
assert.equal(
deployment.spec.template.spec.affinity.podAntiAffinity
.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey,
@@ -218,6 +224,79 @@ test('runs the credential ceremony against two real anti-affine control replicas
);
});
test('keeps failed control rollout evidence bounded and content-free', () => {
const failure = JSON.stringify({
schemaVersion: 1,
component: 'qinglong3-cluster-control',
level: 'error',
event: 'process_failed',
name: 'ClusterControlDatabaseUnavailableError',
code: 'ECONNREFUSED',
});
assert.deepEqual(controlTerminationFact(`ignored\n${failure}\n`), {
name: 'ClusterControlDatabaseUnavailableError',
code: 'ECONNREFUSED',
});
assert.equal(
controlTerminationFact(
JSON.stringify({
schemaVersion: 1,
component: 'qinglong3-cluster-control',
level: 'error',
event: 'process_failed',
name: 'Error',
secret: 'must-not-escape',
}),
),
'rejected',
);
const evidence = controlRolloutFailureEvidence({
kubectlJson(arguments_) {
if (arguments_.includes('deployment')) {
return {
metadata: { generation: 3 },
status: {
observedGeneration: 3,
replicas: 2,
updatedReplicas: 2,
unavailableReplicas: 2,
conditions: [
{ type: 'Available', status: 'False', reason: 'MinimumReplicasUnavailable' },
],
},
};
}
return {
items: [{
metadata: { name: 'control-1' },
spec: { nodeName: 'worker-1' },
status: {
phase: 'Running',
conditions: [
{ type: 'Ready', status: 'False', reason: 'ContainersNotReady' },
],
containerStatuses: [{
name: 'cluster-control',
ready: false,
restartCount: 2,
state: { waiting: { reason: 'CrashLoopBackOff' } },
lastState: { terminated: { exitCode: 1, reason: 'Error', message: failure } },
}],
},
}],
};
},
});
assert.equal(evidence.deployment.availableReplicas, 0);
assert.equal(evidence.pods[0].containers[0].state.reason, 'CrashLoopBackOff');
assert.deepEqual(evidence.pods[0].containers[0].failure, {
name: 'ClusterControlDatabaseUnavailableError',
code: 'ECONNREFUSED',
});
assert.doesNotMatch(JSON.stringify(evidence), /must-not-escape/);
});
test('keeps the real authentication probe content-free', () => {
const source = credentialAuthenticationProbeSource();
assert.match(source, /ql3-security-live-control/);