open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-20 16:07:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

ci(ql3): close network policy startup race

Browse Source
This commit is contained in:
whyour
2026-08-25 22:38:40 +08:00
parent 309694ebde
commit d2ebce85fa
2 changed files with 109 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ql3-security-administration-kubernetes-live-contract.cjs
+92 -6
View File
@@ -228,9 +228,48 @@ function deliveryVolumeProvisionSource() {
return `'use strict';const fs=require('node:fs');const root='/delivery';const facts=()=>{const status=fs.lstatSync(root);return{mode:(status.mode&0o7777).toString(8),uid:status.uid,gid:status.gid,directory:status.isDirectory(),symlink:status.isSymbolicLink()}};const before=facts();if(!before.directory||before.symlink||before.uid!==0||before.gid!==10001||before.mode!=='2777')throw new Error('UNEXPECTED_VOLUME_ROOT');fs.chmodSync(root,0o2770);const descriptor=fs.openSync(root,fs.constants.O_RDONLY|(fs.constants.O_DIRECTORY??0));try{fs.fsyncSync(descriptor)}finally{fs.closeSync(descriptor)}const after=facts();if(!after.directory||after.symlink||after.uid!==0||after.gid!==10001||after.mode!=='2770')throw new Error('VOLUME_ROOT_NOT_CONSTRAINED');fs.writeFileSync('/dev/termination-log',JSON.stringify({schema:'qinglong/security-administration-delivery-volume-provision@v1',beforeMode:before.mode,afterMode:after.mode,uid:after.uid,gid:after.gid,passed:true}));`;
}
function networkPolicyReadinessSource() {
return String.raw`
const fs=require('node:fs');const net=require('node:net');
const connect=(host,port)=>new Promise((resolve)=>{let settled=false;const socket=net.createConnection({host,port});const done=(ok)=>{if(settled)return;settled=true;socket.destroy();resolve(ok)};socket.setTimeout(500);socket.once('connect',()=>done(true));socket.once('timeout',()=>done(false));socket.once('error',()=>done(false));});
const sleep=(milliseconds)=>new Promise((resolve)=>setTimeout(resolve,milliseconds));
(async()=>{let consecutive=0;let networkEvidence={databaseConnected:false,kubernetesApiConnected:false,publicInternetConnected:false};for(let attempt=1;attempt<=120;attempt+=1){const kubernetesApiConnected=await connect(process.argv[2],443);const publicInternetConnected=await connect('1.1.1.1',443);const databaseConnected=await connect(process.argv[1],5432);networkEvidence={databaseConnected,kubernetesApiConnected,publicInternetConnected};consecutive=databaseConnected&&!kubernetesApiConnected&&!publicInternetConnected?consecutive+1:0;if(consecutive>=2){fs.writeFileSync('/dev/termination-log',JSON.stringify({schemaVersion:1,passed:true,attempt,consecutive,...networkEvidence}));return}await sleep(250)}fs.writeFileSync('/dev/termination-log',JSON.stringify({schemaVersion:1,passed:false,code:'NETWORK_POLICY_NOT_ENFORCED',...networkEvidence}));process.exitCode=1})().catch(()=>{fs.writeFileSync('/dev/termination-log',JSON.stringify({schemaVersion:1,passed:false,code:'NETWORK_POLICY_READINESS_UNAVAILABLE'}));process.exitCode=1});`;
}
function networkPolicyReadinessContainer(adminImage, kubernetesServiceIp) {
return {
name: 'wait-network-policy',
image: adminImage,
imagePullPolicy: 'Never',
command: [
'node',
'-e',
networkPolicyReadinessSource(),
'ql3-postgres-rw.qinglong3-system.svc',
kubernetesServiceIp,
],
terminationMessagePolicy: 'File',
securityContext: {
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
capabilities: { drop: ['ALL'] },
},
resources: {
requests: { cpu: '5m', memory: '16Mi' },
limits: { cpu: '100m', memory: '64Mi' },
},
};
}
function administrationJob(
template,
{ name, inputSecretName, deliveryFile, projectedMode },
{
name,
inputSecretName,
deliveryFile,
projectedMode,
kubernetesServiceIp,
},
) {
const job = structuredClone(template);
job.metadata.name = name;
@@ -256,6 +295,9 @@ function administrationJob(
),
);
job.spec.template.spec.initContainers.unshift(inspector);
job.spec.template.spec.initContainers.unshift(
networkPolicyReadinessContainer(stager.image, kubernetesServiceIp),
);
const administrator = findNamed(
job.spec.template.spec.containers,
'administrator',
@@ -275,6 +317,10 @@ function administrationJob(
}
function administrationFailureEvidence(snapshot) {
const readinessStatus = snapshot.pod.status.initContainerStatuses?.find(
(container) => container.name === 'wait-network-policy',
);
const readinessTerminated = readinessStatus?.state?.terminated;
const inspectorStatus = snapshot.pod.status.initContainerStatuses?.find(
(container) => container.name === 'inspect-input-authority',
);
@@ -289,6 +335,11 @@ function administrationFailureEvidence(snapshot) {
const evidence = {
jobComplete: snapshot.complete,
jobFailed: snapshot.failed,
networkPolicyExitCode: readinessTerminated?.exitCode ?? null,
networkPolicyReason:
readinessTerminated?.reason ??
readinessStatus?.state?.waiting?.reason ??
null,
initExitCode: initTerminated?.exitCode ?? null,
initReason:
initTerminated?.reason ?? initStatus?.state?.waiting?.reason ?? null,
@@ -378,13 +429,36 @@ async function terminalJobSnapshot(fixture, name, timeoutMs = 180_000) {
'-l',
`batch.kubernetes.io/job-name=${name}`,
]).items;
const podFacts = pods.map((pod) => ({
phase: pod.status.phase ?? null,
nodeAssigned: typeof pod.spec.nodeName === 'string',
init: (pod.status.initContainerStatuses ?? []).map((container) => ({
name: container.name,
waitingReason: container.state?.waiting?.reason ?? null,
terminatedReason: container.state?.terminated?.reason ?? null,
exitCode: container.state?.terminated?.exitCode ?? null,
})),
containers: (pod.status.containerStatuses ?? []).map((container) => ({
name: container.name,
waitingReason: container.state?.waiting?.reason ?? null,
terminatedReason: container.state?.terminated?.reason ?? null,
exitCode: container.state?.terminated?.exitCode ?? null,
})),
}));
return (complete || failed) && pods.length === 1
? { ready: true, value: { job, pod: pods[0], complete, failed } }
: {
ready: false,
fact: `${pods.length} Pods; status=${JSON.stringify(
job.status ?? {},
)}`,
fact: JSON.stringify({
podCount: pods.length,
job: {
active: job.status?.active ?? 0,
ready: job.status?.ready ?? 0,
succeeded: job.status?.succeeded ?? 0,
failed: job.status?.failed ?? 0,
},
pods: podFacts,
}),
};
})
).value;
@@ -428,6 +502,13 @@ async function runAdministrationJob({
createdJobs,
createdSecrets,
}) {
const kubernetesServiceIp = fixture.kubectlJson([
'-n',
'default',
'get',
'service',
'kubernetes',
]).spec.clusterIP;
const inputSecretName = `${name}-input`;
fixture.create({
apiVersion: 'v1',
@@ -448,6 +529,7 @@ async function runAdministrationJob({
inputSecretName,
deliveryFile,
projectedMode,
kubernetesServiceIp,
});
fixture.create(job);
createdJobs.add(name);
@@ -540,6 +622,9 @@ async function runCustodyEvidence({
fsGroupChangePolicy: 'OnRootMismatch',
seccompProfile: { type: 'RuntimeDefault' },
},
initContainers: [
networkPolicyReadinessContainer(adminImage, kubernetesServiceIp),
],
containers: [
{
name: 'evidence',
@@ -584,7 +669,7 @@ async function runCustodyEvidence({
});
createdEvidenceJobs.add(name);
try {
const snapshot = await terminalJobSnapshot(fixture, name, 120_000);
const snapshot = await terminalJobSnapshot(fixture, name, 180_000);
const state = snapshot.pod.status.containerStatuses?.[0]?.state?.terminated;
const statusEvidence = state?.message || JSON.stringify({
exitCode: state?.exitCode ?? null,
@@ -680,7 +765,7 @@ async function provisionDeliveryVolume({
});
createdEvidenceJobs.add(name);
try {
const snapshot = await terminalJobSnapshot(fixture, name, 120_000);
const snapshot = await terminalJobSnapshot(fixture, name, 180_000);
assert.equal(snapshot.complete, true);
assert.equal(snapshot.failed, false);
const state = snapshot.pod.status.containerStatuses?.[0]?.state?.terminated;
@@ -1437,4 +1522,5 @@ module.exports = {
identityRegisterCommand,
inputAuthorityEvidenceSource,
migrationFailureEvidence,
networkPolicyReadinessSource,
};
test/back/ql3SecurityAdministrationKubernetesLiveContract.test.cjs
+17
View File
@@ -15,6 +15,7 @@ const {
identityRegisterCommand,
inputAuthorityEvidenceSource,
migrationFailureEvidence,
networkPolicyReadinessSource,
} = require('../../scripts/ql3-security-administration-kubernetes-live-contract.cjs');
const values = Object.freeze({
@@ -164,6 +165,21 @@ test('keeps migration failure evidence content-free', () => {
);
});
test('waits for per-Pod network policy before mounting private material', () => {
const source = networkPolicyReadinessSource();
assert.match(source, /NETWORK_POLICY_NOT_ENFORCED/);
assert.match(source, /databaseConnected/);
assert.match(source, /kubernetesApiConnected/);
assert.match(source, /publicInternetConnected/);
assert.match(source, /attempt<=120/);
assert.match(source, /consecutive>=2/);
assert.ok(
source.indexOf('kubernetesApiConnected=await connect') <
source.indexOf('databaseConnected=await connect'),
);
assert.doesNotMatch(source, /readFile|process\.env|console\.log/);
});
test('live runner remains opt-in, reviewed, cleanup-bound and log-free', () => {
const source = fs.readFileSync(
path.resolve(
@@ -180,6 +196,7 @@ test('live runner remains opt-in, reviewed, cleanup-bound and log-free', () => {
assert.match(source, /validateSecurityAdministrationKubernetesLiveReport/);
assert.match(source, /net\.ipv4\.ip_forward=1/);
assert.match(source, /net\.bridge\.bridge-nf-call-iptables=1/);
assert.match(source, /wait-network-policy/);
assert.match(source, /projectedMode: 0o444/);
assert.match(source, /credential\.issue\.replay/);
assert.match(source, /FallbackToLogsOnError/);