mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
2426 lines
88 KiB
JavaScript
2426 lines
88 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
'use strict';
|
|
|
|
const assert = require('node:assert/strict');
|
|
const crypto = require('node:crypto');
|
|
const fs = require('node:fs');
|
|
const path = require('node:path');
|
|
const yaml = require('js-yaml');
|
|
|
|
const {
|
|
K3sDockerLiveFixture,
|
|
run,
|
|
waitFor,
|
|
} = require('./lib/ql3-k3s-docker-live.cjs');
|
|
const {
|
|
applySecret,
|
|
currentPrimaryPod,
|
|
imageIdDigest,
|
|
localManifest,
|
|
psql,
|
|
} = require('./lib/ql3-management-kubernetes-live-platform.cjs');
|
|
const {
|
|
podReady,
|
|
} = require('./lib/ql3-management-kubernetes-live.cjs');
|
|
const {
|
|
createManagementIdentityCeremony,
|
|
} = require('./lib/ql3-management-live-identity.cjs');
|
|
const {
|
|
imageDigest,
|
|
imageTag,
|
|
reviewedOperatorManifest,
|
|
} = require('./ql3-cloudnativepg-live-contract.cjs');
|
|
const {
|
|
FIXTURE,
|
|
LIMITATIONS,
|
|
validateSecurityAdministrationKubernetesLiveReport,
|
|
} = require('./ql3-security-administration-kubernetes-live-audit.cjs');
|
|
|
|
const ROOT = path.resolve(__dirname, '..');
|
|
const NAMESPACE = 'qinglong3-system';
|
|
const NAME = 'ql3-security-administration';
|
|
const POSTGRES_CLUSTER = 'ql3-postgres';
|
|
const DELIVERY_CLAIM = 'ql3-security-administration-delivery';
|
|
const CONTROL_NAME = 'ql3-security-live-control';
|
|
const CONTROL_RUNTIME_SECRET = 'ql3-security-live-control-runtime';
|
|
const ADMIN_IMAGE_BASE = 'ql3-security-administration-live';
|
|
const CONTROL_IMAGE_BASE = 'ql3-security-administration-migration-live';
|
|
const ISSUER = 'https://identity.qinglong.test/';
|
|
const ROLE_NAMES = Object.freeze([
|
|
'ql3_migration',
|
|
'ql3_ai_maintenance',
|
|
'ql3_ai_credential_manager',
|
|
'ql3_ai_credential_tester',
|
|
'ql3_runtime',
|
|
'ql3_admin',
|
|
'ql3_package_manager',
|
|
'ql3_package_executor',
|
|
'ql3_automation_manager',
|
|
'ql3_approval_manager',
|
|
'ql3_run_manager',
|
|
'ql3_worker_credential_manager',
|
|
'ql3_worker_credential_executor',
|
|
'ql3_worker_ingress',
|
|
]);
|
|
const LOCK = JSON.parse(
|
|
fs.readFileSync(
|
|
path.join(
|
|
ROOT,
|
|
'deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/operator-lock.json',
|
|
),
|
|
'utf8',
|
|
),
|
|
);
|
|
const OPERATOR_IMAGE = LOCK.operator.image;
|
|
const POSTGRES_IMAGE = LOCK.operand.image;
|
|
const identity = createManagementIdentityCeremony({
|
|
issuer: ISSUER,
|
|
audience: 'qinglong3-security-administration',
|
|
purpose: 'security-administration',
|
|
tokenType: 'ql3-security-administration+jwt',
|
|
subject: 'security-owner',
|
|
jtiPrefix: 'ql3-security-administration-live',
|
|
});
|
|
|
|
function randomSecret() {
|
|
return crypto.randomBytes(32).toString('base64url');
|
|
}
|
|
|
|
function apiCredentialPepperKeyring(activePepperKeyId, keys) {
|
|
assert.ok(keys.length === 1 || keys.length === 2);
|
|
assert.ok(keys.some((key) => key.pepperKeyId === activePepperKeyId));
|
|
return Object.freeze({
|
|
schemaVersion: 1,
|
|
activePepperKeyId,
|
|
keys: Object.freeze(
|
|
keys.map((key) =>
|
|
Object.freeze({
|
|
pepperKeyId: key.pepperKeyId,
|
|
pepper: key.pepper,
|
|
}),
|
|
),
|
|
),
|
|
});
|
|
}
|
|
|
|
function sqlString(value) {
|
|
assert.equal(typeof value, 'string');
|
|
return `'${value.replaceAll("'", "''")}'`;
|
|
}
|
|
|
|
function identityRegisterCommand(values) {
|
|
return Object.freeze({
|
|
schemaVersion: 1,
|
|
operation: 'identity.register',
|
|
request: Object.freeze({
|
|
mutationId: values.identityMutationId,
|
|
requestId: values.registerRequestId,
|
|
expectedCurrentVersion: 0,
|
|
subject: values.subject,
|
|
}),
|
|
});
|
|
}
|
|
|
|
function auditListCommand() {
|
|
return Object.freeze({
|
|
schemaVersion: 1,
|
|
operation: 'audit.list',
|
|
request: Object.freeze({
|
|
limit: 25,
|
|
filter: Object.freeze({ outcome: 'allowed' }),
|
|
}),
|
|
});
|
|
}
|
|
|
|
function pepperReferencesCommand(pepperKeyId) {
|
|
return Object.freeze({
|
|
schemaVersion: 1,
|
|
operation: 'pepper.references',
|
|
request: Object.freeze({ pepperKeyId, limit: 64 }),
|
|
});
|
|
}
|
|
|
|
function credentialIssueCommand(values) {
|
|
return Object.freeze({
|
|
schemaVersion: 1,
|
|
operation: 'credential.issue',
|
|
request: Object.freeze({
|
|
mutationId: values.issueMutationId,
|
|
requestId: values.issueRequestId,
|
|
expectedCurrentVersion: 0,
|
|
credentialId: values.credentialId,
|
|
subject: values.subject,
|
|
notBeforeAtMs: values.notBeforeAtMs,
|
|
expiresAtMs: values.expiresAtMs,
|
|
}),
|
|
});
|
|
}
|
|
|
|
function credentialRotateCommand(values) {
|
|
return Object.freeze({
|
|
schemaVersion: 1,
|
|
operation: 'credential.rotate',
|
|
request: Object.freeze({
|
|
mutationId: values.rotateMutationId,
|
|
requestId: values.rotateRequestId,
|
|
expectedCurrentVersion: 1,
|
|
credentialId: values.credentialId,
|
|
subject: values.subject,
|
|
notBeforeAtMs: values.notBeforeAtMs,
|
|
expiresAtMs: values.expiresAtMs,
|
|
}),
|
|
});
|
|
}
|
|
|
|
function credentialRevokeCommand(values) {
|
|
return Object.freeze({
|
|
schemaVersion: 1,
|
|
operation: 'credential.revoke',
|
|
request: Object.freeze({
|
|
mutationId: values.revokeMutationId,
|
|
requestId: values.revokeRequestId,
|
|
expectedCurrentVersion: values.expectedCurrentVersion ?? 2,
|
|
credentialId: values.credentialId,
|
|
subject: values.subject,
|
|
}),
|
|
});
|
|
}
|
|
|
|
function privateReportPath(argv) {
|
|
if (
|
|
argv.length !== 1 ||
|
|
!argv[0].startsWith('--report=') ||
|
|
!path.isAbsolute(argv[0].slice('--report='.length))
|
|
) {
|
|
throw new Error(
|
|
'usage: ql3-security-administration-kubernetes-live-contract --report=/absolute/private-report.json',
|
|
);
|
|
}
|
|
const reportFile = argv[0].slice('--report='.length);
|
|
if (fs.existsSync(reportFile)) {
|
|
throw new Error('refusing to overwrite the Security Administration report');
|
|
}
|
|
const parent = fs.lstatSync(path.dirname(reportFile));
|
|
if (!parent.isDirectory() || parent.isSymbolicLink()) {
|
|
throw new Error('Security Administration report parent must be real');
|
|
}
|
|
return reportFile;
|
|
}
|
|
|
|
function localAdministrationManifest(rendered, localImage) {
|
|
const placeholder =
|
|
'registry.example.com/qinglong/qinglong3-cluster-admin@sha256:' +
|
|
'0'.repeat(64);
|
|
assert.equal(rendered.split(placeholder).length - 1, 2);
|
|
return rendered
|
|
.replaceAll(placeholder, localImage)
|
|
.replaceAll('imagePullPolicy: IfNotPresent', 'imagePullPolicy: Never');
|
|
}
|
|
|
|
function renderedResources(fixture, directory, localImage) {
|
|
const rendered = localAdministrationManifest(
|
|
fixture.kubectl(['kustomize', directory], {
|
|
capture: true,
|
|
quiet: true,
|
|
}).stdout,
|
|
localImage,
|
|
);
|
|
const resources = [];
|
|
yaml.loadAll(rendered, (resource) => {
|
|
if (resource) resources.push(resource);
|
|
});
|
|
return resources;
|
|
}
|
|
|
|
function namedResource(resources, kind, name) {
|
|
const selected = resources.find(
|
|
(resource) => resource.kind === kind && resource.metadata?.name === name,
|
|
);
|
|
assert.ok(selected, `${kind}/${name} is absent`);
|
|
return selected;
|
|
}
|
|
|
|
function findNamed(values, name) {
|
|
const selected = values.find((value) => value.name === name);
|
|
assert.ok(selected, `${name} is absent`);
|
|
return selected;
|
|
}
|
|
|
|
function inputAuthorityEvidenceSource() {
|
|
return `'use strict';const fs=require('node:fs');const path=require('node:path');const names=['command.json','assertion.jwt','keyset.json','pepper-keyring.json'];const source='/var/run/secrets/qinglong3/security-administration-projected';const parent='/var/run/qinglong3/security-administration-private';const target=parent+'/input';const deliveryRoot='/var/lib/qinglong3/security-administration-delivery';const deliveryPrivate=deliveryRoot+'/private';const facts=(value)=>{const status=fs.lstatSync(value);return{mode:(status.mode&0o7777).toString(8),uid:status.uid,gid:status.gid,directory:status.isDirectory(),file:status.isFile(),symlink:status.isSymbolicLink()}};const sourceReal=fs.realpathSync(source);const files=names.map((name)=>{const candidate=source+'/'+name;const resolved=fs.realpathSync(candidate);const relative=path.relative(sourceReal,resolved);return{name,link:facts(candidate),resolved:facts(resolved),confined:relative!==''&&relative!=='..'&&!relative.startsWith('../')&&!path.isAbsolute(relative)}});fs.writeFileSync('/dev/termination-log',JSON.stringify({schema:'qinglong/security-administration-input-authority@v1',source:facts(source),parent:facts(parent),targetExists:fs.existsSync(target),deliveryRoot:fs.existsSync(deliveryRoot)?facts(deliveryRoot):null,deliveryPrivateExists:fs.existsSync(deliveryPrivate),files}));`;
|
|
}
|
|
|
|
function deliveryVolumeProvisionSource() {
|
|
return `'use strict';const fs=require('node:fs');const root='/delivery';const facts=()=>{const status=fs.lstatSync(root);return{mode:(status.mode&0o7777).toString(8),uid:status.uid,gid:status.gid,directory:status.isDirectory(),symlink:status.isSymbolicLink()}};const before=facts();if(!before.directory||before.symlink||before.uid!==0||before.gid!==10001||before.mode!=='2777')throw new Error('UNEXPECTED_VOLUME_ROOT');fs.chmodSync(root,0o2770);const descriptor=fs.openSync(root,fs.constants.O_RDONLY|(fs.constants.O_DIRECTORY??0));try{fs.fsyncSync(descriptor)}finally{fs.closeSync(descriptor)}const after=facts();if(!after.directory||after.symlink||after.uid!==0||after.gid!==10001||after.mode!=='2770')throw new Error('VOLUME_ROOT_NOT_CONSTRAINED');fs.writeFileSync('/dev/termination-log',JSON.stringify({schema:'qinglong/security-administration-delivery-volume-provision@v1',beforeMode:before.mode,afterMode:after.mode,uid:after.uid,gid:after.gid,passed:true}));`;
|
|
}
|
|
|
|
function networkPolicyReadinessSource() {
|
|
return String.raw`
|
|
const fs=require('node:fs');const net=require('node:net');
|
|
const connect=(host,port)=>new Promise((resolve)=>{let settled=false;const socket=net.createConnection({host,port});const done=(ok)=>{if(settled)return;settled=true;socket.destroy();resolve(ok)};socket.setTimeout(500);socket.once('connect',()=>done(true));socket.once('timeout',()=>done(false));socket.once('error',()=>done(false));});
|
|
const sleep=(milliseconds)=>new Promise((resolve)=>setTimeout(resolve,milliseconds));
|
|
(async()=>{let consecutive=0;let networkEvidence={databaseConnected:false,kubernetesApiConnected:false,publicInternetConnected:false};for(let attempt=1;attempt<=120;attempt+=1){const kubernetesApiConnected=await connect(process.argv[2],443);const publicInternetConnected=await connect('1.1.1.1',443);const databaseConnected=await connect(process.argv[1],5432);networkEvidence={databaseConnected,kubernetesApiConnected,publicInternetConnected};consecutive=databaseConnected&&!kubernetesApiConnected&&!publicInternetConnected?consecutive+1:0;if(consecutive>=2){fs.writeFileSync('/dev/termination-log',JSON.stringify({schemaVersion:1,passed:true,attempt,consecutive,...networkEvidence}));return}await sleep(250)}fs.writeFileSync('/dev/termination-log',JSON.stringify({schemaVersion:1,passed:false,code:'NETWORK_POLICY_NOT_ENFORCED',...networkEvidence}));process.exitCode=1})().catch(()=>{fs.writeFileSync('/dev/termination-log',JSON.stringify({schemaVersion:1,passed:false,code:'NETWORK_POLICY_READINESS_UNAVAILABLE'}));process.exitCode=1});`;
|
|
}
|
|
|
|
function networkPolicyReadinessContainer(adminImage, kubernetesServiceIp) {
|
|
return {
|
|
name: 'wait-network-policy',
|
|
image: adminImage,
|
|
imagePullPolicy: 'Never',
|
|
command: [
|
|
'node',
|
|
'-e',
|
|
networkPolicyReadinessSource(),
|
|
'ql3-postgres-rw.qinglong3-system.svc',
|
|
kubernetesServiceIp,
|
|
],
|
|
terminationMessagePolicy: 'File',
|
|
securityContext: {
|
|
allowPrivilegeEscalation: false,
|
|
readOnlyRootFilesystem: true,
|
|
capabilities: { drop: ['ALL'] },
|
|
},
|
|
resources: {
|
|
requests: { cpu: '5m', memory: '16Mi' },
|
|
limits: { cpu: '100m', memory: '64Mi' },
|
|
},
|
|
};
|
|
}
|
|
|
|
function administrationJob(
|
|
template,
|
|
{
|
|
name,
|
|
inputSecretName,
|
|
deliveryFile,
|
|
projectedMode,
|
|
kubernetesServiceIp,
|
|
},
|
|
) {
|
|
const job = structuredClone(template);
|
|
job.metadata.name = name;
|
|
delete job.metadata.uid;
|
|
delete job.metadata.resourceVersion;
|
|
job.spec.template.metadata.labels['ql3.live/job'] = name;
|
|
const projected = findNamed(job.spec.template.spec.volumes, 'projected-input');
|
|
projected.secret.secretName = inputSecretName;
|
|
projected.secret.defaultMode = projectedMode ?? 0o440;
|
|
const stager = findNamed(
|
|
job.spec.template.spec.initContainers,
|
|
'stage-private-input',
|
|
);
|
|
stager.terminationMessagePolicy = 'FallbackToLogsOnError';
|
|
const inspector = structuredClone(stager);
|
|
inspector.name = 'inspect-input-authority';
|
|
inspector.command = ['node', '-e', inputAuthorityEvidenceSource()];
|
|
inspector.args = [];
|
|
inspector.terminationMessagePolicy = 'File';
|
|
inspector.volumeMounts = inspector.volumeMounts.filter((mount) =>
|
|
['projected-input', 'private-input', 'credential-delivery'].includes(
|
|
mount.name,
|
|
),
|
|
);
|
|
job.spec.template.spec.initContainers.unshift(inspector);
|
|
job.spec.template.spec.initContainers.unshift(
|
|
networkPolicyReadinessContainer(stager.image, kubernetesServiceIp),
|
|
);
|
|
const administrator = findNamed(
|
|
job.spec.template.spec.containers,
|
|
'administrator',
|
|
);
|
|
administrator.terminationMessagePolicy = 'FallbackToLogsOnError';
|
|
const deliveryIndex = administrator.args.findIndex((argument) =>
|
|
argument.startsWith('--delivery='),
|
|
);
|
|
if (deliveryFile === undefined) {
|
|
assert.equal(deliveryIndex, -1);
|
|
} else {
|
|
assert.ok(deliveryIndex >= 0);
|
|
administrator.args[deliveryIndex] =
|
|
`--delivery=/var/lib/qinglong3/security-administration-delivery/private/${deliveryFile}`;
|
|
}
|
|
return job;
|
|
}
|
|
|
|
function administrationFailureEvidence(snapshot) {
|
|
const readinessStatus = snapshot.pod.status.initContainerStatuses?.find(
|
|
(container) => container.name === 'wait-network-policy',
|
|
);
|
|
const readinessTerminated = readinessStatus?.state?.terminated;
|
|
const inspectorStatus = snapshot.pod.status.initContainerStatuses?.find(
|
|
(container) => container.name === 'inspect-input-authority',
|
|
);
|
|
const initStatus = snapshot.pod.status.initContainerStatuses?.find(
|
|
(container) => container.name === 'stage-private-input',
|
|
);
|
|
const initTerminated = initStatus?.state?.terminated;
|
|
const status = snapshot.pod.status.containerStatuses?.find(
|
|
(container) => container.name === 'administrator',
|
|
);
|
|
const terminated = status?.state?.terminated;
|
|
const evidence = {
|
|
jobComplete: snapshot.complete,
|
|
jobFailed: snapshot.failed,
|
|
networkPolicyExitCode: readinessTerminated?.exitCode ?? null,
|
|
networkPolicyReason:
|
|
readinessTerminated?.reason ??
|
|
readinessStatus?.state?.waiting?.reason ??
|
|
null,
|
|
initExitCode: initTerminated?.exitCode ?? null,
|
|
initReason:
|
|
initTerminated?.reason ?? initStatus?.state?.waiting?.reason ?? null,
|
|
mainExitCode: terminated?.exitCode ?? null,
|
|
mainReason: terminated?.reason ?? status?.state?.waiting?.reason ?? null,
|
|
};
|
|
let inputAuthority;
|
|
try {
|
|
const parsed = JSON.parse(
|
|
inspectorStatus?.state?.terminated?.message?.trim() || '',
|
|
);
|
|
if (parsed.schema === 'qinglong/security-administration-input-authority@v1') {
|
|
inputAuthority = parsed;
|
|
}
|
|
} catch {
|
|
inputAuthority = 'unavailable';
|
|
}
|
|
const message = (
|
|
initTerminated?.message || terminated?.message || ''
|
|
).trim();
|
|
if (!message) return { ...evidence, inputAuthority };
|
|
try {
|
|
const parsed = JSON.parse(message);
|
|
const allowedKeys = ['schemaVersion', 'component', 'event', 'name', 'code'];
|
|
if (
|
|
!parsed ||
|
|
typeof parsed !== 'object' ||
|
|
Array.isArray(parsed) ||
|
|
Object.keys(parsed).some((key) => !allowedKeys.includes(key))
|
|
) {
|
|
return { ...evidence, inputAuthority, failureMessage: 'rejected' };
|
|
}
|
|
return { ...evidence, inputAuthority, failureMessage: parsed };
|
|
} catch {
|
|
return { ...evidence, inputAuthority, failureMessage: 'unparseable' };
|
|
}
|
|
}
|
|
|
|
function migrationFailureEvidence(snapshot) {
|
|
const status = snapshot.pod.status.containerStatuses?.find(
|
|
(container) => container.name === 'migration',
|
|
);
|
|
const terminated = status?.state?.terminated;
|
|
const evidence = {
|
|
jobComplete: snapshot.complete,
|
|
jobFailed: snapshot.failed,
|
|
podPhase: snapshot.pod.status.phase ?? null,
|
|
exitCode: terminated?.exitCode ?? null,
|
|
reason: terminated?.reason ?? status?.state?.waiting?.reason ?? null,
|
|
};
|
|
const lines = (terminated?.message || '')
|
|
.split('\n')
|
|
.map((line) => line.trim())
|
|
.filter(Boolean)
|
|
.reverse();
|
|
if (lines.length === 0) return evidence;
|
|
let rejected = false;
|
|
for (const line of lines) {
|
|
let parsed;
|
|
try {
|
|
parsed = JSON.parse(line);
|
|
} catch {
|
|
continue;
|
|
}
|
|
const allowedKeys = ['schemaVersion', 'component', 'event', 'name', 'code'];
|
|
if (
|
|
!parsed ||
|
|
typeof parsed !== 'object' ||
|
|
Array.isArray(parsed) ||
|
|
Object.keys(parsed).some((key) => !allowedKeys.includes(key))
|
|
) {
|
|
rejected = true;
|
|
continue;
|
|
}
|
|
return { ...evidence, failureMessage: parsed };
|
|
}
|
|
return {
|
|
...evidence,
|
|
failureMessage: rejected ? 'rejected' : 'unparseable',
|
|
};
|
|
}
|
|
|
|
async function terminalJobSnapshot(fixture, name, timeoutMs = 180_000) {
|
|
return (
|
|
await waitFor(`${name} terminal`, timeoutMs, () => {
|
|
const job = fixture.kubectlJson(['-n', NAMESPACE, 'get', 'job', name]);
|
|
const complete = job.status.conditions?.some(
|
|
(condition) =>
|
|
condition.type === 'Complete' && condition.status === 'True',
|
|
);
|
|
const failed = job.status.conditions?.some(
|
|
(condition) =>
|
|
condition.type === 'Failed' && condition.status === 'True',
|
|
);
|
|
const pods = fixture.kubectlJson([
|
|
'-n',
|
|
NAMESPACE,
|
|
'get',
|
|
'pods',
|
|
'-l',
|
|
`batch.kubernetes.io/job-name=${name}`,
|
|
]).items;
|
|
const podFacts = pods.map((pod) => ({
|
|
phase: pod.status.phase ?? null,
|
|
nodeAssigned: typeof pod.spec.nodeName === 'string',
|
|
init: (pod.status.initContainerStatuses ?? []).map((container) => ({
|
|
name: container.name,
|
|
waitingReason: container.state?.waiting?.reason ?? null,
|
|
terminatedReason: container.state?.terminated?.reason ?? null,
|
|
exitCode: container.state?.terminated?.exitCode ?? null,
|
|
})),
|
|
containers: (pod.status.containerStatuses ?? []).map((container) => ({
|
|
name: container.name,
|
|
waitingReason: container.state?.waiting?.reason ?? null,
|
|
terminatedReason: container.state?.terminated?.reason ?? null,
|
|
exitCode: container.state?.terminated?.exitCode ?? null,
|
|
})),
|
|
}));
|
|
return (complete || failed) && pods.length === 1
|
|
? { ready: true, value: { job, pod: pods[0], complete, failed } }
|
|
: {
|
|
ready: false,
|
|
fact: JSON.stringify({
|
|
podCount: pods.length,
|
|
job: {
|
|
active: job.status?.active ?? 0,
|
|
ready: job.status?.ready ?? 0,
|
|
succeeded: job.status?.succeeded ?? 0,
|
|
failed: job.status?.failed ?? 0,
|
|
},
|
|
pods: podFacts,
|
|
}),
|
|
};
|
|
})
|
|
).value;
|
|
}
|
|
|
|
function deleteResource(fixture, resource) {
|
|
fixture.kubectl(
|
|
[
|
|
'-n',
|
|
NAMESPACE,
|
|
'delete',
|
|
resource,
|
|
'--ignore-not-found=true',
|
|
'--wait=true',
|
|
],
|
|
{ capture: true, quiet: true },
|
|
);
|
|
}
|
|
|
|
function resourceAbsent(fixture, resource) {
|
|
return (
|
|
fixture.kubectl(['-n', NAMESPACE, 'get', resource], {
|
|
capture: true,
|
|
quiet: true,
|
|
allowFailure: true,
|
|
}).status !== 0
|
|
);
|
|
}
|
|
|
|
async function runAdministrationJob({
|
|
fixture,
|
|
template,
|
|
name,
|
|
command,
|
|
assertion,
|
|
keyset,
|
|
pepperKeyring,
|
|
deliveryFile,
|
|
expectedComplete,
|
|
projectedMode,
|
|
createdJobs,
|
|
createdSecrets,
|
|
}) {
|
|
const kubernetesServiceIp = fixture.kubectlJson([
|
|
'-n',
|
|
'default',
|
|
'get',
|
|
'service',
|
|
'kubernetes',
|
|
]).spec.clusterIP;
|
|
const inputSecretName = `${name}-input`;
|
|
fixture.create({
|
|
apiVersion: 'v1',
|
|
kind: 'Secret',
|
|
metadata: { name: inputSecretName, namespace: NAMESPACE },
|
|
immutable: true,
|
|
type: 'Opaque',
|
|
stringData: {
|
|
'command.json': `${JSON.stringify(command)}\n`,
|
|
'assertion.jwt': `${assertion}\n`,
|
|
'keyset.json': `${JSON.stringify(keyset)}\n`,
|
|
'pepper-keyring.json': `${JSON.stringify(pepperKeyring)}\n`,
|
|
},
|
|
});
|
|
createdSecrets.add(inputSecretName);
|
|
const job = administrationJob(template, {
|
|
name,
|
|
inputSecretName,
|
|
deliveryFile,
|
|
projectedMode,
|
|
kubernetesServiceIp,
|
|
});
|
|
fixture.create(job);
|
|
createdJobs.add(name);
|
|
try {
|
|
const snapshot = await terminalJobSnapshot(fixture, name);
|
|
const failureEvidence = JSON.stringify(
|
|
administrationFailureEvidence(snapshot),
|
|
);
|
|
assert.equal(snapshot.complete, expectedComplete, failureEvidence);
|
|
assert.equal(snapshot.failed, !expectedComplete, failureEvidence);
|
|
assert.equal(snapshot.job.spec.backoffLimit, 0);
|
|
assert.equal(snapshot.job.spec.activeDeadlineSeconds, 300);
|
|
assert.equal(snapshot.job.spec.ttlSecondsAfterFinished, 600);
|
|
assert.equal(snapshot.pod.spec.serviceAccountName, NAME);
|
|
assert.equal(snapshot.pod.spec.automountServiceAccountToken, false);
|
|
assert.equal(
|
|
snapshot.pod.spec.volumes.some((volume) =>
|
|
volume.projected?.sources?.some(
|
|
(source) => source.serviceAccountToken !== undefined,
|
|
),
|
|
),
|
|
false,
|
|
);
|
|
const init = snapshot.pod.status.initContainerStatuses?.find(
|
|
(container) => container.name === 'stage-private-input',
|
|
);
|
|
assert.ok(init?.state?.terminated);
|
|
const main = snapshot.pod.status.containerStatuses?.find(
|
|
(container) => container.name === 'administrator',
|
|
);
|
|
if (expectedComplete) {
|
|
assert.equal(init.state.terminated.exitCode, 0);
|
|
assert.equal(main?.state?.terminated?.exitCode, 0);
|
|
} else {
|
|
assert.notEqual(init.state.terminated.exitCode, 0);
|
|
assert.equal(main?.state?.terminated, undefined);
|
|
assert.equal(main?.state?.running, undefined);
|
|
}
|
|
return snapshot;
|
|
} finally {
|
|
deleteResource(fixture, `job/${name}`);
|
|
createdJobs.delete(name);
|
|
deleteResource(fixture, `secret/${inputSecretName}`);
|
|
createdSecrets.delete(inputSecretName);
|
|
}
|
|
}
|
|
|
|
function custodyEvidenceSource() {
|
|
return String.raw`
|
|
const crypto=require('node:crypto');const fs=require('node:fs');const net=require('node:net');
|
|
const finish=(value,status)=>{fs.writeFileSync('/dev/termination-log',JSON.stringify(value),{encoding:'utf8',mode:0o600});process.exitCode=status};
|
|
const connect=(host,port)=>new Promise((resolve)=>{let settled=false;const socket=net.createConnection({host,port});const done=(ok)=>{if(settled)return;settled=true;socket.destroy();resolve(ok)};socket.setTimeout(3000);socket.once('connect',()=>done(true));socket.once('timeout',()=>done(false));socket.once('error',()=>done(false));});
|
|
(async()=>{let networkEvidence;try{const directory='/delivery/private';const expected=JSON.parse(process.argv[1]);const directoryStatus=fs.lstatSync(directory);if(!directoryStatus.isDirectory()||directoryStatus.isSymbolicLink()||(directoryStatus.mode&0o777)!==0o700||directoryStatus.uid!==process.geteuid())throw new Error('DIRECTORY');const names=fs.readdirSync(directory).sort();if(JSON.stringify(names)!==JSON.stringify(expected.map((entry)=>entry.name).sort()))throw new Error('COUNT');const files=[];for(const entry of expected){const file=directory+'/'+entry.name;const status=fs.lstatSync(file);if(!status.isFile()||status.isSymbolicLink()||(status.mode&0o777)!==0o600||status.uid!==process.geteuid()||status.nlink!==1||status.size<1||status.size>32768)throw new Error('FILE');const bytes=fs.readFileSync(file);try{const value=JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes));const keys=Object.keys(value).sort();const required=['credentialId','expiresAtMs','kind','mutationId','notBeforeAtMs','operation','requestId','schemaVersion','subject','token','version'].sort();if(JSON.stringify(keys)!==JSON.stringify(required)||value.schemaVersion!==1||value.kind!=='qinglong3-cluster-api-credential-delivery'||value.operation!==entry.operation||typeof value.token!=='string'||!/^ql3c_[A-Za-z0-9_-]+_[A-Za-z0-9_-]{43}$/.test(value.token))throw new Error('SCHEMA');files.push({name:entry.name,operation:entry.operation,digest:'sha256:'+crypto.createHash('sha256').update(bytes).digest('hex'),bytes:status.size,inode:String(status.ino),mtimeMs:status.mtimeMs});}finally{bytes.fill(0)}}const databaseConnected=await connect(process.argv[2],5432);const kubernetesApiConnected=await connect(process.argv[3],443);const publicInternetConnected=await connect('1.1.1.1',443);networkEvidence={databaseConnected,kubernetesApiConnected,publicInternetConnected};if(!databaseConnected||kubernetesApiConnected||publicInternetConnected)throw new Error('NETWORK');finish({schemaVersion:1,passed:true,directoryMode:'0700',fileMode:'0600',files,...networkEvidence},0)}catch(error){finish({schemaVersion:1,passed:false,code:error instanceof Error&&/^[A-Z]+$/.test(error.message)?error.message:'UNAVAILABLE',...(networkEvidence?{networkEvidence}: {})},1)}})();`;
|
|
}
|
|
|
|
async function runCustodyEvidence({
|
|
fixture,
|
|
adminImage,
|
|
name,
|
|
expected,
|
|
kubernetesServiceIp,
|
|
createdEvidenceJobs,
|
|
}) {
|
|
fixture.create({
|
|
apiVersion: 'batch/v1',
|
|
kind: 'Job',
|
|
metadata: { name, namespace: NAMESPACE },
|
|
spec: {
|
|
backoffLimit: 0,
|
|
activeDeadlineSeconds: 60,
|
|
ttlSecondsAfterFinished: 600,
|
|
template: {
|
|
metadata: {
|
|
labels: {
|
|
'app.kubernetes.io/name': NAME,
|
|
'app.kubernetes.io/component': 'security-administration',
|
|
'ql3.live/evidence': 'credential-custody',
|
|
},
|
|
},
|
|
spec: {
|
|
serviceAccountName: NAME,
|
|
automountServiceAccountToken: false,
|
|
enableServiceLinks: false,
|
|
restartPolicy: 'Never',
|
|
securityContext: {
|
|
runAsNonRoot: true,
|
|
runAsUser: 10001,
|
|
runAsGroup: 10001,
|
|
fsGroup: 10001,
|
|
fsGroupChangePolicy: 'OnRootMismatch',
|
|
seccompProfile: { type: 'RuntimeDefault' },
|
|
},
|
|
initContainers: [
|
|
networkPolicyReadinessContainer(adminImage, kubernetesServiceIp),
|
|
],
|
|
containers: [
|
|
{
|
|
name: 'evidence',
|
|
image: adminImage,
|
|
imagePullPolicy: 'Never',
|
|
command: [
|
|
'node',
|
|
'-e',
|
|
custodyEvidenceSource(),
|
|
JSON.stringify(expected),
|
|
'ql3-postgres-rw.qinglong3-system.svc',
|
|
kubernetesServiceIp,
|
|
],
|
|
terminationMessagePolicy: 'File',
|
|
securityContext: {
|
|
allowPrivilegeEscalation: false,
|
|
readOnlyRootFilesystem: true,
|
|
capabilities: { drop: ['ALL'] },
|
|
},
|
|
resources: {
|
|
requests: { cpu: '5m', memory: '16Mi' },
|
|
limits: { cpu: '100m', memory: '64Mi' },
|
|
},
|
|
volumeMounts: [
|
|
{
|
|
name: 'delivery',
|
|
mountPath: '/delivery',
|
|
readOnly: true,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
volumes: [
|
|
{
|
|
name: 'delivery',
|
|
persistentVolumeClaim: { claimName: DELIVERY_CLAIM },
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
});
|
|
createdEvidenceJobs.add(name);
|
|
try {
|
|
const snapshot = await terminalJobSnapshot(fixture, name, 180_000);
|
|
const state = snapshot.pod.status.containerStatuses?.[0]?.state?.terminated;
|
|
const statusEvidence = state?.message || JSON.stringify({
|
|
exitCode: state?.exitCode ?? null,
|
|
reason: state?.reason ?? null,
|
|
});
|
|
assert.equal(snapshot.complete, true, statusEvidence);
|
|
assert.equal(snapshot.failed, false, statusEvidence);
|
|
assert.equal(state?.exitCode, 0, state?.message);
|
|
const evidence = JSON.parse(state.message);
|
|
assert.equal(evidence.schemaVersion, 1);
|
|
assert.equal(evidence.passed, true);
|
|
assert.equal(evidence.databaseConnected, true);
|
|
assert.equal(evidence.kubernetesApiConnected, false);
|
|
assert.equal(evidence.publicInternetConnected, false);
|
|
return evidence;
|
|
} finally {
|
|
deleteResource(fixture, `job/${name}`);
|
|
createdEvidenceJobs.delete(name);
|
|
}
|
|
}
|
|
|
|
function runtimeFileMaterializationSource({
|
|
postgresDirectory = '/var/run/secrets/qinglong3/postgres-projected',
|
|
keyringDirectory = '/var/run/secrets/qinglong3/api-credential-projected',
|
|
targetDirectory = '/var/run/secrets/qinglong3/runtime',
|
|
} = {}) {
|
|
for (const value of [
|
|
postgresDirectory,
|
|
keyringDirectory,
|
|
targetDirectory,
|
|
]) {
|
|
assert.equal(path.isAbsolute(value), true);
|
|
}
|
|
return [
|
|
"const fs=require('node:fs')",
|
|
"const path=require('node:path')",
|
|
`const target=${JSON.stringify(targetDirectory)}`,
|
|
`const files=[[${JSON.stringify(postgresDirectory + '/..data')},'ca.crt'],[${JSON.stringify(keyringDirectory + '/..data')},'keyring.json']]`,
|
|
"for(const [directory,name] of files){const source=fs.realpathSync(directory);const output=path.join(target,name);fs.copyFileSync(path.join(source,name),output,fs.constants.COPYFILE_EXCL);fs.chmodSync(output,0o400)}",
|
|
].join(';');
|
|
}
|
|
|
|
function clusterControlResources(controlImage) {
|
|
const labels = Object.freeze({
|
|
'app.kubernetes.io/name': CONTROL_NAME,
|
|
'app.kubernetes.io/component': 'control-plane',
|
|
'app.kubernetes.io/part-of': 'qinglong3',
|
|
});
|
|
return Object.freeze([
|
|
{
|
|
apiVersion: 'v1',
|
|
kind: 'Service',
|
|
metadata: { name: CONTROL_NAME, namespace: NAMESPACE, labels },
|
|
spec: {
|
|
selector: labels,
|
|
ports: [{ name: 'http', port: 5800, targetPort: 'http' }],
|
|
},
|
|
},
|
|
{
|
|
apiVersion: 'apps/v1',
|
|
kind: 'Deployment',
|
|
metadata: { name: CONTROL_NAME, namespace: NAMESPACE, labels },
|
|
spec: {
|
|
replicas: 2,
|
|
minReadySeconds: 2,
|
|
progressDeadlineSeconds: 300,
|
|
strategy: {
|
|
type: 'RollingUpdate',
|
|
rollingUpdate: { maxUnavailable: 0, maxSurge: 1 },
|
|
},
|
|
selector: { matchLabels: labels },
|
|
template: {
|
|
metadata: { labels },
|
|
spec: {
|
|
serviceAccountName: 'ql3-cluster-control',
|
|
automountServiceAccountToken: false,
|
|
enableServiceLinks: false,
|
|
terminationGracePeriodSeconds: 20,
|
|
securityContext: {
|
|
runAsNonRoot: true,
|
|
runAsUser: 10001,
|
|
runAsGroup: 10001,
|
|
fsGroup: 10001,
|
|
fsGroupChangePolicy: 'OnRootMismatch',
|
|
seccompProfile: { type: 'RuntimeDefault' },
|
|
},
|
|
affinity: {
|
|
podAntiAffinity: {
|
|
requiredDuringSchedulingIgnoredDuringExecution: [{
|
|
topologyKey: 'kubernetes.io/hostname',
|
|
labelSelector: { matchLabels: labels },
|
|
}],
|
|
},
|
|
},
|
|
containers: [{
|
|
name: 'cluster-control',
|
|
image: controlImage,
|
|
imagePullPolicy: 'Never',
|
|
terminationMessagePolicy: 'FallbackToLogsOnError',
|
|
securityContext: {
|
|
allowPrivilegeEscalation: false,
|
|
readOnlyRootFilesystem: true,
|
|
capabilities: { drop: ['ALL'] },
|
|
},
|
|
env: [
|
|
{ name: 'QL_DEPLOYMENT_PROFILE', value: 'cluster-control' },
|
|
{ name: 'QL3_CLUSTER_CONTROL_ENABLED', value: 'true' },
|
|
{ name: 'QL3_CLUSTER_HTTP_HOST', value: '0.0.0.0' },
|
|
{ name: 'QL3_CLUSTER_HTTP_PORT', value: '5800' },
|
|
{ name: 'QL3_CLUSTER_HTTP_DRAIN_TIMEOUT_MS', value: '10000' },
|
|
{ name: 'QL3_WORKER_INGRESS_ENABLED', value: 'false' },
|
|
{ name: 'QL3_POSTGRES_TLS_MODE', value: 'verify-full' },
|
|
{
|
|
name: 'QL3_POSTGRES_TLS_CA_FILE',
|
|
value: '/var/run/secrets/qinglong3/runtime/ca.crt',
|
|
},
|
|
{
|
|
name: 'QL3_POSTGRES_TLS_SERVERNAME',
|
|
value: 'ql3-postgres-rw.qinglong3-system.svc',
|
|
},
|
|
{ name: 'QL3_POSTGRES_MAX_CONNECTIONS', value: '2' },
|
|
{
|
|
name: 'QL3_POSTGRES_APPLICATION_NAME',
|
|
value: 'qinglong3-security-live-control',
|
|
},
|
|
{
|
|
name: 'QL3_CLUSTER_REPLICA_ID',
|
|
valueFrom: {
|
|
fieldRef: {
|
|
apiVersion: 'v1',
|
|
fieldPath: 'metadata.name',
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: 'QL3_POSTGRES_RUNTIME_URL',
|
|
valueFrom: {
|
|
secretKeyRef: {
|
|
name: CONTROL_RUNTIME_SECRET,
|
|
key: 'postgres-runtime-url',
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE',
|
|
value: '/var/run/secrets/qinglong3/runtime/keyring.json',
|
|
},
|
|
],
|
|
ports: [{ name: 'http', containerPort: 5800 }],
|
|
startupProbe: {
|
|
httpGet: { path: '/livez', port: 'http' },
|
|
periodSeconds: 2,
|
|
timeoutSeconds: 1,
|
|
failureThreshold: 60,
|
|
},
|
|
readinessProbe: {
|
|
httpGet: { path: '/readyz', port: 'http' },
|
|
periodSeconds: 2,
|
|
timeoutSeconds: 1,
|
|
failureThreshold: 10,
|
|
},
|
|
resources: {
|
|
requests: { cpu: '25m', memory: '96Mi' },
|
|
limits: { cpu: '500m', memory: '256Mi' },
|
|
},
|
|
volumeMounts: [
|
|
{ name: 'tmp', mountPath: '/tmp' },
|
|
{
|
|
name: 'runtime-private',
|
|
mountPath: '/var/run/secrets/qinglong3/runtime',
|
|
readOnly: true,
|
|
},
|
|
],
|
|
}],
|
|
initContainers: [{
|
|
name: 'materialize-runtime-files',
|
|
image: controlImage,
|
|
imagePullPolicy: 'Never',
|
|
command: [
|
|
'node',
|
|
'-e',
|
|
runtimeFileMaterializationSource(),
|
|
],
|
|
securityContext: {
|
|
allowPrivilegeEscalation: false,
|
|
readOnlyRootFilesystem: true,
|
|
capabilities: { drop: ['ALL'] },
|
|
},
|
|
resources: {
|
|
requests: { cpu: '10m', memory: '32Mi' },
|
|
limits: { cpu: '100m', memory: '64Mi' },
|
|
},
|
|
volumeMounts: [
|
|
{
|
|
name: 'postgres-ca-projected',
|
|
mountPath: '/var/run/secrets/qinglong3/postgres-projected',
|
|
readOnly: true,
|
|
},
|
|
{
|
|
name: 'api-credential-keyring-projected',
|
|
mountPath: '/var/run/secrets/qinglong3/api-credential-projected',
|
|
readOnly: true,
|
|
},
|
|
{
|
|
name: 'runtime-private',
|
|
mountPath: '/var/run/secrets/qinglong3/runtime',
|
|
},
|
|
],
|
|
}],
|
|
volumes: [
|
|
{
|
|
name: 'tmp',
|
|
emptyDir: { medium: 'Memory', sizeLimit: '16Mi' },
|
|
},
|
|
{
|
|
name: 'postgres-ca-projected',
|
|
secret: {
|
|
secretName: 'ql3-postgres-ca',
|
|
defaultMode: 292,
|
|
items: [{ key: 'ca.crt', path: 'ca.crt' }],
|
|
},
|
|
},
|
|
{
|
|
name: 'api-credential-keyring-projected',
|
|
secret: {
|
|
secretName: CONTROL_RUNTIME_SECRET,
|
|
defaultMode: 292,
|
|
items: [{
|
|
key: 'api-credential-pepper-keyring.json',
|
|
path: 'keyring.json',
|
|
}],
|
|
},
|
|
},
|
|
{
|
|
name: 'runtime-private',
|
|
emptyDir: { medium: 'Memory', sizeLimit: '1Mi' },
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
]);
|
|
}
|
|
|
|
function applyControlRuntimeSecret(fixture, runtimeDatabaseUrl, keyring) {
|
|
applySecret(fixture, CONTROL_RUNTIME_SECRET, 'Opaque', {
|
|
'postgres-runtime-url': runtimeDatabaseUrl,
|
|
'api-credential-pepper-keyring.json': `${JSON.stringify(keyring)}\n`,
|
|
});
|
|
}
|
|
|
|
function controlTerminationFact(message) {
|
|
if (typeof message !== 'string' || message.length < 1 || message.length > 4096) {
|
|
return 'rejected';
|
|
}
|
|
const lines = message.trim().split('\n');
|
|
try {
|
|
const fact = JSON.parse(lines.at(-1));
|
|
const keys = Object.keys(fact).sort();
|
|
const expected = [
|
|
'component',
|
|
'event',
|
|
'level',
|
|
'name',
|
|
'schemaVersion',
|
|
...(fact.code === undefined ? [] : ['code']),
|
|
].sort();
|
|
if (
|
|
JSON.stringify(keys) !== JSON.stringify(expected) ||
|
|
fact.schemaVersion !== 1 ||
|
|
fact.component !== 'qinglong3-cluster-control' ||
|
|
fact.level !== 'error' ||
|
|
fact.event !== 'process_failed' ||
|
|
typeof fact.name !== 'string' ||
|
|
!/^[A-Za-z][A-Za-z0-9]{0,127}$/.test(fact.name) ||
|
|
(fact.code !== undefined &&
|
|
(typeof fact.code !== 'string' ||
|
|
!/^[A-Z][A-Z0-9_]{0,127}$/.test(fact.code)))
|
|
) {
|
|
return 'rejected';
|
|
}
|
|
return Object.freeze({
|
|
name: fact.name,
|
|
...(fact.code === undefined ? {} : { code: fact.code }),
|
|
});
|
|
} catch {
|
|
return 'rejected';
|
|
}
|
|
}
|
|
|
|
function controlRolloutFailureEvidence(fixture) {
|
|
const deployment = fixture.kubectlJson([
|
|
'-n',
|
|
NAMESPACE,
|
|
'get',
|
|
'deployment',
|
|
CONTROL_NAME,
|
|
]);
|
|
const pods = fixture.kubectlJson([
|
|
'-n',
|
|
NAMESPACE,
|
|
'get',
|
|
'pods',
|
|
'-l',
|
|
`app.kubernetes.io/name=${CONTROL_NAME}`,
|
|
]).items;
|
|
return Object.freeze({
|
|
deployment: Object.freeze({
|
|
generation: deployment.metadata.generation ?? null,
|
|
observedGeneration: deployment.status?.observedGeneration ?? null,
|
|
replicas: deployment.status?.replicas ?? 0,
|
|
updatedReplicas: deployment.status?.updatedReplicas ?? 0,
|
|
availableReplicas: deployment.status?.availableReplicas ?? 0,
|
|
unavailableReplicas: deployment.status?.unavailableReplicas ?? 0,
|
|
conditions: Object.freeze(
|
|
(deployment.status?.conditions ?? []).map((condition) =>
|
|
Object.freeze({
|
|
type: condition.type,
|
|
status: condition.status,
|
|
reason: condition.reason ?? null,
|
|
}),
|
|
),
|
|
),
|
|
}),
|
|
pods: Object.freeze(
|
|
pods.map((pod) =>
|
|
Object.freeze({
|
|
name: pod.metadata.name,
|
|
node: pod.spec.nodeName ?? null,
|
|
phase: pod.status?.phase ?? null,
|
|
conditions: Object.freeze(
|
|
(pod.status?.conditions ?? []).map((condition) =>
|
|
Object.freeze({
|
|
type: condition.type,
|
|
status: condition.status,
|
|
reason: condition.reason ?? null,
|
|
}),
|
|
),
|
|
),
|
|
containers: Object.freeze(
|
|
(pod.status?.containerStatuses ?? []).map((container) => {
|
|
const terminated =
|
|
container.state?.terminated ??
|
|
container.lastState?.terminated;
|
|
return Object.freeze({
|
|
name: container.name,
|
|
ready: container.ready,
|
|
restartCount: container.restartCount,
|
|
state: container.state?.waiting
|
|
? Object.freeze({
|
|
kind: 'waiting',
|
|
reason: container.state.waiting.reason ?? null,
|
|
})
|
|
: container.state?.running
|
|
? Object.freeze({ kind: 'running' })
|
|
: terminated
|
|
? Object.freeze({
|
|
kind: 'terminated',
|
|
reason: terminated.reason ?? null,
|
|
exitCode: terminated.exitCode,
|
|
})
|
|
: Object.freeze({ kind: 'unknown' }),
|
|
failure:
|
|
terminated?.message === undefined
|
|
? null
|
|
: controlTerminationFact(terminated.message),
|
|
});
|
|
}),
|
|
),
|
|
}),
|
|
),
|
|
),
|
|
});
|
|
}
|
|
|
|
async function waitForControlRollout(fixture, restart) {
|
|
if (restart) {
|
|
fixture.kubectl([
|
|
'-n',
|
|
NAMESPACE,
|
|
'rollout',
|
|
'restart',
|
|
`deployment/${CONTROL_NAME}`,
|
|
]);
|
|
}
|
|
const rollout = fixture.kubectl(
|
|
[
|
|
'-n',
|
|
NAMESPACE,
|
|
'rollout',
|
|
'status',
|
|
`deployment/${CONTROL_NAME}`,
|
|
'--timeout=5m',
|
|
],
|
|
{ capture: true, quiet: true, allowFailure: true },
|
|
);
|
|
if (rollout.status !== 0) {
|
|
throw new Error(
|
|
`Cluster Control rollout unavailable: ${JSON.stringify(
|
|
controlRolloutFailureEvidence(fixture),
|
|
)}`,
|
|
);
|
|
}
|
|
return (
|
|
await waitFor('two ready Cluster Control replicas', 120_000, () => {
|
|
const deployment = fixture.kubectlJson([
|
|
'-n',
|
|
NAMESPACE,
|
|
'get',
|
|
'deployment',
|
|
CONTROL_NAME,
|
|
]);
|
|
const pods = fixture.kubectlJson([
|
|
'-n',
|
|
NAMESPACE,
|
|
'get',
|
|
'pods',
|
|
'-l',
|
|
`app.kubernetes.io/name=${CONTROL_NAME}`,
|
|
]).items;
|
|
const ready = pods.filter(podReady);
|
|
return deployment.status.availableReplicas === 2 && ready.length === 2
|
|
? {
|
|
ready: true,
|
|
value: Object.freeze({
|
|
replicas: ready.length,
|
|
nodes: Object.freeze(
|
|
ready.map((pod) => pod.spec.nodeName).sort(),
|
|
),
|
|
}),
|
|
}
|
|
: {
|
|
ready: false,
|
|
fact: JSON.stringify({
|
|
availableReplicas: deployment.status.availableReplicas ?? 0,
|
|
readyPods: ready.length,
|
|
}),
|
|
};
|
|
})
|
|
).value;
|
|
}
|
|
|
|
function credentialAuthenticationProbeSource() {
|
|
return String.raw`
|
|
const fs=require('node:fs');const http=require('node:http');
|
|
const finish=(value,status)=>{fs.writeFileSync('/dev/termination-log',JSON.stringify(value),{encoding:'utf8',mode:0o600});process.exitCode=status};
|
|
const sleep=(milliseconds)=>new Promise((resolve)=>setTimeout(resolve,milliseconds));
|
|
const request=(token,requestId)=>new Promise((resolve)=>{let settled=false;const done=(status)=>{if(settled)return;settled=true;resolve(status)};const call=http.get({host:'ql3-security-live-control',port:5800,path:'/api/v3/projects/prj_default/runs?limit=1',headers:{authorization:'Bearer '+token,'x-request-id':requestId}},(response)=>{const status=response.statusCode??null;response.resume();response.once('end',()=>done(status));response.once('error',()=>done(null))});call.setTimeout(5000,()=>{call.destroy();done(null)});call.once('error',()=>done(null))});
|
|
(async()=>{let bytes;try{const file='/delivery/private/'+process.argv[1];const expected=Number(process.argv[2]);const requestId=process.argv[3];const status=fs.lstatSync(file);if(!status.isFile()||status.isSymbolicLink()||(status.mode&0o777)!==0o600||status.uid!==process.geteuid()||status.size<1||status.size>32768)throw new Error('FILE');bytes=fs.readFileSync(file);const delivery=JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes));if(typeof delivery.token!=='string'||!/^ql3c_[A-Za-z0-9_-]+_[A-Za-z0-9_-]{43}$/.test(delivery.token)||!Number.isSafeInteger(delivery.notBeforeAtMs)||!Number.isSafeInteger(delivery.expiresAtMs))throw new Error('SCHEMA');const wait=Math.max(0,delivery.notBeforeAtMs-Date.now()+1000);if(wait>360000||Date.now()+wait>=delivery.expiresAtMs)throw new Error('LIFETIME');if(wait>0)await sleep(wait);const observed=await request(delivery.token,requestId);if(observed!==expected)throw Object.assign(new Error('STATUS'),{observed});finish({schemaVersion:1,passed:true,expectedStatus:expected,observedStatus:observed},0)}catch(error){finish({schemaVersion:1,passed:false,code:error instanceof Error&&/^[A-Z]+$/.test(error.message)?error.message:'UNAVAILABLE',observedStatus:Number.isInteger(error?.observed)?error.observed:null},1)}finally{bytes?.fill(0)}})();`;
|
|
}
|
|
|
|
async function runCredentialAuthenticationProbe({
|
|
fixture,
|
|
adminImage,
|
|
name,
|
|
deliveryFile,
|
|
expectedStatus,
|
|
requestId,
|
|
createdEvidenceJobs,
|
|
}) {
|
|
fixture.create({
|
|
apiVersion: 'batch/v1',
|
|
kind: 'Job',
|
|
metadata: { name, namespace: NAMESPACE },
|
|
spec: {
|
|
backoffLimit: 0,
|
|
activeDeadlineSeconds: 420,
|
|
ttlSecondsAfterFinished: 600,
|
|
template: {
|
|
metadata: {
|
|
labels: {
|
|
'app.kubernetes.io/name': NAME,
|
|
'app.kubernetes.io/component': 'credential-authentication-evidence',
|
|
'ql3.live/evidence': 'credential-authentication',
|
|
},
|
|
},
|
|
spec: {
|
|
serviceAccountName: NAME,
|
|
automountServiceAccountToken: false,
|
|
enableServiceLinks: false,
|
|
restartPolicy: 'Never',
|
|
securityContext: {
|
|
runAsNonRoot: true,
|
|
runAsUser: 10001,
|
|
runAsGroup: 10001,
|
|
fsGroup: 10001,
|
|
fsGroupChangePolicy: 'OnRootMismatch',
|
|
seccompProfile: { type: 'RuntimeDefault' },
|
|
},
|
|
containers: [{
|
|
name: 'authentication-evidence',
|
|
image: adminImage,
|
|
imagePullPolicy: 'Never',
|
|
command: [
|
|
'node',
|
|
'-e',
|
|
credentialAuthenticationProbeSource(),
|
|
deliveryFile,
|
|
String(expectedStatus),
|
|
requestId,
|
|
],
|
|
terminationMessagePolicy: 'File',
|
|
securityContext: {
|
|
allowPrivilegeEscalation: false,
|
|
readOnlyRootFilesystem: true,
|
|
capabilities: { drop: ['ALL'] },
|
|
},
|
|
resources: {
|
|
requests: { cpu: '5m', memory: '16Mi' },
|
|
limits: { cpu: '100m', memory: '64Mi' },
|
|
},
|
|
volumeMounts: [{
|
|
name: 'delivery',
|
|
mountPath: '/delivery',
|
|
readOnly: true,
|
|
}],
|
|
}],
|
|
volumes: [{
|
|
name: 'delivery',
|
|
persistentVolumeClaim: { claimName: DELIVERY_CLAIM },
|
|
}],
|
|
},
|
|
},
|
|
},
|
|
});
|
|
createdEvidenceJobs.add(name);
|
|
try {
|
|
const snapshot = await terminalJobSnapshot(fixture, name, 480_000);
|
|
const state = snapshot.pod.status.containerStatuses?.[0]?.state?.terminated;
|
|
const statusEvidence = state?.message || JSON.stringify({
|
|
exitCode: state?.exitCode ?? null,
|
|
reason: state?.reason ?? null,
|
|
});
|
|
assert.equal(snapshot.complete, true, statusEvidence);
|
|
assert.equal(snapshot.failed, false, statusEvidence);
|
|
assert.equal(state?.exitCode, 0, state?.message);
|
|
const evidence = JSON.parse(state.message);
|
|
assert.deepEqual(evidence, {
|
|
schemaVersion: 1,
|
|
passed: true,
|
|
expectedStatus,
|
|
observedStatus: expectedStatus,
|
|
});
|
|
return evidence;
|
|
} finally {
|
|
deleteResource(fixture, `job/${name}`);
|
|
createdEvidenceJobs.delete(name);
|
|
}
|
|
}
|
|
|
|
async function provisionDeliveryVolume({
|
|
fixture,
|
|
adminImage,
|
|
createdEvidenceJobs,
|
|
}) {
|
|
const name = 'ql3-security-live-delivery-provision';
|
|
fixture.create({
|
|
apiVersion: 'batch/v1',
|
|
kind: 'Job',
|
|
metadata: {
|
|
name,
|
|
namespace: NAMESPACE,
|
|
labels: {
|
|
'app.kubernetes.io/name': NAME,
|
|
'app.kubernetes.io/component': 'security-administration',
|
|
'app.kubernetes.io/part-of': 'qinglong3',
|
|
'ql3.live/fixture-role': 'storage-provision',
|
|
},
|
|
},
|
|
spec: {
|
|
backoffLimit: 0,
|
|
activeDeadlineSeconds: 120,
|
|
ttlSecondsAfterFinished: 600,
|
|
template: {
|
|
metadata: {
|
|
labels: {
|
|
'app.kubernetes.io/name': NAME,
|
|
'app.kubernetes.io/component': 'security-administration',
|
|
'app.kubernetes.io/part-of': 'qinglong3',
|
|
'ql3.live/fixture-role': 'storage-provision',
|
|
},
|
|
},
|
|
spec: {
|
|
serviceAccountName: NAME,
|
|
automountServiceAccountToken: false,
|
|
enableServiceLinks: false,
|
|
restartPolicy: 'Never',
|
|
securityContext: {
|
|
runAsUser: 0,
|
|
runAsGroup: 0,
|
|
fsGroup: 10001,
|
|
seccompProfile: { type: 'RuntimeDefault' },
|
|
},
|
|
containers: [
|
|
{
|
|
name: 'storage-provision',
|
|
image: adminImage,
|
|
imagePullPolicy: 'Never',
|
|
command: ['node', '-e', deliveryVolumeProvisionSource()],
|
|
terminationMessagePolicy: 'File',
|
|
securityContext: {
|
|
allowPrivilegeEscalation: false,
|
|
readOnlyRootFilesystem: true,
|
|
capabilities: { drop: ['ALL'] },
|
|
},
|
|
resources: {
|
|
requests: { cpu: '5m', memory: '16Mi' },
|
|
limits: { cpu: '100m', memory: '64Mi' },
|
|
},
|
|
volumeMounts: [{ name: 'delivery', mountPath: '/delivery' }],
|
|
},
|
|
],
|
|
volumes: [
|
|
{
|
|
name: 'delivery',
|
|
persistentVolumeClaim: { claimName: DELIVERY_CLAIM },
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
});
|
|
createdEvidenceJobs.add(name);
|
|
try {
|
|
const snapshot = await terminalJobSnapshot(fixture, name, 180_000);
|
|
assert.equal(snapshot.complete, true);
|
|
assert.equal(snapshot.failed, false);
|
|
const state = snapshot.pod.status.containerStatuses?.[0]?.state?.terminated;
|
|
assert.equal(state?.exitCode, 0, state?.message);
|
|
const evidence = JSON.parse(state.message);
|
|
assert.deepEqual(evidence, {
|
|
schema: 'qinglong/security-administration-delivery-volume-provision@v1',
|
|
beforeMode: '2777',
|
|
afterMode: '2770',
|
|
uid: 0,
|
|
gid: 10001,
|
|
passed: true,
|
|
});
|
|
return evidence;
|
|
} finally {
|
|
deleteResource(fixture, `job/${name}`);
|
|
createdEvidenceJobs.delete(name);
|
|
}
|
|
}
|
|
|
|
function canI(fixture, verb, resource) {
|
|
const result = fixture.kubectl(
|
|
[
|
|
'auth',
|
|
'can-i',
|
|
verb,
|
|
resource,
|
|
'-n',
|
|
NAMESPACE,
|
|
`--as=system:serviceaccount:${NAMESPACE}:${NAME}`,
|
|
],
|
|
{ capture: true, quiet: true, allowFailure: true },
|
|
);
|
|
const decision = result.stdout.trim();
|
|
assert.ok(decision === 'yes' || decision === 'no');
|
|
assert.equal(result.status === 0, decision === 'yes');
|
|
return decision;
|
|
}
|
|
|
|
async function main(argv = process.argv.slice(2)) {
|
|
const reportFile = privateReportPath(argv);
|
|
if (process.env.QL3_SECURITY_ADMINISTRATION_KUBERNETES_LIVE !== '1') {
|
|
throw new Error(
|
|
'Refusing to mutate Docker/Kubernetes without QL3_SECURITY_ADMINISTRATION_KUBERNETES_LIVE=1',
|
|
);
|
|
}
|
|
const operatorManifestFile = process.env.QL3_CNPG_OPERATOR_MANIFEST_FILE;
|
|
if (!operatorManifestFile) {
|
|
throw new Error('QL3_CNPG_OPERATOR_MANIFEST_FILE is required');
|
|
}
|
|
const reviewedManifest = reviewedOperatorManifest(operatorManifestFile);
|
|
const fixture = new K3sDockerLiveFixture({ prefix: 'ql3-security-live' });
|
|
const suffix =
|
|
process.pid.toString(36) + '-' + crypto.randomBytes(3).toString('hex');
|
|
const adminImage = `${ADMIN_IMAGE_BASE}:${suffix}`;
|
|
const controlImage = `${CONTROL_IMAGE_BASE}:${suffix}`;
|
|
let adminImageBuilt = false;
|
|
let controlImageBuilt = false;
|
|
const createdJobs = new Set();
|
|
const createdSecrets = new Set();
|
|
const createdEvidenceJobs = new Set();
|
|
try {
|
|
const nodes = await fixture.start();
|
|
for (const node of fixture.nodes) {
|
|
for (const setting of [
|
|
'net.ipv4.ip_forward=1',
|
|
'net.bridge.bridge-nf-call-iptables=1',
|
|
]) {
|
|
const configured = fixture.dockerRun(
|
|
['exec', node, 'sysctl', '-w', setting],
|
|
{ capture: true, quiet: true },
|
|
).stdout;
|
|
assert.equal(configured.trim().endsWith(' = 1'), true);
|
|
}
|
|
}
|
|
const architecture = fixture.inspectImage(fixture.k3sImage).Architecture;
|
|
assert.ok(['amd64', 'arm64'].includes(architecture));
|
|
for (const reviewedImage of [OPERATOR_IMAGE, POSTGRES_IMAGE]) {
|
|
run(fixture.docker, ['pull', reviewedImage]);
|
|
const inspected = fixture.inspectImage(reviewedImage);
|
|
assert.ok(
|
|
inspected.RepoDigests?.some((entry) =>
|
|
entry.endsWith(`@${imageDigest(reviewedImage)}`),
|
|
),
|
|
);
|
|
const preloadTag = imageTag(reviewedImage);
|
|
run(fixture.docker, ['tag', reviewedImage, preloadTag]);
|
|
fixture.loadImage(preloadTag, `${path.basename(preloadTag)}.tar`);
|
|
}
|
|
|
|
const sourceRevision = run('git', ['rev-parse', 'HEAD'], {
|
|
capture: true,
|
|
quiet: true,
|
|
}).stdout;
|
|
for (const [dockerfile, image, archive] of [
|
|
[
|
|
'deploy/containers/ql3-cluster-admin/Dockerfile',
|
|
adminImage,
|
|
'security-administration-admin.tar',
|
|
],
|
|
[
|
|
'deploy/containers/ql3-cluster-control/Dockerfile',
|
|
controlImage,
|
|
'security-administration-control.tar',
|
|
],
|
|
]) {
|
|
run(fixture.docker, [
|
|
'build',
|
|
'--file',
|
|
dockerfile,
|
|
'--tag',
|
|
image,
|
|
'--build-arg',
|
|
`SOURCE_REVISION=${sourceRevision}`,
|
|
'.',
|
|
]);
|
|
if (image === adminImage) adminImageBuilt = true;
|
|
else controlImageBuilt = true;
|
|
fixture.loadImage(image, archive);
|
|
}
|
|
const adminImageInfo = fixture.inspectImage(adminImage);
|
|
const controlImageInfo = fixture.inspectImage(controlImage);
|
|
const postgresImageInfo = fixture.inspectImage(POSTGRES_IMAGE);
|
|
const k3sImageInfo = fixture.inspectImage(fixture.k3sImage);
|
|
|
|
fixture.kubectl(['apply', '--server-side', '-f', reviewedManifest]);
|
|
fixture.kubectl([
|
|
'-n',
|
|
'cnpg-system',
|
|
'set',
|
|
'image',
|
|
'deployment/cnpg-controller-manager',
|
|
`manager=${imageTag(OPERATOR_IMAGE)}`,
|
|
]);
|
|
fixture.kubectl([
|
|
'wait',
|
|
'--for=condition=Established',
|
|
'crd/clusters.postgresql.cnpg.io',
|
|
'crd/databaseroles.postgresql.cnpg.io',
|
|
'crd/databases.postgresql.cnpg.io',
|
|
'--timeout=5m',
|
|
]);
|
|
fixture.kubectl([
|
|
'-n',
|
|
'cnpg-system',
|
|
'rollout',
|
|
'status',
|
|
'deployment/cnpg-controller-manager',
|
|
'--timeout=5m',
|
|
]);
|
|
|
|
fixture.kubectl([
|
|
'apply',
|
|
'-f',
|
|
'deploy/kubernetes/ql3-cluster/base/namespace.yaml',
|
|
]);
|
|
fixture.kubectl([
|
|
'-n',
|
|
NAMESPACE,
|
|
'apply',
|
|
'-f',
|
|
'deploy/kubernetes/ql3-cluster/base/service-account.yaml',
|
|
]);
|
|
const passwords = Object.fromEntries(
|
|
ROLE_NAMES.map((role) => [role, randomSecret()]),
|
|
);
|
|
for (const role of ROLE_NAMES) {
|
|
applySecret(
|
|
fixture,
|
|
`ql3-postgres-${role
|
|
.replace(/^ql3_/, '')
|
|
.replaceAll('_', '-')}-auth`,
|
|
'kubernetes.io/basic-auth',
|
|
{ username: role, password: passwords[role] },
|
|
);
|
|
}
|
|
const databaseManifest = fixture
|
|
.kubectl(
|
|
['kustomize', 'deploy/kubernetes/ql3-cluster/operators/cloudnative-pg'],
|
|
{ capture: true, quiet: true },
|
|
)
|
|
.stdout.replace(POSTGRES_IMAGE, imageTag(POSTGRES_IMAGE));
|
|
assert.equal(databaseManifest.includes(POSTGRES_IMAGE), false);
|
|
fixture.kubectl(['apply', '-f', '-'], { input: `${databaseManifest}\n` });
|
|
fixture.kubectl([
|
|
'-n',
|
|
NAMESPACE,
|
|
'wait',
|
|
'--for=condition=Ready',
|
|
`cluster/${POSTGRES_CLUSTER}`,
|
|
'--timeout=20m',
|
|
]);
|
|
const databasePods = (
|
|
await waitFor('three ready CloudNativePG instances', 600_000, () => {
|
|
const pods = fixture
|
|
.kubectlJson([
|
|
'-n',
|
|
NAMESPACE,
|
|
'get',
|
|
'pods',
|
|
'-l',
|
|
`cnpg.io/cluster=${POSTGRES_CLUSTER}`,
|
|
])
|
|
.items.filter(podReady);
|
|
return pods.length === 3
|
|
? { ready: true, value: pods }
|
|
: { ready: false, fact: `${pods.length}/3 Ready database Pods` };
|
|
})
|
|
).value;
|
|
await waitFor('CloudNativePG managed resources', 5 * 60_000, () => {
|
|
const roles = fixture.kubectlJson([
|
|
'-n',
|
|
NAMESPACE,
|
|
'get',
|
|
'databaseroles.postgresql.cnpg.io',
|
|
]).items;
|
|
const readyRoles = roles.filter(
|
|
(role) =>
|
|
role.status?.applied === true &&
|
|
role.status?.observedGeneration === role.metadata.generation,
|
|
);
|
|
const database = fixture.kubectlJson([
|
|
'-n',
|
|
NAMESPACE,
|
|
'get',
|
|
'database.postgresql.cnpg.io/ql3-postgres-qinglong',
|
|
]);
|
|
const databaseReady =
|
|
database.status?.applied === true &&
|
|
database.status?.observedGeneration === database.metadata.generation;
|
|
return readyRoles.length === ROLE_NAMES.length && databaseReady
|
|
? { ready: true, value: true }
|
|
: {
|
|
ready: false,
|
|
fact: `${readyRoles.length}/${ROLE_NAMES.length} roles applied; ` +
|
|
`databaseApplied=${String(databaseReady)}`,
|
|
};
|
|
});
|
|
|
|
const migrationManifest = localManifest(
|
|
fixture.kubectl(
|
|
[
|
|
'kustomize',
|
|
'deploy/kubernetes/ql3-cluster/operations/cloudnative-pg',
|
|
],
|
|
{ capture: true, quiet: true },
|
|
).stdout,
|
|
'registry.example.com/qinglong/qinglong3-cluster-control',
|
|
controlImage,
|
|
);
|
|
const migrationJob = yaml.load(migrationManifest);
|
|
assert.equal(migrationJob?.kind, 'Job');
|
|
const migrationContainer = findNamed(
|
|
migrationJob.spec.template.spec.containers,
|
|
'migration',
|
|
);
|
|
migrationContainer.terminationMessagePolicy = 'FallbackToLogsOnError';
|
|
fixture.create(migrationJob);
|
|
const migrationSnapshot = await terminalJobSnapshot(
|
|
fixture,
|
|
'ql3-cluster-migration',
|
|
10 * 60_000,
|
|
);
|
|
const migrationEvidence = JSON.stringify(
|
|
migrationFailureEvidence(migrationSnapshot),
|
|
);
|
|
assert.equal(migrationSnapshot.complete, true, migrationEvidence);
|
|
assert.equal(migrationSnapshot.failed, false, migrationEvidence);
|
|
const primary = currentPrimaryPod(fixture);
|
|
const migrationState = JSON.parse(
|
|
psql(
|
|
fixture,
|
|
primary.metadata.name,
|
|
[
|
|
'SELECT json_build_object(',
|
|
` 'migrationCount', (SELECT count(*)::integer FROM "ql3"."schema_migrations"),`,
|
|
` 'controlCoreCapability', (SELECT contract_version::integer FROM "ql3"."schema_capabilities" WHERE contract_name = 'control-core'),`,
|
|
` 'postgresVersionNumber', current_setting('server_version_num')::integer)`,
|
|
].join('\n'),
|
|
),
|
|
);
|
|
assert.deepEqual(migrationState, {
|
|
migrationCount: 71,
|
|
controlCoreCapability: 70,
|
|
postgresVersionNumber: 180004,
|
|
});
|
|
|
|
const baseResources = renderedResources(
|
|
fixture,
|
|
'deploy/kubernetes/ql3-cluster/operations/security-administration/cloudnative-pg',
|
|
adminImage,
|
|
);
|
|
const deliveryResources = renderedResources(
|
|
fixture,
|
|
'deploy/kubernetes/ql3-cluster/operations/security-administration/cloudnative-pg-credential-delivery',
|
|
adminImage,
|
|
);
|
|
const serviceAccount = namedResource(baseResources, 'ServiceAccount', NAME);
|
|
const networkPolicy = namedResource(baseResources, 'NetworkPolicy', NAME);
|
|
const baseTemplate = namedResource(baseResources, 'Job', NAME);
|
|
const deliveryTemplate = namedResource(deliveryResources, 'Job', NAME);
|
|
fixture.apply(serviceAccount);
|
|
fixture.apply(networkPolicy);
|
|
fixture.create({
|
|
apiVersion: 'v1',
|
|
kind: 'PersistentVolumeClaim',
|
|
metadata: { name: DELIVERY_CLAIM, namespace: NAMESPACE },
|
|
spec: {
|
|
accessModes: ['ReadWriteOnce'],
|
|
resources: { requests: { storage: '16Mi' } },
|
|
},
|
|
});
|
|
const deliveryProvision = await provisionDeliveryVolume({
|
|
fixture,
|
|
adminImage,
|
|
createdEvidenceJobs,
|
|
});
|
|
|
|
const values = Object.freeze({
|
|
subject: Object.freeze({ type: 'api_app', id: `d406-live-${suffix}` }),
|
|
identityMutationId: crypto.randomUUID(),
|
|
registerRequestId: `d406-register-${suffix}`,
|
|
});
|
|
const oldCredential = Object.freeze({
|
|
subject: values.subject,
|
|
credentialId: `d407-old-${suffix}`,
|
|
issueMutationId: crypto.randomUUID(),
|
|
revokeMutationId: crypto.randomUUID(),
|
|
issueRequestId: `d407-old-issue-${suffix}`,
|
|
revokeRequestId: `d407-old-revoke-${suffix}`,
|
|
});
|
|
const newCredential = Object.freeze({
|
|
subject: values.subject,
|
|
credentialId: `d407-new-${suffix}`,
|
|
issueMutationId: crypto.randomUUID(),
|
|
rotateMutationId: crypto.randomUUID(),
|
|
issueRequestId: `d407-new-issue-${suffix}`,
|
|
rotateRequestId: `d407-new-rotate-${suffix}`,
|
|
});
|
|
const authenticationRequestIds = Object.freeze({
|
|
oldBeforeActivation: `d407-auth-old-before-${suffix}`,
|
|
oldDuringOverlap: `d407-auth-old-overlap-${suffix}`,
|
|
newDuringOverlap: `d407-auth-new-overlap-${suffix}`,
|
|
oldAfterConvergence: `d407-auth-old-contracted-${suffix}`,
|
|
newAfterContraction: `d407-auth-new-contracted-${suffix}`,
|
|
});
|
|
const key = identity.reviewedKey('security-administration-live-key-1');
|
|
const keyset = identity.keyset(1, [key]);
|
|
const oldPepperKeyId = 'd407-old';
|
|
const newPepperKeyId = 'd407-new';
|
|
const oldPepper = randomSecret();
|
|
const newPepper = randomSecret();
|
|
const overlapOldActive = apiCredentialPepperKeyring(oldPepperKeyId, [
|
|
{ pepperKeyId: oldPepperKeyId, pepper: oldPepper },
|
|
{ pepperKeyId: newPepperKeyId, pepper: newPepper },
|
|
]);
|
|
const overlapNewActive = apiCredentialPepperKeyring(newPepperKeyId, [
|
|
{ pepperKeyId: oldPepperKeyId, pepper: oldPepper },
|
|
{ pepperKeyId: newPepperKeyId, pepper: newPepper },
|
|
]);
|
|
const contractedNew = apiCredentialPepperKeyring(newPepperKeyId, [
|
|
{ pepperKeyId: newPepperKeyId, pepper: newPepper },
|
|
]);
|
|
const register = identityRegisterCommand(values);
|
|
const revokeOld = credentialRevokeCommand({
|
|
...oldCredential,
|
|
expectedCurrentVersion: 1,
|
|
});
|
|
const runtimeDatabaseUrl =
|
|
`postgresql://ql3_runtime:${passwords.ql3_runtime}` +
|
|
'@ql3-postgres-rw.qinglong3-system.svc:5432/qinglong';
|
|
applyControlRuntimeSecret(
|
|
fixture,
|
|
runtimeDatabaseUrl,
|
|
overlapOldActive,
|
|
);
|
|
for (const resource of clusterControlResources(controlImage)) {
|
|
fixture.apply(resource);
|
|
}
|
|
const controlBeforeActivation = await waitForControlRollout(
|
|
fixture,
|
|
false,
|
|
);
|
|
assert.equal(new Set(controlBeforeActivation.nodes).size, 2);
|
|
|
|
await runAdministrationJob({
|
|
fixture,
|
|
template: baseTemplate,
|
|
name: 'ql3-security-live-invalid-input',
|
|
command: register,
|
|
assertion: identity.assertion(key, 'invalid-input'),
|
|
keyset,
|
|
pepperKeyring: overlapOldActive,
|
|
expectedComplete: false,
|
|
projectedMode: 0o444,
|
|
createdJobs,
|
|
createdSecrets,
|
|
});
|
|
await runAdministrationJob({
|
|
fixture,
|
|
template: baseTemplate,
|
|
name: 'ql3-security-live-register',
|
|
command: register,
|
|
assertion: identity.assertion(key, 'register'),
|
|
keyset,
|
|
pepperKeyring: overlapOldActive,
|
|
expectedComplete: true,
|
|
createdJobs,
|
|
createdSecrets,
|
|
});
|
|
await runAdministrationJob({
|
|
fixture,
|
|
template: baseTemplate,
|
|
name: 'ql3-security-live-audit',
|
|
command: auditListCommand(),
|
|
assertion: identity.assertion(key, 'audit'),
|
|
keyset,
|
|
pepperKeyring: overlapOldActive,
|
|
expectedComplete: true,
|
|
createdJobs,
|
|
createdSecrets,
|
|
});
|
|
const oldIssueNotBeforeAtMs = Date.now() + 2 * 60 * 1000;
|
|
const issueOld = credentialIssueCommand({
|
|
...oldCredential,
|
|
notBeforeAtMs: oldIssueNotBeforeAtMs,
|
|
expiresAtMs: oldIssueNotBeforeAtMs + 60 * 60 * 1000,
|
|
});
|
|
const oldIssueAssertion = identity.assertion(
|
|
key,
|
|
'old-issue-response-loss',
|
|
);
|
|
await runAdministrationJob({
|
|
fixture,
|
|
template: deliveryTemplate,
|
|
name: 'ql3-security-live-old-issue',
|
|
command: issueOld,
|
|
assertion: oldIssueAssertion,
|
|
keyset,
|
|
pepperKeyring: overlapOldActive,
|
|
deliveryFile: 'old-issue.json',
|
|
expectedComplete: true,
|
|
createdJobs,
|
|
createdSecrets,
|
|
});
|
|
const kubernetesServiceIp = fixture.kubectlJson([
|
|
'-n',
|
|
'default',
|
|
'get',
|
|
'service',
|
|
'kubernetes',
|
|
]).spec.clusterIP;
|
|
const issuedEvidence = await runCustodyEvidence({
|
|
fixture,
|
|
adminImage,
|
|
name: 'ql3-security-live-evidence-issued',
|
|
expected: [{ name: 'old-issue.json', operation: 'credential.issue' }],
|
|
kubernetesServiceIp,
|
|
createdEvidenceJobs,
|
|
});
|
|
await runAdministrationJob({
|
|
fixture,
|
|
template: deliveryTemplate,
|
|
name: 'ql3-security-live-old-issue-replay',
|
|
command: issueOld,
|
|
assertion: oldIssueAssertion,
|
|
keyset,
|
|
pepperKeyring: overlapOldActive,
|
|
deliveryFile: 'old-issue.json',
|
|
expectedComplete: true,
|
|
createdJobs,
|
|
createdSecrets,
|
|
});
|
|
const replayEvidence = await runCustodyEvidence({
|
|
fixture,
|
|
adminImage,
|
|
name: 'ql3-security-live-evidence-replay',
|
|
expected: [{ name: 'old-issue.json', operation: 'credential.issue' }],
|
|
kubernetesServiceIp,
|
|
createdEvidenceJobs,
|
|
});
|
|
assert.deepEqual(replayEvidence.files, issuedEvidence.files);
|
|
const oldAuthenticationBeforeActivation =
|
|
await runCredentialAuthenticationProbe({
|
|
fixture,
|
|
adminImage,
|
|
name: 'ql3-security-live-auth-old-before-activate',
|
|
deliveryFile: 'old-issue.json',
|
|
expectedStatus: 403,
|
|
requestId: authenticationRequestIds.oldBeforeActivation,
|
|
createdEvidenceJobs,
|
|
});
|
|
await runAdministrationJob({
|
|
fixture,
|
|
template: baseTemplate,
|
|
name: 'ql3-security-live-old-references-before-activate',
|
|
command: pepperReferencesCommand(oldPepperKeyId),
|
|
assertion: identity.assertion(key, 'old-references-before-activate'),
|
|
keyset,
|
|
pepperKeyring: overlapOldActive,
|
|
expectedComplete: true,
|
|
createdJobs,
|
|
createdSecrets,
|
|
});
|
|
const referencesBeforeActivate = Number(
|
|
psql(
|
|
fixture,
|
|
currentPrimaryPod(fixture).metadata.name,
|
|
`SELECT count(*)::integer FROM "ql3"."api_credentials" current_record WHERE credential_id = ${sqlString(
|
|
oldCredential.credentialId,
|
|
)} AND pepper_key_id = ${sqlString(
|
|
oldPepperKeyId,
|
|
)} AND state = 'active' AND expires_at_ms > (extract(epoch FROM statement_timestamp()) * 1000)::bigint AND NOT EXISTS (SELECT 1 FROM "ql3"."api_credentials" newer WHERE newer.credential_id = current_record.credential_id AND newer.version > current_record.version)`,
|
|
),
|
|
);
|
|
assert.equal(referencesBeforeActivate, 1);
|
|
applyControlRuntimeSecret(
|
|
fixture,
|
|
runtimeDatabaseUrl,
|
|
overlapNewActive,
|
|
);
|
|
const controlDuringOverlap = await waitForControlRollout(fixture, true);
|
|
assert.equal(new Set(controlDuringOverlap.nodes).size, 2);
|
|
const oldAuthenticationDuringOverlap =
|
|
await runCredentialAuthenticationProbe({
|
|
fixture,
|
|
adminImage,
|
|
name: 'ql3-security-live-auth-old-overlap',
|
|
deliveryFile: 'old-issue.json',
|
|
expectedStatus: 403,
|
|
requestId: authenticationRequestIds.oldDuringOverlap,
|
|
createdEvidenceJobs,
|
|
});
|
|
const newIssueNotBeforeAtMs = Date.now() + 2 * 60 * 1000;
|
|
const issueNew = credentialIssueCommand({
|
|
...newCredential,
|
|
notBeforeAtMs: newIssueNotBeforeAtMs,
|
|
expiresAtMs: newIssueNotBeforeAtMs + 60 * 60 * 1000,
|
|
});
|
|
await runAdministrationJob({
|
|
fixture,
|
|
template: deliveryTemplate,
|
|
name: 'ql3-security-live-new-issue',
|
|
command: issueNew,
|
|
assertion: identity.assertion(key, 'new-issue'),
|
|
keyset,
|
|
pepperKeyring: overlapNewActive,
|
|
deliveryFile: 'new-issue.json',
|
|
expectedComplete: true,
|
|
createdJobs,
|
|
createdSecrets,
|
|
});
|
|
const newRotateNotBeforeAtMs = Date.now() + 2 * 60 * 1000;
|
|
const rotateNew = credentialRotateCommand({
|
|
...newCredential,
|
|
notBeforeAtMs: newRotateNotBeforeAtMs,
|
|
expiresAtMs: newRotateNotBeforeAtMs + 60 * 60 * 1000,
|
|
});
|
|
await runAdministrationJob({
|
|
fixture,
|
|
template: deliveryTemplate,
|
|
name: 'ql3-security-live-new-rotate',
|
|
command: rotateNew,
|
|
assertion: identity.assertion(key, 'new-rotate'),
|
|
keyset,
|
|
pepperKeyring: overlapNewActive,
|
|
deliveryFile: 'new-rotate.json',
|
|
expectedComplete: true,
|
|
createdJobs,
|
|
createdSecrets,
|
|
});
|
|
const newAuthenticationDuringOverlap =
|
|
await runCredentialAuthenticationProbe({
|
|
fixture,
|
|
adminImage,
|
|
name: 'ql3-security-live-auth-new-overlap',
|
|
deliveryFile: 'new-rotate.json',
|
|
expectedStatus: 403,
|
|
requestId: authenticationRequestIds.newDuringOverlap,
|
|
createdEvidenceJobs,
|
|
});
|
|
await runAdministrationJob({
|
|
fixture,
|
|
template: baseTemplate,
|
|
name: 'ql3-security-live-old-references-after-activate',
|
|
command: pepperReferencesCommand(oldPepperKeyId),
|
|
assertion: identity.assertion(key, 'old-references-after-activate'),
|
|
keyset,
|
|
pepperKeyring: overlapNewActive,
|
|
expectedComplete: true,
|
|
createdJobs,
|
|
createdSecrets,
|
|
});
|
|
const referencesAfterActivate = Number(
|
|
psql(
|
|
fixture,
|
|
currentPrimaryPod(fixture).metadata.name,
|
|
`SELECT count(*)::integer FROM "ql3"."api_credentials" current_record WHERE credential_id = ${sqlString(
|
|
oldCredential.credentialId,
|
|
)} AND pepper_key_id = ${sqlString(
|
|
oldPepperKeyId,
|
|
)} AND version = (SELECT max(version) FROM "ql3"."api_credentials" WHERE credential_id = current_record.credential_id) AND state = 'active' AND expires_at_ms > (extract(epoch FROM statement_timestamp()) * 1000)::bigint`,
|
|
),
|
|
);
|
|
assert.equal(referencesAfterActivate, 1);
|
|
await runAdministrationJob({
|
|
fixture,
|
|
template: baseTemplate,
|
|
name: 'ql3-security-live-old-revoke',
|
|
command: revokeOld,
|
|
assertion: identity.assertion(key, 'old-revoke'),
|
|
keyset,
|
|
pepperKeyring: overlapNewActive,
|
|
expectedComplete: true,
|
|
createdJobs,
|
|
createdSecrets,
|
|
});
|
|
await runAdministrationJob({
|
|
fixture,
|
|
template: baseTemplate,
|
|
name: 'ql3-security-live-old-references-after-converge',
|
|
command: pepperReferencesCommand(oldPepperKeyId),
|
|
assertion: identity.assertion(key, 'old-references-after-converge'),
|
|
keyset,
|
|
pepperKeyring: overlapNewActive,
|
|
expectedComplete: true,
|
|
createdJobs,
|
|
createdSecrets,
|
|
});
|
|
const referencesAfterConvergence = Number(
|
|
psql(
|
|
fixture,
|
|
currentPrimaryPod(fixture).metadata.name,
|
|
`SELECT count(*)::integer FROM "ql3"."api_credentials" current_record WHERE credential_id = ${sqlString(
|
|
oldCredential.credentialId,
|
|
)} AND pepper_key_id = ${sqlString(
|
|
oldPepperKeyId,
|
|
)} AND version = (SELECT max(version) FROM "ql3"."api_credentials" WHERE credential_id = current_record.credential_id) AND state = 'active' AND expires_at_ms > (extract(epoch FROM statement_timestamp()) * 1000)::bigint`,
|
|
),
|
|
);
|
|
assert.equal(referencesAfterConvergence, 0);
|
|
applyControlRuntimeSecret(fixture, runtimeDatabaseUrl, contractedNew);
|
|
const controlAfterContraction = await waitForControlRollout(fixture, true);
|
|
assert.equal(new Set(controlAfterContraction.nodes).size, 2);
|
|
const oldAuthenticationAfterConvergence =
|
|
await runCredentialAuthenticationProbe({
|
|
fixture,
|
|
adminImage,
|
|
name: 'ql3-security-live-auth-old-contracted',
|
|
deliveryFile: 'old-issue.json',
|
|
expectedStatus: 401,
|
|
requestId: authenticationRequestIds.oldAfterConvergence,
|
|
createdEvidenceJobs,
|
|
});
|
|
const newAuthenticationAfterContraction =
|
|
await runCredentialAuthenticationProbe({
|
|
fixture,
|
|
adminImage,
|
|
name: 'ql3-security-live-auth-new-contracted',
|
|
deliveryFile: 'new-rotate.json',
|
|
expectedStatus: 403,
|
|
requestId: authenticationRequestIds.newAfterContraction,
|
|
createdEvidenceJobs,
|
|
});
|
|
const finalEvidence = await runCustodyEvidence({
|
|
fixture,
|
|
adminImage,
|
|
name: 'ql3-security-live-evidence-final',
|
|
expected: [
|
|
{ name: 'old-issue.json', operation: 'credential.issue' },
|
|
{ name: 'new-issue.json', operation: 'credential.issue' },
|
|
{ name: 'new-rotate.json', operation: 'credential.rotate' },
|
|
],
|
|
kubernetesServiceIp,
|
|
createdEvidenceJobs,
|
|
});
|
|
assert.equal(finalEvidence.files.length, 3);
|
|
const issueFile = finalEvidence.files.find(
|
|
(file) => file.name === 'old-issue.json',
|
|
);
|
|
const rotationFile = finalEvidence.files.find(
|
|
(file) => file.name === 'new-rotate.json',
|
|
);
|
|
assert.ok(issueFile && rotationFile);
|
|
assert.notEqual(issueFile.digest, rotationFile.digest);
|
|
assert.deepEqual(issueFile, issuedEvidence.files[0]);
|
|
|
|
const finalPrimary = currentPrimaryPod(fixture);
|
|
const durable = JSON.parse(
|
|
psql(
|
|
fixture,
|
|
finalPrimary.metadata.name,
|
|
[
|
|
'SELECT json_build_object(',
|
|
` 'identityVersion', (SELECT max(version)::integer FROM "ql3"."identity_subjects" WHERE subject_type = ${sqlString(
|
|
values.subject.type,
|
|
)} AND subject_id = ${sqlString(values.subject.id)}),`,
|
|
` 'identityStatus', (SELECT status FROM "ql3"."identity_subjects" WHERE subject_type = ${sqlString(
|
|
values.subject.type,
|
|
)} AND subject_id = ${sqlString(values.subject.id)}),`,
|
|
` 'oldCredentialVersion', (SELECT max(version)::integer FROM "ql3"."api_credentials" WHERE credential_id = ${sqlString(
|
|
oldCredential.credentialId,
|
|
)}),`,
|
|
` 'oldCredentialState', (SELECT state FROM "ql3"."api_credentials" WHERE credential_id = ${sqlString(
|
|
oldCredential.credentialId,
|
|
)} ORDER BY version DESC LIMIT 1),`,
|
|
` 'newCredentialVersion', (SELECT max(version)::integer FROM "ql3"."api_credentials" WHERE credential_id = ${sqlString(
|
|
newCredential.credentialId,
|
|
)}),`,
|
|
` 'newCredentialState', (SELECT state FROM "ql3"."api_credentials" WHERE credential_id = ${sqlString(
|
|
newCredential.credentialId,
|
|
)} ORDER BY version DESC LIMIT 1),`,
|
|
` 'identityMutationCount', (SELECT count(*)::integer FROM "ql3"."identity_subject_mutations" WHERE subject_type = ${sqlString(
|
|
values.subject.type,
|
|
)} AND subject_id = ${sqlString(values.subject.id)}),`,
|
|
` 'credentialMutationCount', (SELECT count(*)::integer FROM "ql3"."api_credential_mutations" WHERE credential_id IN (${[
|
|
oldCredential.credentialId,
|
|
newCredential.credentialId,
|
|
].map(sqlString).join(',')})),`,
|
|
` 'issueMutationCount', (SELECT count(*)::integer FROM "ql3"."api_credential_mutations" WHERE mutation_id = ${sqlString(
|
|
oldCredential.issueMutationId,
|
|
)}::uuid),`,
|
|
` 'credentialVersionCount', (SELECT count(*)::integer FROM "ql3"."api_credentials" WHERE credential_id IN (${[
|
|
oldCredential.credentialId,
|
|
newCredential.credentialId,
|
|
].map(sqlString).join(',')})),`,
|
|
` 'oldGenerationVersionCount', (SELECT count(*)::integer FROM "ql3"."api_credentials" WHERE credential_id IN (${[
|
|
oldCredential.credentialId,
|
|
newCredential.credentialId,
|
|
].map(sqlString).join(',')}) AND pepper_key_id = ${sqlString(oldPepperKeyId)}),`,
|
|
` 'newGenerationVersionCount', (SELECT count(*)::integer FROM "ql3"."api_credentials" WHERE credential_id IN (${[
|
|
oldCredential.credentialId,
|
|
newCredential.credentialId,
|
|
].map(sqlString).join(',')}) AND pepper_key_id = ${sqlString(newPepperKeyId)}),`,
|
|
` 'latestGenerationsAreNew', (SELECT count(*) = 2 FROM "ql3"."api_credentials" current_record WHERE credential_id IN (${[
|
|
oldCredential.credentialId,
|
|
newCredential.credentialId,
|
|
].map(sqlString).join(',')}) AND version = (SELECT max(version) FROM "ql3"."api_credentials" WHERE credential_id = current_record.credential_id) AND pepper_key_id = ${sqlString(newPepperKeyId)}),`,
|
|
` 'allowedAuditCount', (SELECT count(*)::integer FROM "ql3"."security_audit_events" WHERE request_id IN (${[
|
|
values.registerRequestId,
|
|
oldCredential.issueRequestId,
|
|
newCredential.issueRequestId,
|
|
newCredential.rotateRequestId,
|
|
oldCredential.revokeRequestId,
|
|
]
|
|
.map(sqlString)
|
|
.join(',')}) AND outcome = 'allowed'),`,
|
|
` 'authenticationDeniedAuditCount', (SELECT count(*)::integer FROM "ql3"."security_audit_events" WHERE request_id IN (${[
|
|
authenticationRequestIds.oldBeforeActivation,
|
|
authenticationRequestIds.oldDuringOverlap,
|
|
authenticationRequestIds.newDuringOverlap,
|
|
authenticationRequestIds.newAfterContraction,
|
|
].map(sqlString).join(',')}) AND outcome = 'denied'),`,
|
|
` 'authenticationRejectedAuditCount', (SELECT count(*)::integer FROM "ql3"."security_audit_events" WHERE request_id = ${sqlString(
|
|
authenticationRequestIds.oldAfterConvergence,
|
|
)} AND outcome = 'authentication_rejected'))`,
|
|
].join('\n'),
|
|
),
|
|
);
|
|
assert.deepEqual(durable, {
|
|
identityVersion: 1,
|
|
identityStatus: 'active',
|
|
oldCredentialVersion: 2,
|
|
oldCredentialState: 'revoked',
|
|
newCredentialVersion: 2,
|
|
newCredentialState: 'active',
|
|
identityMutationCount: 1,
|
|
credentialMutationCount: 4,
|
|
issueMutationCount: 1,
|
|
credentialVersionCount: 4,
|
|
oldGenerationVersionCount: 1,
|
|
newGenerationVersionCount: 3,
|
|
latestGenerationsAreNew: true,
|
|
allowedAuditCount: 5,
|
|
authenticationDeniedAuditCount: 4,
|
|
authenticationRejectedAuditCount: 1,
|
|
});
|
|
const adminRole = JSON.parse(
|
|
psql(
|
|
fixture,
|
|
finalPrimary.metadata.name,
|
|
`SELECT json_build_object('login', rolcanlogin, 'superuser', rolsuper, 'createDatabase', rolcreatedb, 'createRole', rolcreaterole, 'replication', rolreplication, 'bypassRls', rolbypassrls, 'connectionLimit', rolconnlimit) FROM pg_roles WHERE rolname = 'ql3_admin'`,
|
|
),
|
|
);
|
|
assert.deepEqual(adminRole, {
|
|
login: true,
|
|
superuser: false,
|
|
createDatabase: false,
|
|
createRole: false,
|
|
replication: false,
|
|
bypassRls: false,
|
|
connectionLimit: 4,
|
|
});
|
|
assert.equal(canI(fixture, 'get', 'secrets'), 'no');
|
|
assert.equal(canI(fixture, 'create', 'jobs.batch'), 'no');
|
|
|
|
const finalCluster = fixture.kubectlJson([
|
|
'-n',
|
|
NAMESPACE,
|
|
'get',
|
|
'cluster',
|
|
POSTGRES_CLUSTER,
|
|
]);
|
|
const finalNodes = fixture.kubectlJson(['get', 'nodes']).items;
|
|
const readyNodes = finalNodes.filter(
|
|
(node) =>
|
|
podReady(node) &&
|
|
Array.isArray(node.spec.podCIDRs) &&
|
|
node.spec.podCIDRs.length === 1,
|
|
);
|
|
assert.equal(readyNodes.length, 3);
|
|
assert.equal(
|
|
new Set(readyNodes.map((node) => node.spec.podCIDRs[0])).size,
|
|
3,
|
|
);
|
|
const serverNode = readyNodes.find(
|
|
(node) => node.metadata.name === fixture.server,
|
|
);
|
|
assert.equal(
|
|
serverNode?.metadata.annotations?.[
|
|
'flannel.alpha.coreos.com/backend-type'
|
|
],
|
|
'vxlan',
|
|
);
|
|
|
|
assert.equal(createdJobs.size, 0);
|
|
assert.equal(createdSecrets.size, 0);
|
|
assert.equal(createdEvidenceJobs.size, 0);
|
|
deleteResource(fixture, `deployment/${CONTROL_NAME}`);
|
|
deleteResource(fixture, `service/${CONTROL_NAME}`);
|
|
deleteResource(fixture, `secret/${CONTROL_RUNTIME_SECRET}`);
|
|
deleteResource(fixture, `persistentvolumeclaim/${DELIVERY_CLAIM}`);
|
|
await waitFor('delivery PVC deletion', 60_000, () =>
|
|
resourceAbsent(
|
|
fixture,
|
|
`persistentvolumeclaim/${DELIVERY_CLAIM}`,
|
|
)
|
|
? { ready: true, value: true }
|
|
: { ready: false, fact: 'PVC still exists' },
|
|
);
|
|
const cleanup = {
|
|
jobsDeleted:
|
|
resourceAbsent(fixture, 'job/ql3-security-live-register') &&
|
|
resourceAbsent(fixture, 'job/ql3-security-live-revoke'),
|
|
inputSecretsDeleted:
|
|
resourceAbsent(
|
|
fixture,
|
|
'secret/ql3-security-live-register-input',
|
|
) &&
|
|
resourceAbsent(
|
|
fixture,
|
|
'secret/ql3-security-live-revoke-input',
|
|
),
|
|
evidenceJobsDeleted: resourceAbsent(
|
|
fixture,
|
|
'job/ql3-security-live-evidence-final',
|
|
),
|
|
storageProvisionJobDeleted: resourceAbsent(
|
|
fixture,
|
|
'job/ql3-security-live-delivery-provision',
|
|
),
|
|
deliveryVolumeClaimDeleted: resourceAbsent(
|
|
fixture,
|
|
`persistentvolumeclaim/${DELIVERY_CLAIM}`,
|
|
),
|
|
controlDeploymentDeleted: resourceAbsent(
|
|
fixture,
|
|
`deployment/${CONTROL_NAME}`,
|
|
),
|
|
controlServiceDeleted: resourceAbsent(
|
|
fixture,
|
|
`service/${CONTROL_NAME}`,
|
|
),
|
|
controlRuntimeSecretDeleted: resourceAbsent(
|
|
fixture,
|
|
`secret/${CONTROL_RUNTIME_SECRET}`,
|
|
),
|
|
};
|
|
assert.equal(Object.values(cleanup).every(Boolean), true);
|
|
|
|
const report = {
|
|
schemaVersion: 1,
|
|
fixture: FIXTURE,
|
|
observedAt: new Date().toISOString(),
|
|
platform: {
|
|
distribution: 'k3s',
|
|
kubernetesVersion: nodes[0].status.nodeInfo.kubeletVersion,
|
|
architecture,
|
|
kubernetesImageId: imageIdDigest(k3sImageInfo),
|
|
administrationImageId: imageIdDigest(adminImageInfo),
|
|
controlImageId: imageIdDigest(controlImageInfo),
|
|
cniName: 'flannel',
|
|
cniDistributionBinding: fixture.k3sImage,
|
|
controlPlaneNodes: 1,
|
|
workerNodes: 2,
|
|
readyNodes: readyNodes.length,
|
|
},
|
|
database: {
|
|
operator: 'cloudnative-pg',
|
|
operatorVersion: LOCK.operator.version,
|
|
postgresVersionNumber: migrationState.postgresVersionNumber,
|
|
postgresImageId: imageIdDigest(postgresImageInfo),
|
|
instances: Number(finalCluster.spec.instances),
|
|
readyInstances: Number(finalCluster.status.readyInstances),
|
|
administrationRole: 'ql3_admin',
|
|
roleConnectionLimit: adminRole.connectionLimit,
|
|
commandConnectionLimit: 1,
|
|
migrationCount: migrationState.migrationCount,
|
|
controlCoreCapability: migrationState.controlCoreCapability,
|
|
tlsVerified: true,
|
|
leastPrivilege: true,
|
|
},
|
|
ceremony: {
|
|
operations: [
|
|
'identity.register',
|
|
'audit.list',
|
|
'credential.issue.old',
|
|
'credential.issue.old.replay',
|
|
'credential.key-references.before-activate',
|
|
'credential.issue.new',
|
|
'credential.rotate.new',
|
|
'credential.key-references.after-activate',
|
|
'credential.revoke.old',
|
|
'credential.key-references.after-converge',
|
|
],
|
|
completedJobs: 10,
|
|
failedJobs: 1,
|
|
authenticationProbeJobs: 5,
|
|
controlReplicas: controlAfterContraction.replicas,
|
|
controlRollouts: 3,
|
|
controlReplicaAntiAffinity:
|
|
new Set(controlBeforeActivation.nodes).size === 2 &&
|
|
new Set(controlDuringOverlap.nodes).size === 2 &&
|
|
new Set(controlAfterContraction.nodes).size === 2,
|
|
callerDriven: true,
|
|
backoffLimit: baseTemplate.spec.backoffLimit,
|
|
activeDeadlineSeconds: baseTemplate.spec.activeDeadlineSeconds,
|
|
ttlSecondsAfterFinished: baseTemplate.spec.ttlSecondsAfterFinished,
|
|
serviceAccount: NAME,
|
|
serviceAccountTokenMounted: false,
|
|
rbacGranted: false,
|
|
responseLossReplayObserved: true,
|
|
overlapGenerationCount: overlapOldActive.keys.length,
|
|
contractedGenerationCount: contractedNew.keys.length,
|
|
activeGenerationChanged:
|
|
overlapOldActive.activePepperKeyId !==
|
|
overlapNewActive.activePepperKeyId,
|
|
oldReferencesBeforeActivation: referencesBeforeActivate,
|
|
oldReferencesAfterActivation: referencesAfterActivate,
|
|
oldReferencesAfterConvergence: referencesAfterConvergence,
|
|
oldAuthenticationBeforeActivation:
|
|
oldAuthenticationBeforeActivation.observedStatus === 403,
|
|
oldAuthenticationDuringOverlap:
|
|
oldAuthenticationDuringOverlap.observedStatus === 403,
|
|
newAuthenticationDuringOverlap:
|
|
newAuthenticationDuringOverlap.observedStatus === 403,
|
|
oldAuthenticationRejectedAfterConvergence:
|
|
oldAuthenticationAfterConvergence.observedStatus === 401,
|
|
newAuthenticationAfterContraction:
|
|
newAuthenticationAfterContraction.observedStatus === 403,
|
|
contractedToActiveGeneration:
|
|
contractedNew.activePepperKeyId ===
|
|
overlapNewActive.activePepperKeyId &&
|
|
contractedNew.keys[0]?.pepperKeyId ===
|
|
contractedNew.activePepperKeyId,
|
|
sensitiveMaterialReported: false,
|
|
},
|
|
inputBoundary: {
|
|
immutableSecret: true,
|
|
projectedMode0440: true,
|
|
memoryBackedPrivateStage: true,
|
|
targetDirectoryMode0700: true,
|
|
targetFilesMode0600: true,
|
|
kubeletAtomicWriterProjectionAccepted: true,
|
|
worldReadableProjectionRejected: true,
|
|
mainContainerNotStartedAfterStageFailure: true,
|
|
},
|
|
deliveryCustody: {
|
|
persistentVolumeClaim: true,
|
|
accessMode: 'ReadWriteOnce',
|
|
fixtureRootProvisioned: deliveryProvision.passed,
|
|
fixtureRootMode: deliveryProvision.afterMode,
|
|
fixtureProvisionerRanAsRoot: deliveryProvision.uid === 0,
|
|
privateDirectoryMode: finalEvidence.directoryMode,
|
|
fileMode: finalEvidence.fileMode,
|
|
fileCount: finalEvidence.files.length,
|
|
issueDigest: issueFile.digest,
|
|
rotationDigest: rotationFile.digest,
|
|
distinctRotationMaterial: issueFile.digest !== rotationFile.digest,
|
|
persistentAcrossJobs: true,
|
|
noReplaceReplayPreserved:
|
|
JSON.stringify(replayEvidence.files) ===
|
|
JSON.stringify(issuedEvidence.files),
|
|
deliverySchemaValidated: true,
|
|
bearerFormatValidatedInPod: true,
|
|
sensitiveMaterialReported: false,
|
|
},
|
|
isolation: {
|
|
dnsAndDatabaseEgressAllowed: finalEvidence.databaseConnected,
|
|
kubernetesApiEgressDenied: !finalEvidence.kubernetesApiConnected,
|
|
publicInternetEgressDenied: !finalEvidence.publicInternetConnected,
|
|
secretReadRbacDenied: canI(fixture, 'get', 'secrets') === 'no',
|
|
jobMutationRbacDenied: canI(fixture, 'create', 'jobs.batch') === 'no',
|
|
},
|
|
durability: durable,
|
|
cleanup,
|
|
gates: {
|
|
realThreeNodeKubernetes: nodes.length === 3,
|
|
realCloudNativePg:
|
|
databasePods.length === 3 &&
|
|
Number(finalCluster.status.readyInstances) === 3,
|
|
realKubeletSecretProjection: true,
|
|
realAdministrationProductCommands: true,
|
|
realPersistentCredentialCustody: finalEvidence.files.length === 3,
|
|
realClusterControlAuthenticationRotation:
|
|
durable.authenticationDeniedAuditCount === 4 &&
|
|
durable.authenticationRejectedAuditCount === 1,
|
|
responseLossReplay: durable.issueMutationCount === 1,
|
|
failedInputStageClosed: true,
|
|
leastPrivilege: true,
|
|
contentFreeEvidence: true,
|
|
passed: true,
|
|
},
|
|
limitations: [...LIMITATIONS],
|
|
};
|
|
const audit = validateSecurityAdministrationKubernetesLiveReport(report);
|
|
assert.deepEqual(audit.findings, []);
|
|
fs.writeFileSync(reportFile, `${JSON.stringify(report, null, 2)}\n`, {
|
|
mode: 0o600,
|
|
flag: 'wx',
|
|
});
|
|
process.stdout.write(
|
|
`${JSON.stringify({
|
|
schemaVersion: 1,
|
|
fixture: FIXTURE,
|
|
reportWritten: true,
|
|
passed: true,
|
|
})}\n`,
|
|
);
|
|
} finally {
|
|
await fixture.cleanup();
|
|
if (adminImageBuilt) {
|
|
run(fixture.docker, ['image', 'rm', '-f', adminImage], {
|
|
capture: true,
|
|
quiet: true,
|
|
allowFailure: true,
|
|
});
|
|
}
|
|
if (controlImageBuilt) {
|
|
run(fixture.docker, ['image', 'rm', '-f', controlImage], {
|
|
capture: true,
|
|
quiet: true,
|
|
allowFailure: true,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
if (require.main === module) {
|
|
main().catch((error) => {
|
|
process.stderr.write(
|
|
`QL3 Security Administration Kubernetes live contract failed: ${
|
|
error instanceof Error ? error.stack || error.message : String(error)
|
|
}\n`,
|
|
);
|
|
process.exitCode = 1;
|
|
});
|
|
}
|
|
|
|
module.exports = {
|
|
auditListCommand,
|
|
clusterControlResources,
|
|
runtimeFileMaterializationSource,
|
|
controlRolloutFailureEvidence,
|
|
controlTerminationFact,
|
|
credentialAuthenticationProbeSource,
|
|
credentialIssueCommand,
|
|
credentialRevokeCommand,
|
|
credentialRotateCommand,
|
|
custodyEvidenceSource,
|
|
deliveryVolumeProvisionSource,
|
|
identity,
|
|
identityRegisterCommand,
|
|
inputAuthorityEvidenceSource,
|
|
migrationFailureEvidence,
|
|
networkPolicyReadinessSource,
|
|
};
|