fix(ql3): materialize projected runtime files

This commit is contained in:
whyour
2026-08-26 04:09:47 +08:00
parent 9c58b15d2b
commit beb490c48c
9 changed files with 273 additions and 30 deletions
+6 -2
View File
@@ -967,8 +967,12 @@ and is fixed to `verify-full` in the committed deployment. The servername is
mandatory, must be an explicit DNS name rather than an IP literal, and must mandatory, must be an explicit DNS name rather than an IP literal, and must
match the endpoint certificate SAN. Only match the endpoint certificate SAN. Only
`postgres-ca.crt` and `api-credential-pepper-keyring.json` are projected from `postgres-ca.crt` and `api-credential-pepper-keyring.json` are projected from
this Secret into the runtime private mount; only the URL and servername remain this Secret only into a hardened init container. It resolves one kubelet
environment values. The keyring is a canonical, bounded 12 generation file; `..data` generation, copies both values without replacement into a Pod-private
memory volume, and changes each regular file to `0400`; the long-running
container can read only that materialized volume, never the symlink-backed
Secret projection. Only the URL and servername remain environment values. The
keyring is a canonical, bounded 12 generation file;
the singleton above is the bootstrap form. The CA loader requires an absolute the singleton above is the bootstrap form. The CA loader requires an absolute
path to a regular file that is not group/world writable, 1256 KiB, and path to a regular file that is not group/world writable, 1256 KiB, and
contains 116 unique PEM X.509 CA certificates with no trailing data. contains 116 unique PEM X.509 CA certificates with no trailing data.
@@ -45,6 +45,42 @@ spec:
matchLabels: matchLabels:
app.kubernetes.io/name: ql3-cluster-control app.kubernetes.io/name: ql3-cluster-control
app.kubernetes.io/component: control-plane app.kubernetes.io/component: control-plane
initContainers:
- name: materialize-runtime-files
image: qinglong3-cluster-control:3.0.0-alpha.1
imagePullPolicy: IfNotPresent
command:
- node
- -e
- |
const fs = require('node:fs');
const path = require('node:path');
const source = fs.realpathSync('/var/run/secrets/qinglong3/postgres-runtime-projected/..data');
const target = '/var/run/secrets/qinglong3/postgres-runtime';
for (const name of ['ca.crt', 'api-credential-pepper-keyring.json']) {
const output = path.join(target, name);
fs.copyFileSync(path.join(source, name), output, fs.constants.COPYFILE_EXCL);
fs.chmodSync(output, 0o400);
}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
resources:
requests:
cpu: 10m
memory: 32Mi
limits:
cpu: 100m
memory: 64Mi
volumeMounts:
- name: postgres-runtime-projected
mountPath: /var/run/secrets/qinglong3/postgres-runtime-projected
readOnly: true
- name: postgres-runtime-private
mountPath: /var/run/secrets/qinglong3/postgres-runtime
containers: containers:
- name: cluster-control - name: cluster-control
image: qinglong3-cluster-control:3.0.0-alpha.1 image: qinglong3-cluster-control:3.0.0-alpha.1
@@ -223,7 +259,7 @@ spec:
volumeMounts: volumeMounts:
- name: tmp - name: tmp
mountPath: /tmp mountPath: /tmp
- name: postgres-runtime-ca - name: postgres-runtime-private
mountPath: /var/run/secrets/qinglong3/postgres-runtime mountPath: /var/run/secrets/qinglong3/postgres-runtime
readOnly: true readOnly: true
- name: postgres-worker-ingress-ca - name: postgres-worker-ingress-ca
@@ -240,7 +276,7 @@ spec:
emptyDir: emptyDir:
medium: Memory medium: Memory
sizeLimit: 16Mi sizeLimit: 16Mi
- name: postgres-runtime-ca - name: postgres-runtime-projected
secret: secret:
secretName: ql3-cluster-control-runtime secretName: ql3-cluster-control-runtime
defaultMode: 292 defaultMode: 292
@@ -249,6 +285,10 @@ spec:
path: ca.crt path: ca.crt
- key: api-credential-pepper-keyring.json - key: api-credential-pepper-keyring.json
path: api-credential-pepper-keyring.json path: api-credential-pepper-keyring.json
- name: postgres-runtime-private
emptyDir:
medium: Memory
sizeLimit: 1Mi
- name: postgres-worker-ingress-ca - name: postgres-worker-ingress-ca
secret: secret:
secretName: ql3-cluster-worker-ingress secretName: ql3-cluster-worker-ingress
+60 -10
View File
@@ -846,18 +846,66 @@ function assertKubernetes(readFile, root, findings) {
), ),
); );
} }
const runtimeCaMount = namedEntry( const runtimeFileMaterializer = namedEntry(
container?.volumeMounts, pod?.initContainers,
'postgres-runtime-ca', 'materialize-runtime-files',
); );
const runtimeCaVolume = namedEntry(pod?.volumes, 'postgres-runtime-ca'); const runtimePrivateMount = namedEntry(
container?.volumeMounts,
'postgres-runtime-private',
);
const projectedMount = namedEntry(
runtimeFileMaterializer?.volumeMounts,
'postgres-runtime-projected',
);
const materializedMount = namedEntry(
runtimeFileMaterializer?.volumeMounts,
'postgres-runtime-private',
);
const projectedVolume = namedEntry(
pod?.volumes,
'postgres-runtime-projected',
);
const privateVolume = namedEntry(pod?.volumes, 'postgres-runtime-private');
const materializerSource = runtimeFileMaterializer?.command?.[2];
if ( if (
runtimeCaMount?.mountPath !== runtimePrivateMount?.mountPath !==
'/var/run/secrets/qinglong3/postgres-runtime' || '/var/run/secrets/qinglong3/postgres-runtime' ||
runtimeCaMount?.readOnly !== true || runtimePrivateMount?.readOnly !== true ||
runtimeCaVolume?.secret?.secretName !== 'ql3-cluster-control-runtime' || container?.volumeMounts?.some(
runtimeCaVolume?.secret?.defaultMode !== 0o444 || (mount) => mount?.name === 'postgres-runtime-projected',
JSON.stringify(runtimeCaVolume?.secret?.items) !== ) ||
runtimeFileMaterializer?.image !== container?.image ||
runtimeFileMaterializer?.imagePullPolicy !== container?.imagePullPolicy ||
JSON.stringify(runtimeFileMaterializer?.command?.slice(0, 2)) !==
JSON.stringify(['node', '-e']) ||
typeof materializerSource !== 'string' ||
!materializerSource.includes(
"realpathSync('/var/run/secrets/qinglong3/postgres-runtime-projected/..data')",
) ||
!materializerSource.includes('COPYFILE_EXCL') ||
!materializerSource.includes('chmodSync(output, 0o400)') ||
materializerSource.includes('console.') ||
runtimeFileMaterializer?.securityContext?.allowPrivilegeEscalation !==
false ||
runtimeFileMaterializer?.securityContext?.readOnlyRootFilesystem !== true ||
JSON.stringify(
runtimeFileMaterializer?.securityContext?.capabilities?.drop,
) !== JSON.stringify(['ALL']) ||
runtimeFileMaterializer?.resources?.requests?.cpu !== '10m' ||
runtimeFileMaterializer?.resources?.requests?.memory !== '32Mi' ||
runtimeFileMaterializer?.resources?.limits?.cpu !== '100m' ||
runtimeFileMaterializer?.resources?.limits?.memory !== '64Mi' ||
runtimeFileMaterializer?.volumeMounts?.length !== 2 ||
projectedMount?.mountPath !==
'/var/run/secrets/qinglong3/postgres-runtime-projected' ||
projectedMount?.readOnly !== true ||
materializedMount?.mountPath !==
'/var/run/secrets/qinglong3/postgres-runtime' ||
materializedMount?.readOnly === true ||
projectedVolume?.secret?.secretName !== 'ql3-cluster-control-runtime' ||
projectedVolume?.secret?.defaultMode !== 0o444 ||
JSON.stringify(projectedVolume?.secret?.items) !==
JSON.stringify([ JSON.stringify([
{ key: 'postgres-ca.crt', path: 'ca.crt' }, { key: 'postgres-ca.crt', path: 'ca.crt' },
{ {
@@ -865,12 +913,14 @@ function assertKubernetes(readFile, root, findings) {
path: 'api-credential-pepper-keyring.json', path: 'api-credential-pepper-keyring.json',
}, },
]) || ]) ||
privateVolume?.emptyDir?.medium !== 'Memory' ||
privateVolume?.emptyDir?.sizeLimit !== '1Mi' ||
env.has('QL3_API_CREDENTIAL_PEPPER') env.has('QL3_API_CREDENTIAL_PEPPER')
) { ) {
findings.push( findings.push(
finding( finding(
'QL3_CLUSTER_KUBERNETES_POSTGRES_CA_BINDING', 'QL3_CLUSTER_KUBERNETES_POSTGRES_CA_BINDING',
'runtime PostgreSQL trust must use the reviewed read-only projected CA file', 'runtime trust and API credential keyring must be copied from one projected Secret generation into bounded Pod-private regular files before the control process starts',
), ),
); );
} }
+1 -1
View File
@@ -28,7 +28,7 @@ const IMAGE_NAMES = Object.freeze({
worker: 'qinglong3-worker', worker: 'qinglong3-worker',
}); });
const EXPECTED_SOURCE_SURFACES = Object.freeze({ const EXPECTED_SOURCE_SURFACES = Object.freeze({
control: 2, control: 3,
'control-ai': 1, 'control-ai': 1,
admin: 28, admin: 28,
worker: 2, worker: 2,
@@ -731,6 +731,27 @@ async function runCustodyEvidence({
} }
} }
function runtimeFileMaterializationSource({
postgresDirectory = '/var/run/secrets/qinglong3/postgres-projected',
keyringDirectory = '/var/run/secrets/qinglong3/api-credential-projected',
targetDirectory = '/var/run/secrets/qinglong3/runtime',
} = {}) {
for (const value of [
postgresDirectory,
keyringDirectory,
targetDirectory,
]) {
assert.equal(path.isAbsolute(value), true);
}
return [
"const fs=require('node:fs')",
"const path=require('node:path')",
`const target=${JSON.stringify(targetDirectory)}`,
`const files=[[${JSON.stringify(postgresDirectory + '/..data')},'ca.crt'],[${JSON.stringify(keyringDirectory + '/..data')},'keyring.json']]`,
"for(const [directory,name] of files){const source=fs.realpathSync(directory);const output=path.join(target,name);fs.copyFileSync(path.join(source,name),output,fs.constants.COPYFILE_EXCL);fs.chmodSync(output,0o400)}",
].join(';');
}
function clusterControlResources(controlImage) { function clusterControlResources(controlImage) {
const labels = Object.freeze({ const labels = Object.freeze({
'app.kubernetes.io/name': CONTROL_NAME, 'app.kubernetes.io/name': CONTROL_NAME,
@@ -803,7 +824,7 @@ function clusterControlResources(controlImage) {
{ name: 'QL3_POSTGRES_TLS_MODE', value: 'verify-full' }, { name: 'QL3_POSTGRES_TLS_MODE', value: 'verify-full' },
{ {
name: 'QL3_POSTGRES_TLS_CA_FILE', name: 'QL3_POSTGRES_TLS_CA_FILE',
value: '/var/run/secrets/qinglong3/postgres/ca.crt', value: '/var/run/secrets/qinglong3/runtime/ca.crt',
}, },
{ {
name: 'QL3_POSTGRES_TLS_SERVERNAME', name: 'QL3_POSTGRES_TLS_SERVERNAME',
@@ -834,7 +855,7 @@ function clusterControlResources(controlImage) {
}, },
{ {
name: 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE', name: 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE',
value: '/var/run/secrets/qinglong3/api-credential/keyring.json', value: '/var/run/secrets/qinglong3/runtime/keyring.json',
}, },
], ],
ports: [{ name: 'http', containerPort: 5800 }], ports: [{ name: 'http', containerPort: 5800 }],
@@ -857,15 +878,45 @@ function clusterControlResources(controlImage) {
volumeMounts: [ volumeMounts: [
{ name: 'tmp', mountPath: '/tmp' }, { name: 'tmp', mountPath: '/tmp' },
{ {
name: 'postgres-ca', name: 'runtime-private',
mountPath: '/var/run/secrets/qinglong3/postgres', mountPath: '/var/run/secrets/qinglong3/runtime',
readOnly: true,
},
],
}],
initContainers: [{
name: 'materialize-runtime-files',
image: controlImage,
imagePullPolicy: 'Never',
command: [
'node',
'-e',
runtimeFileMaterializationSource(),
],
securityContext: {
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
capabilities: { drop: ['ALL'] },
},
resources: {
requests: { cpu: '10m', memory: '32Mi' },
limits: { cpu: '100m', memory: '64Mi' },
},
volumeMounts: [
{
name: 'postgres-ca-projected',
mountPath: '/var/run/secrets/qinglong3/postgres-projected',
readOnly: true, readOnly: true,
}, },
{ {
name: 'api-credential-keyring', name: 'api-credential-keyring-projected',
mountPath: '/var/run/secrets/qinglong3/api-credential', mountPath: '/var/run/secrets/qinglong3/api-credential-projected',
readOnly: true, readOnly: true,
}, },
{
name: 'runtime-private',
mountPath: '/var/run/secrets/qinglong3/runtime',
},
], ],
}], }],
volumes: [ volumes: [
@@ -874,7 +925,7 @@ function clusterControlResources(controlImage) {
emptyDir: { medium: 'Memory', sizeLimit: '16Mi' }, emptyDir: { medium: 'Memory', sizeLimit: '16Mi' },
}, },
{ {
name: 'postgres-ca', name: 'postgres-ca-projected',
secret: { secret: {
secretName: 'ql3-postgres-ca', secretName: 'ql3-postgres-ca',
defaultMode: 292, defaultMode: 292,
@@ -882,7 +933,7 @@ function clusterControlResources(controlImage) {
}, },
}, },
{ {
name: 'api-credential-keyring', name: 'api-credential-keyring-projected',
secret: { secret: {
secretName: CONTROL_RUNTIME_SECRET, secretName: CONTROL_RUNTIME_SECRET,
defaultMode: 292, defaultMode: 292,
@@ -892,6 +943,10 @@ function clusterControlResources(controlImage) {
}], }],
}, },
}, },
{
name: 'runtime-private',
emptyDir: { medium: 'Memory', sizeLimit: '1Mi' },
},
], ],
}, },
}, },
@@ -2353,6 +2408,7 @@ if (require.main === module) {
module.exports = { module.exports = {
auditListCommand, auditListCommand,
clusterControlResources, clusterControlResources,
runtimeFileMaterializationSource,
controlRolloutFailureEvidence, controlRolloutFailureEvidence,
controlTerminationFact, controlTerminationFact,
credentialAuthenticationProbeSource, credentialAuthenticationProbeSource,
+40 -2
View File
@@ -47,6 +47,44 @@ test('accepts the exact locked non-root multi-replica cluster deployment', () =>
]); ]);
}); });
test('rejects exposing symlink-backed runtime keyring projections to Cluster Control', () => {
const withoutMaterializer = auditClusterDeployment({
root: ROOT,
readFile: intercept(
'deploy/kubernetes/ql3-cluster/base/deployment.yaml',
(source) => source.replace(
' - name: materialize-runtime-files\n',
' - name: materialize-runtime-files-disabled\n',
),
),
});
assert.equal(withoutMaterializer.compatible, false);
assert.equal(
withoutMaterializer.findings.some(
({ code }) => code === 'QL3_CLUSTER_KUBERNETES_POSTGRES_CA_BINDING',
),
true,
);
const projectedIntoRuntime = auditClusterDeployment({
root: ROOT,
readFile: intercept(
'deploy/kubernetes/ql3-cluster/base/deployment.yaml',
(source) => source.replace(
' - name: postgres-runtime-private\n mountPath: /var/run/secrets/qinglong3/postgres-runtime\n readOnly: true\n',
' - name: postgres-runtime-projected\n mountPath: /var/run/secrets/qinglong3/postgres-runtime\n readOnly: true\n',
),
),
});
assert.equal(projectedIntoRuntime.compatible, false);
assert.equal(
projectedIntoRuntime.findings.some(
({ code }) => code === 'QL3_CLUSTER_KUBERNETES_POSTGRES_CA_BINDING',
),
true,
);
});
test('keeps Cluster Copilot MCP external, digest-pinned and resource-bounded', () => { test('keeps Cluster Copilot MCP external, digest-pinned and resource-bounded', () => {
const widened = auditClusterDeployment({ const widened = auditClusterDeployment({
root: ROOT, root: ROOT,
@@ -237,7 +275,7 @@ test('keeps Cluster AI optional with projected authority and an independent dige
readFile: intercept( readFile: intercept(
'deploy/kubernetes/ql3-cluster/base/deployment.yaml', 'deploy/kubernetes/ql3-cluster/base/deployment.yaml',
(source) => (source) =>
source.replace( source.replaceAll(
`image: qinglong3-cluster-control:${VERSION}`, `image: qinglong3-cluster-control:${VERSION}`,
`image: qinglong3-cluster-control-ai:${VERSION}`, `image: qinglong3-cluster-control-ai:${VERSION}`,
), ),
@@ -1261,7 +1299,7 @@ test('rejects inline runtime credentials and a privileged container', () => {
/valueFrom:\n\s+secretKeyRef:\n\s+name: ql3-cluster-control-runtime\n\s+key: postgres-runtime-url/, /valueFrom:\n\s+secretKeyRef:\n\s+name: ql3-cluster-control-runtime\n\s+key: postgres-runtime-url/,
'value: postgresql://inline-secret', 'value: postgresql://inline-secret',
) )
.replace( .replaceAll(
'allowPrivilegeEscalation: false', 'allowPrivilegeEscalation: false',
'allowPrivilegeEscalation: true', 'allowPrivilegeEscalation: true',
), ),
+1 -1
View File
@@ -921,7 +921,7 @@ test('source-surface audit freezes every reviewed cluster and worker authority',
schemaVersion: 1, schemaVersion: 1,
deploymentYamlFiles: 241, deploymentYamlFiles: 241,
imageOccurrences: { imageOccurrences: {
control: 2, control: 3,
'control-ai': 1, 'control-ai': 1,
admin: 28, admin: 28,
worker: 2, worker: 2,
@@ -1,6 +1,7 @@
const assert = require('node:assert/strict'); const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process'); const { spawnSync } = require('node:child_process');
const fs = require('node:fs'); const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path'); const path = require('node:path');
const { test } = require('node:test'); const { test } = require('node:test');
@@ -20,6 +21,7 @@ const {
inputAuthorityEvidenceSource, inputAuthorityEvidenceSource,
migrationFailureEvidence, migrationFailureEvidence,
networkPolicyReadinessSource, networkPolicyReadinessSource,
runtimeFileMaterializationSource,
} = require('../../scripts/ql3-security-administration-kubernetes-live-contract.cjs'); } = require('../../scripts/ql3-security-administration-kubernetes-live-contract.cjs');
const values = Object.freeze({ const values = Object.freeze({
@@ -205,6 +207,21 @@ test('runs the credential ceremony against two real anti-affine control replicas
deployment.spec.template.spec.containers[0].terminationMessagePolicy, deployment.spec.template.spec.containers[0].terminationMessagePolicy,
'FallbackToLogsOnError', 'FallbackToLogsOnError',
); );
const materializer = deployment.spec.template.spec.initContainers[0];
assert.equal(materializer.name, 'materialize-runtime-files');
assert.deepEqual(materializer.command.slice(0, 2), ['node', '-e']);
assert.match(materializer.command[2], /realpathSync/);
assert.match(materializer.command[2], /COPYFILE_EXCL/);
assert.match(materializer.command[2], /chmodSync\(output,0o400\)/);
assert.equal(materializer.securityContext.runAsNonRoot, undefined);
assert.equal(materializer.securityContext.readOnlyRootFilesystem, true);
assert.deepEqual(materializer.securityContext.capabilities.drop, ['ALL']);
assert.equal(
deployment.spec.template.spec.containers[0].volumeMounts.some(
(mount) => mount.name.includes('projected'),
),
false,
);
assert.equal( assert.equal(
deployment.spec.template.spec.affinity.podAntiAffinity deployment.spec.template.spec.affinity.podAntiAffinity
.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey, .requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey,
@@ -215,7 +232,7 @@ test('runs the credential ceremony against two real anti-affine control replicas
environment.some( environment.some(
(entry) => (entry) =>
entry.name === 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE' && entry.name === 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE' &&
entry.value.endsWith('/keyring.json'), entry.value === '/var/run/secrets/qinglong3/runtime/keyring.json',
), ),
); );
assert.equal( assert.equal(
@@ -224,6 +241,44 @@ test('runs the credential ceremony against two real anti-affine control replicas
); );
}); });
test('materializes kubelet symlink projections as private regular files', (context) => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-runtime-files-'));
context.after(() => fs.rmSync(directory, { recursive: true, force: true }));
const postgresDirectory = path.join(directory, 'postgres-projected');
const keyringDirectory = path.join(directory, 'keyring-projected');
const targetDirectory = path.join(directory, 'runtime');
for (const [projectedDirectory, name, value] of [
[postgresDirectory, 'ca.crt', 'test-ca'],
[keyringDirectory, 'keyring.json', '{"schemaVersion":1}'],
]) {
const generation = path.join(projectedDirectory, '..2026_08_26');
fs.mkdirSync(generation, { recursive: true });
fs.writeFileSync(path.join(generation, name), value);
fs.symlinkSync('..2026_08_26', path.join(projectedDirectory, '..data'));
}
fs.mkdirSync(targetDirectory);
const result = spawnSync(
process.execPath,
['-e', runtimeFileMaterializationSource({
postgresDirectory,
keyringDirectory,
targetDirectory,
})],
{ encoding: 'utf8' },
);
assert.equal(result.status, 0, result.stderr);
for (const [name, value] of [
['ca.crt', 'test-ca'],
['keyring.json', '{"schemaVersion":1}'],
]) {
const output = path.join(targetDirectory, name);
assert.equal(fs.lstatSync(output).isFile(), true);
assert.equal(fs.lstatSync(output).isSymbolicLink(), false);
assert.equal(fs.statSync(output).mode & 0o777, 0o400);
assert.equal(fs.readFileSync(output, 'utf8'), value);
}
});
test('keeps failed control rollout evidence bounded and content-free', () => { test('keeps failed control rollout evidence bounded and content-free', () => {
const failure = JSON.stringify({ const failure = JSON.stringify({
schemaVersion: 1, schemaVersion: 1,
+3 -3
View File
@@ -103,8 +103,8 @@ test('audits one source-derived QingLong 3 release identity', () => {
workspacePackageCount: 18, workspacePackageCount: 18,
containerRootCount: 4, containerRootCount: 4,
deploymentFileCount: 260, deploymentFileCount: 260,
deploymentImageReferences: 34, deploymentImageReferences: 35,
deploymentVersionOccurrences: 38, deploymentVersionOccurrences: 39,
compatible: true, compatible: true,
}); });
}); });
@@ -116,7 +116,7 @@ test('plans the exact governed version surface without touching legacy 2.x', ()
targetVersion: TARGET_VERSION, targetVersion: TARGET_VERSION,
}); });
assert.equal(plan.fileCount, 66); assert.equal(plan.fileCount, 66);
assert.equal(plan.replacementCount, 85); assert.equal(plan.replacementCount, 86);
assert.equal(plan.legacyRootPackageVersion, LEGACY_VERSION); assert.equal(plan.legacyRootPackageVersion, LEGACY_VERSION);
assert.equal(plan.legacyRootExcluded, true); assert.equal(plan.legacyRootExcluded, true);
assert.equal( assert.equal(