diff --git a/deploy/kubernetes/ql3-cluster/README.md b/deploy/kubernetes/ql3-cluster/README.md index c9ea1ff8..9e4421c8 100644 --- a/deploy/kubernetes/ql3-cluster/README.md +++ b/deploy/kubernetes/ql3-cluster/README.md @@ -967,8 +967,12 @@ and is fixed to `verify-full` in the committed deployment. The servername is mandatory, must be an explicit DNS name rather than an IP literal, and must match the endpoint certificate SAN. Only `postgres-ca.crt` and `api-credential-pepper-keyring.json` are projected from -this Secret into the runtime private mount; only the URL and servername remain -environment values. The keyring is a canonical, bounded 1–2 generation file; +this Secret only into a hardened init container. It resolves one kubelet +`..data` generation, copies both values without replacement into a Pod-private +memory volume, and changes each regular file to `0400`; the long-running +container can read only that materialized volume, never the symlink-backed +Secret projection. Only the URL and servername remain environment values. The +keyring is a canonical, bounded 1–2 generation file; the singleton above is the bootstrap form. The CA loader requires an absolute path to a regular file that is not group/world writable, 1–256 KiB, and contains 1–16 unique PEM X.509 CA certificates with no trailing data. diff --git a/deploy/kubernetes/ql3-cluster/base/deployment.yaml b/deploy/kubernetes/ql3-cluster/base/deployment.yaml index 98d798cb..6b15f953 100644 --- a/deploy/kubernetes/ql3-cluster/base/deployment.yaml +++ b/deploy/kubernetes/ql3-cluster/base/deployment.yaml @@ -45,6 +45,42 @@ spec: matchLabels: app.kubernetes.io/name: ql3-cluster-control app.kubernetes.io/component: control-plane + initContainers: + - name: materialize-runtime-files + image: qinglong3-cluster-control:3.0.0-alpha.1 + imagePullPolicy: IfNotPresent + command: + - node + - -e + - | + const fs = require('node:fs'); + const path = require('node:path'); + const source = fs.realpathSync('/var/run/secrets/qinglong3/postgres-runtime-projected/..data'); + const target = '/var/run/secrets/qinglong3/postgres-runtime'; + for (const name of ['ca.crt', 'api-credential-pepper-keyring.json']) { + const output = path.join(target, name); + fs.copyFileSync(path.join(source, name), output, fs.constants.COPYFILE_EXCL); + fs.chmodSync(output, 0o400); + } + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + volumeMounts: + - name: postgres-runtime-projected + mountPath: /var/run/secrets/qinglong3/postgres-runtime-projected + readOnly: true + - name: postgres-runtime-private + mountPath: /var/run/secrets/qinglong3/postgres-runtime containers: - name: cluster-control image: qinglong3-cluster-control:3.0.0-alpha.1 @@ -223,7 +259,7 @@ spec: volumeMounts: - name: tmp mountPath: /tmp - - name: postgres-runtime-ca + - name: postgres-runtime-private mountPath: /var/run/secrets/qinglong3/postgres-runtime readOnly: true - name: postgres-worker-ingress-ca @@ -240,7 +276,7 @@ spec: emptyDir: medium: Memory sizeLimit: 16Mi - - name: postgres-runtime-ca + - name: postgres-runtime-projected secret: secretName: ql3-cluster-control-runtime defaultMode: 292 @@ -249,6 +285,10 @@ spec: path: ca.crt - key: api-credential-pepper-keyring.json path: api-credential-pepper-keyring.json + - name: postgres-runtime-private + emptyDir: + medium: Memory + sizeLimit: 1Mi - name: postgres-worker-ingress-ca secret: secretName: ql3-cluster-worker-ingress diff --git a/scripts/ql3-cluster-deployment-audit.cjs b/scripts/ql3-cluster-deployment-audit.cjs index 25e1effc..537d1ad5 100644 --- a/scripts/ql3-cluster-deployment-audit.cjs +++ b/scripts/ql3-cluster-deployment-audit.cjs @@ -846,18 +846,66 @@ function assertKubernetes(readFile, root, findings) { ), ); } - const runtimeCaMount = namedEntry( - container?.volumeMounts, - 'postgres-runtime-ca', + const runtimeFileMaterializer = namedEntry( + pod?.initContainers, + 'materialize-runtime-files', ); - const runtimeCaVolume = namedEntry(pod?.volumes, 'postgres-runtime-ca'); + const runtimePrivateMount = namedEntry( + container?.volumeMounts, + 'postgres-runtime-private', + ); + const projectedMount = namedEntry( + runtimeFileMaterializer?.volumeMounts, + 'postgres-runtime-projected', + ); + const materializedMount = namedEntry( + runtimeFileMaterializer?.volumeMounts, + 'postgres-runtime-private', + ); + const projectedVolume = namedEntry( + pod?.volumes, + 'postgres-runtime-projected', + ); + const privateVolume = namedEntry(pod?.volumes, 'postgres-runtime-private'); + const materializerSource = runtimeFileMaterializer?.command?.[2]; if ( - runtimeCaMount?.mountPath !== + runtimePrivateMount?.mountPath !== '/var/run/secrets/qinglong3/postgres-runtime' || - runtimeCaMount?.readOnly !== true || - runtimeCaVolume?.secret?.secretName !== 'ql3-cluster-control-runtime' || - runtimeCaVolume?.secret?.defaultMode !== 0o444 || - JSON.stringify(runtimeCaVolume?.secret?.items) !== + runtimePrivateMount?.readOnly !== true || + container?.volumeMounts?.some( + (mount) => mount?.name === 'postgres-runtime-projected', + ) || + runtimeFileMaterializer?.image !== container?.image || + runtimeFileMaterializer?.imagePullPolicy !== container?.imagePullPolicy || + JSON.stringify(runtimeFileMaterializer?.command?.slice(0, 2)) !== + JSON.stringify(['node', '-e']) || + typeof materializerSource !== 'string' || + !materializerSource.includes( + "realpathSync('/var/run/secrets/qinglong3/postgres-runtime-projected/..data')", + ) || + !materializerSource.includes('COPYFILE_EXCL') || + !materializerSource.includes('chmodSync(output, 0o400)') || + materializerSource.includes('console.') || + runtimeFileMaterializer?.securityContext?.allowPrivilegeEscalation !== + false || + runtimeFileMaterializer?.securityContext?.readOnlyRootFilesystem !== true || + JSON.stringify( + runtimeFileMaterializer?.securityContext?.capabilities?.drop, + ) !== JSON.stringify(['ALL']) || + runtimeFileMaterializer?.resources?.requests?.cpu !== '10m' || + runtimeFileMaterializer?.resources?.requests?.memory !== '32Mi' || + runtimeFileMaterializer?.resources?.limits?.cpu !== '100m' || + runtimeFileMaterializer?.resources?.limits?.memory !== '64Mi' || + runtimeFileMaterializer?.volumeMounts?.length !== 2 || + projectedMount?.mountPath !== + '/var/run/secrets/qinglong3/postgres-runtime-projected' || + projectedMount?.readOnly !== true || + materializedMount?.mountPath !== + '/var/run/secrets/qinglong3/postgres-runtime' || + materializedMount?.readOnly === true || + projectedVolume?.secret?.secretName !== 'ql3-cluster-control-runtime' || + projectedVolume?.secret?.defaultMode !== 0o444 || + JSON.stringify(projectedVolume?.secret?.items) !== JSON.stringify([ { key: 'postgres-ca.crt', path: 'ca.crt' }, { @@ -865,12 +913,14 @@ function assertKubernetes(readFile, root, findings) { path: 'api-credential-pepper-keyring.json', }, ]) || + privateVolume?.emptyDir?.medium !== 'Memory' || + privateVolume?.emptyDir?.sizeLimit !== '1Mi' || env.has('QL3_API_CREDENTIAL_PEPPER') ) { findings.push( finding( 'QL3_CLUSTER_KUBERNETES_POSTGRES_CA_BINDING', - 'runtime PostgreSQL trust must use the reviewed read-only projected CA file', + 'runtime trust and API credential keyring must be copied from one projected Secret generation into bounded Pod-private regular files before the control process starts', ), ); } diff --git a/scripts/ql3-deployment-lock-contract.cjs b/scripts/ql3-deployment-lock-contract.cjs index fe1e7840..9b4954ed 100644 --- a/scripts/ql3-deployment-lock-contract.cjs +++ b/scripts/ql3-deployment-lock-contract.cjs @@ -28,7 +28,7 @@ const IMAGE_NAMES = Object.freeze({ worker: 'qinglong3-worker', }); const EXPECTED_SOURCE_SURFACES = Object.freeze({ - control: 2, + control: 3, 'control-ai': 1, admin: 28, worker: 2, diff --git a/scripts/ql3-security-administration-kubernetes-live-contract.cjs b/scripts/ql3-security-administration-kubernetes-live-contract.cjs index d7018474..9381f0e9 100644 --- a/scripts/ql3-security-administration-kubernetes-live-contract.cjs +++ b/scripts/ql3-security-administration-kubernetes-live-contract.cjs @@ -731,6 +731,27 @@ async function runCustodyEvidence({ } } +function runtimeFileMaterializationSource({ + postgresDirectory = '/var/run/secrets/qinglong3/postgres-projected', + keyringDirectory = '/var/run/secrets/qinglong3/api-credential-projected', + targetDirectory = '/var/run/secrets/qinglong3/runtime', +} = {}) { + for (const value of [ + postgresDirectory, + keyringDirectory, + targetDirectory, + ]) { + assert.equal(path.isAbsolute(value), true); + } + return [ + "const fs=require('node:fs')", + "const path=require('node:path')", + `const target=${JSON.stringify(targetDirectory)}`, + `const files=[[${JSON.stringify(postgresDirectory + '/..data')},'ca.crt'],[${JSON.stringify(keyringDirectory + '/..data')},'keyring.json']]`, + "for(const [directory,name] of files){const source=fs.realpathSync(directory);const output=path.join(target,name);fs.copyFileSync(path.join(source,name),output,fs.constants.COPYFILE_EXCL);fs.chmodSync(output,0o400)}", + ].join(';'); +} + function clusterControlResources(controlImage) { const labels = Object.freeze({ 'app.kubernetes.io/name': CONTROL_NAME, @@ -803,7 +824,7 @@ function clusterControlResources(controlImage) { { name: 'QL3_POSTGRES_TLS_MODE', value: 'verify-full' }, { name: 'QL3_POSTGRES_TLS_CA_FILE', - value: '/var/run/secrets/qinglong3/postgres/ca.crt', + value: '/var/run/secrets/qinglong3/runtime/ca.crt', }, { name: 'QL3_POSTGRES_TLS_SERVERNAME', @@ -834,7 +855,7 @@ function clusterControlResources(controlImage) { }, { name: 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE', - value: '/var/run/secrets/qinglong3/api-credential/keyring.json', + value: '/var/run/secrets/qinglong3/runtime/keyring.json', }, ], ports: [{ name: 'http', containerPort: 5800 }], @@ -857,15 +878,45 @@ function clusterControlResources(controlImage) { volumeMounts: [ { name: 'tmp', mountPath: '/tmp' }, { - name: 'postgres-ca', - mountPath: '/var/run/secrets/qinglong3/postgres', + name: 'runtime-private', + mountPath: '/var/run/secrets/qinglong3/runtime', + readOnly: true, + }, + ], + }], + initContainers: [{ + name: 'materialize-runtime-files', + image: controlImage, + imagePullPolicy: 'Never', + command: [ + 'node', + '-e', + runtimeFileMaterializationSource(), + ], + securityContext: { + allowPrivilegeEscalation: false, + readOnlyRootFilesystem: true, + capabilities: { drop: ['ALL'] }, + }, + resources: { + requests: { cpu: '10m', memory: '32Mi' }, + limits: { cpu: '100m', memory: '64Mi' }, + }, + volumeMounts: [ + { + name: 'postgres-ca-projected', + mountPath: '/var/run/secrets/qinglong3/postgres-projected', readOnly: true, }, { - name: 'api-credential-keyring', - mountPath: '/var/run/secrets/qinglong3/api-credential', + name: 'api-credential-keyring-projected', + mountPath: '/var/run/secrets/qinglong3/api-credential-projected', readOnly: true, }, + { + name: 'runtime-private', + mountPath: '/var/run/secrets/qinglong3/runtime', + }, ], }], volumes: [ @@ -874,7 +925,7 @@ function clusterControlResources(controlImage) { emptyDir: { medium: 'Memory', sizeLimit: '16Mi' }, }, { - name: 'postgres-ca', + name: 'postgres-ca-projected', secret: { secretName: 'ql3-postgres-ca', defaultMode: 292, @@ -882,7 +933,7 @@ function clusterControlResources(controlImage) { }, }, { - name: 'api-credential-keyring', + name: 'api-credential-keyring-projected', secret: { secretName: CONTROL_RUNTIME_SECRET, defaultMode: 292, @@ -892,6 +943,10 @@ function clusterControlResources(controlImage) { }], }, }, + { + name: 'runtime-private', + emptyDir: { medium: 'Memory', sizeLimit: '1Mi' }, + }, ], }, }, @@ -2353,6 +2408,7 @@ if (require.main === module) { module.exports = { auditListCommand, clusterControlResources, + runtimeFileMaterializationSource, controlRolloutFailureEvidence, controlTerminationFact, credentialAuthenticationProbeSource, diff --git a/test/back/ql3ClusterDeploymentAudit.test.cjs b/test/back/ql3ClusterDeploymentAudit.test.cjs index 1f8bd182..95efc4e0 100644 --- a/test/back/ql3ClusterDeploymentAudit.test.cjs +++ b/test/back/ql3ClusterDeploymentAudit.test.cjs @@ -47,6 +47,44 @@ test('accepts the exact locked non-root multi-replica cluster deployment', () => ]); }); +test('rejects exposing symlink-backed runtime keyring projections to Cluster Control', () => { + const withoutMaterializer = auditClusterDeployment({ + root: ROOT, + readFile: intercept( + 'deploy/kubernetes/ql3-cluster/base/deployment.yaml', + (source) => source.replace( + ' - name: materialize-runtime-files\n', + ' - name: materialize-runtime-files-disabled\n', + ), + ), + }); + assert.equal(withoutMaterializer.compatible, false); + assert.equal( + withoutMaterializer.findings.some( + ({ code }) => code === 'QL3_CLUSTER_KUBERNETES_POSTGRES_CA_BINDING', + ), + true, + ); + + const projectedIntoRuntime = auditClusterDeployment({ + root: ROOT, + readFile: intercept( + 'deploy/kubernetes/ql3-cluster/base/deployment.yaml', + (source) => source.replace( + ' - name: postgres-runtime-private\n mountPath: /var/run/secrets/qinglong3/postgres-runtime\n readOnly: true\n', + ' - name: postgres-runtime-projected\n mountPath: /var/run/secrets/qinglong3/postgres-runtime\n readOnly: true\n', + ), + ), + }); + assert.equal(projectedIntoRuntime.compatible, false); + assert.equal( + projectedIntoRuntime.findings.some( + ({ code }) => code === 'QL3_CLUSTER_KUBERNETES_POSTGRES_CA_BINDING', + ), + true, + ); +}); + test('keeps Cluster Copilot MCP external, digest-pinned and resource-bounded', () => { const widened = auditClusterDeployment({ root: ROOT, @@ -237,7 +275,7 @@ test('keeps Cluster AI optional with projected authority and an independent dige readFile: intercept( 'deploy/kubernetes/ql3-cluster/base/deployment.yaml', (source) => - source.replace( + source.replaceAll( `image: qinglong3-cluster-control:${VERSION}`, `image: qinglong3-cluster-control-ai:${VERSION}`, ), @@ -1261,7 +1299,7 @@ test('rejects inline runtime credentials and a privileged container', () => { /valueFrom:\n\s+secretKeyRef:\n\s+name: ql3-cluster-control-runtime\n\s+key: postgres-runtime-url/, 'value: postgresql://inline-secret', ) - .replace( + .replaceAll( 'allowPrivilegeEscalation: false', 'allowPrivilegeEscalation: true', ), diff --git a/test/back/ql3DeploymentLockContract.test.cjs b/test/back/ql3DeploymentLockContract.test.cjs index 2e33d303..e6c686e6 100644 --- a/test/back/ql3DeploymentLockContract.test.cjs +++ b/test/back/ql3DeploymentLockContract.test.cjs @@ -921,7 +921,7 @@ test('source-surface audit freezes every reviewed cluster and worker authority', schemaVersion: 1, deploymentYamlFiles: 241, imageOccurrences: { - control: 2, + control: 3, 'control-ai': 1, admin: 28, worker: 2, diff --git a/test/back/ql3SecurityAdministrationKubernetesLiveContract.test.cjs b/test/back/ql3SecurityAdministrationKubernetesLiveContract.test.cjs index f998c32b..e2a779b6 100644 --- a/test/back/ql3SecurityAdministrationKubernetesLiveContract.test.cjs +++ b/test/back/ql3SecurityAdministrationKubernetesLiveContract.test.cjs @@ -1,6 +1,7 @@ const assert = require('node:assert/strict'); const { spawnSync } = require('node:child_process'); const fs = require('node:fs'); +const os = require('node:os'); const path = require('node:path'); const { test } = require('node:test'); @@ -20,6 +21,7 @@ const { inputAuthorityEvidenceSource, migrationFailureEvidence, networkPolicyReadinessSource, + runtimeFileMaterializationSource, } = require('../../scripts/ql3-security-administration-kubernetes-live-contract.cjs'); const values = Object.freeze({ @@ -205,6 +207,21 @@ test('runs the credential ceremony against two real anti-affine control replicas deployment.spec.template.spec.containers[0].terminationMessagePolicy, 'FallbackToLogsOnError', ); + const materializer = deployment.spec.template.spec.initContainers[0]; + assert.equal(materializer.name, 'materialize-runtime-files'); + assert.deepEqual(materializer.command.slice(0, 2), ['node', '-e']); + assert.match(materializer.command[2], /realpathSync/); + assert.match(materializer.command[2], /COPYFILE_EXCL/); + assert.match(materializer.command[2], /chmodSync\(output,0o400\)/); + assert.equal(materializer.securityContext.runAsNonRoot, undefined); + assert.equal(materializer.securityContext.readOnlyRootFilesystem, true); + assert.deepEqual(materializer.securityContext.capabilities.drop, ['ALL']); + assert.equal( + deployment.spec.template.spec.containers[0].volumeMounts.some( + (mount) => mount.name.includes('projected'), + ), + false, + ); assert.equal( deployment.spec.template.spec.affinity.podAntiAffinity .requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey, @@ -215,7 +232,7 @@ test('runs the credential ceremony against two real anti-affine control replicas environment.some( (entry) => entry.name === 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE' && - entry.value.endsWith('/keyring.json'), + entry.value === '/var/run/secrets/qinglong3/runtime/keyring.json', ), ); assert.equal( @@ -224,6 +241,44 @@ test('runs the credential ceremony against two real anti-affine control replicas ); }); +test('materializes kubelet symlink projections as private regular files', (context) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-runtime-files-')); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const postgresDirectory = path.join(directory, 'postgres-projected'); + const keyringDirectory = path.join(directory, 'keyring-projected'); + const targetDirectory = path.join(directory, 'runtime'); + for (const [projectedDirectory, name, value] of [ + [postgresDirectory, 'ca.crt', 'test-ca'], + [keyringDirectory, 'keyring.json', '{"schemaVersion":1}'], + ]) { + const generation = path.join(projectedDirectory, '..2026_08_26'); + fs.mkdirSync(generation, { recursive: true }); + fs.writeFileSync(path.join(generation, name), value); + fs.symlinkSync('..2026_08_26', path.join(projectedDirectory, '..data')); + } + fs.mkdirSync(targetDirectory); + const result = spawnSync( + process.execPath, + ['-e', runtimeFileMaterializationSource({ + postgresDirectory, + keyringDirectory, + targetDirectory, + })], + { encoding: 'utf8' }, + ); + assert.equal(result.status, 0, result.stderr); + for (const [name, value] of [ + ['ca.crt', 'test-ca'], + ['keyring.json', '{"schemaVersion":1}'], + ]) { + const output = path.join(targetDirectory, name); + assert.equal(fs.lstatSync(output).isFile(), true); + assert.equal(fs.lstatSync(output).isSymbolicLink(), false); + assert.equal(fs.statSync(output).mode & 0o777, 0o400); + assert.equal(fs.readFileSync(output, 'utf8'), value); + } +}); + test('keeps failed control rollout evidence bounded and content-free', () => { const failure = JSON.stringify({ schemaVersion: 1, diff --git a/test/back/ql3VersionTransition.test.cjs b/test/back/ql3VersionTransition.test.cjs index 153c1d26..1e44e32e 100644 --- a/test/back/ql3VersionTransition.test.cjs +++ b/test/back/ql3VersionTransition.test.cjs @@ -103,8 +103,8 @@ test('audits one source-derived QingLong 3 release identity', () => { workspacePackageCount: 18, containerRootCount: 4, deploymentFileCount: 260, - deploymentImageReferences: 34, - deploymentVersionOccurrences: 38, + deploymentImageReferences: 35, + deploymentVersionOccurrences: 39, compatible: true, }); }); @@ -116,7 +116,7 @@ test('plans the exact governed version surface without touching legacy 2.x', () targetVersion: TARGET_VERSION, }); assert.equal(plan.fileCount, 66); - assert.equal(plan.replacementCount, 85); + assert.equal(plan.replacementCount, 86); assert.equal(plan.legacyRootPackageVersion, LEGACY_VERSION); assert.equal(plan.legacyRootExcluded, true); assert.equal(