open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-20 16:07:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(ql3): materialize projected runtime files

Browse Source
This commit is contained in:
whyour
2026-08-26 04:09:47 +08:00
parent 9c58b15d2b
commit beb490c48c
9 changed files with 273 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/kubernetes/ql3-cluster/README.md
+6 -2
View File
@@ -967,8 +967,12 @@ and is fixed to `verify-full` in the committed deployment. The servername is
mandatory, must be an explicit DNS name rather than an IP literal, and must
match the endpoint certificate SAN. Only
`postgres-ca.crt` and `api-credential-pepper-keyring.json` are projected from
this Secret into the runtime private mount; only the URL and servername remain
environment values. The keyring is a canonical, bounded 1–2 generation file;
this Secret only into a hardened init container. It resolves one kubelet
`..data` generation, copies both values without replacement into a Pod-private
memory volume, and changes each regular file to `0400`; the long-running
container can read only that materialized volume, never the symlink-backed
Secret projection. Only the URL and servername remain environment values. The
keyring is a canonical, bounded 1–2 generation file;
the singleton above is the bootstrap form. The CA loader requires an absolute
path to a regular file that is not group/world writable, 1–256 KiB, and
contains 1–16 unique PEM X.509 CA certificates with no trailing data.
deploy/kubernetes/ql3-cluster/base/deployment.yaml
+42 -2
View File
@@ -45,6 +45,42 @@ spec:
matchLabels:
app.kubernetes.io/name: ql3-cluster-control
app.kubernetes.io/component: control-plane
initContainers:
- name: materialize-runtime-files
image: qinglong3-cluster-control:3.0.0-alpha.1
imagePullPolicy: IfNotPresent
command:
- node
- -e
- |
const fs = require('node:fs');
const path = require('node:path');
const source = fs.realpathSync('/var/run/secrets/qinglong3/postgres-runtime-projected/..data');
const target = '/var/run/secrets/qinglong3/postgres-runtime';
for (const name of ['ca.crt', 'api-credential-pepper-keyring.json']) {
const output = path.join(target, name);
fs.copyFileSync(path.join(source, name), output, fs.constants.COPYFILE_EXCL);
fs.chmodSync(output, 0o400);
}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
resources:
requests:
cpu: 10m
memory: 32Mi
limits:
cpu: 100m
memory: 64Mi
volumeMounts:
- name: postgres-runtime-projected
mountPath: /var/run/secrets/qinglong3/postgres-runtime-projected
readOnly: true
- name: postgres-runtime-private
mountPath: /var/run/secrets/qinglong3/postgres-runtime
containers:
- name: cluster-control
image: qinglong3-cluster-control:3.0.0-alpha.1
@@ -223,7 +259,7 @@ spec:
volumeMounts:
- name: tmp
mountPath: /tmp
- name: postgres-runtime-ca
- name: postgres-runtime-private
mountPath: /var/run/secrets/qinglong3/postgres-runtime
readOnly: true
- name: postgres-worker-ingress-ca
@@ -240,7 +276,7 @@ spec:
emptyDir:
medium: Memory
sizeLimit: 16Mi
- name: postgres-runtime-ca
- name: postgres-runtime-projected
secret:
secretName: ql3-cluster-control-runtime
defaultMode: 292
@@ -249,6 +285,10 @@ spec:
path: ca.crt
- key: api-credential-pepper-keyring.json
path: api-credential-pepper-keyring.json
- name: postgres-runtime-private
emptyDir:
medium: Memory
sizeLimit: 1Mi
- name: postgres-worker-ingress-ca
secret:
secretName: ql3-cluster-worker-ingress
scripts/ql3-cluster-deployment-audit.cjs
+60 -10
View File
@@ -846,18 +846,66 @@ function assertKubernetes(readFile, root, findings) {
),
);
}
const runtimeCaMount = namedEntry(
container?.volumeMounts,
'postgres-runtime-ca',
const runtimeFileMaterializer = namedEntry(
pod?.initContainers,
'materialize-runtime-files',
);
const runtimeCaVolume = namedEntry(pod?.volumes, 'postgres-runtime-ca');
const runtimePrivateMount = namedEntry(
container?.volumeMounts,
'postgres-runtime-private',
);
const projectedMount = namedEntry(
runtimeFileMaterializer?.volumeMounts,
'postgres-runtime-projected',
);
const materializedMount = namedEntry(
runtimeFileMaterializer?.volumeMounts,
'postgres-runtime-private',
);
const projectedVolume = namedEntry(
pod?.volumes,
'postgres-runtime-projected',
);
const privateVolume = namedEntry(pod?.volumes, 'postgres-runtime-private');
const materializerSource = runtimeFileMaterializer?.command?.[2];
if (
runtimeCaMount?.mountPath !==
runtimePrivateMount?.mountPath !==
'/var/run/secrets/qinglong3/postgres-runtime' ||
runtimeCaMount?.readOnly !== true ||
runtimeCaVolume?.secret?.secretName !== 'ql3-cluster-control-runtime' ||
runtimeCaVolume?.secret?.defaultMode !== 0o444 ||
JSON.stringify(runtimeCaVolume?.secret?.items) !==
runtimePrivateMount?.readOnly !== true ||
container?.volumeMounts?.some(
(mount) => mount?.name === 'postgres-runtime-projected',
) ||
runtimeFileMaterializer?.image !== container?.image ||
runtimeFileMaterializer?.imagePullPolicy !== container?.imagePullPolicy ||
JSON.stringify(runtimeFileMaterializer?.command?.slice(0, 2)) !==
JSON.stringify(['node', '-e']) ||
typeof materializerSource !== 'string' ||
!materializerSource.includes(
"realpathSync('/var/run/secrets/qinglong3/postgres-runtime-projected/..data')",
) ||
!materializerSource.includes('COPYFILE_EXCL') ||
!materializerSource.includes('chmodSync(output, 0o400)') ||
materializerSource.includes('console.') ||
runtimeFileMaterializer?.securityContext?.allowPrivilegeEscalation !==
false ||
runtimeFileMaterializer?.securityContext?.readOnlyRootFilesystem !== true ||
JSON.stringify(
runtimeFileMaterializer?.securityContext?.capabilities?.drop,
) !== JSON.stringify(['ALL']) ||
runtimeFileMaterializer?.resources?.requests?.cpu !== '10m' ||
runtimeFileMaterializer?.resources?.requests?.memory !== '32Mi' ||
runtimeFileMaterializer?.resources?.limits?.cpu !== '100m' ||
runtimeFileMaterializer?.resources?.limits?.memory !== '64Mi' ||
runtimeFileMaterializer?.volumeMounts?.length !== 2 ||
projectedMount?.mountPath !==
'/var/run/secrets/qinglong3/postgres-runtime-projected' ||
projectedMount?.readOnly !== true ||
materializedMount?.mountPath !==
'/var/run/secrets/qinglong3/postgres-runtime' ||
materializedMount?.readOnly === true ||
projectedVolume?.secret?.secretName !== 'ql3-cluster-control-runtime' ||
projectedVolume?.secret?.defaultMode !== 0o444 ||
JSON.stringify(projectedVolume?.secret?.items) !==
JSON.stringify([
{ key: 'postgres-ca.crt', path: 'ca.crt' },
{
@@ -865,12 +913,14 @@ function assertKubernetes(readFile, root, findings) {
path: 'api-credential-pepper-keyring.json',
},
]) ||
privateVolume?.emptyDir?.medium !== 'Memory' ||
privateVolume?.emptyDir?.sizeLimit !== '1Mi' ||
env.has('QL3_API_CREDENTIAL_PEPPER')
) {
findings.push(
finding(
'QL3_CLUSTER_KUBERNETES_POSTGRES_CA_BINDING',
'runtime PostgreSQL trust must use the reviewed read-only projected CA file',
'runtime trust and API credential keyring must be copied from one projected Secret generation into bounded Pod-private regular files before the control process starts',
),
);
}
scripts/ql3-deployment-lock-contract.cjs
+1 -1
View File
@@ -28,7 +28,7 @@ const IMAGE_NAMES = Object.freeze({
worker: 'qinglong3-worker',
});
const EXPECTED_SOURCE_SURFACES = Object.freeze({
control: 2,
control: 3,
'control-ai': 1,
admin: 28,
worker: 2,
scripts/ql3-security-administration-kubernetes-live-contract.cjs
+64 -8
View File
@@ -731,6 +731,27 @@ async function runCustodyEvidence({
}
}
function runtimeFileMaterializationSource({
postgresDirectory = '/var/run/secrets/qinglong3/postgres-projected',
keyringDirectory = '/var/run/secrets/qinglong3/api-credential-projected',
targetDirectory = '/var/run/secrets/qinglong3/runtime',
} = {}) {
for (const value of [
postgresDirectory,
keyringDirectory,
targetDirectory,
]) {
assert.equal(path.isAbsolute(value), true);
}
return [
"const fs=require('node:fs')",
"const path=require('node:path')",
`const target=${JSON.stringify(targetDirectory)}`,
`const files=[[${JSON.stringify(postgresDirectory + '/..data')},'ca.crt'],[${JSON.stringify(keyringDirectory + '/..data')},'keyring.json']]`,
"for(const [directory,name] of files){const source=fs.realpathSync(directory);const output=path.join(target,name);fs.copyFileSync(path.join(source,name),output,fs.constants.COPYFILE_EXCL);fs.chmodSync(output,0o400)}",
].join(';');
}
function clusterControlResources(controlImage) {
const labels = Object.freeze({
'app.kubernetes.io/name': CONTROL_NAME,
@@ -803,7 +824,7 @@ function clusterControlResources(controlImage) {
{ name: 'QL3_POSTGRES_TLS_MODE', value: 'verify-full' },
{
name: 'QL3_POSTGRES_TLS_CA_FILE',
value: '/var/run/secrets/qinglong3/postgres/ca.crt',
value: '/var/run/secrets/qinglong3/runtime/ca.crt',
},
{
name: 'QL3_POSTGRES_TLS_SERVERNAME',
@@ -834,7 +855,7 @@ function clusterControlResources(controlImage) {
},
{
name: 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE',
value: '/var/run/secrets/qinglong3/api-credential/keyring.json',
value: '/var/run/secrets/qinglong3/runtime/keyring.json',
},
],
ports: [{ name: 'http', containerPort: 5800 }],
@@ -857,15 +878,45 @@ function clusterControlResources(controlImage) {
volumeMounts: [
{ name: 'tmp', mountPath: '/tmp' },
{
name: 'postgres-ca',
mountPath: '/var/run/secrets/qinglong3/postgres',
name: 'runtime-private',
mountPath: '/var/run/secrets/qinglong3/runtime',
readOnly: true,
},
],
}],
initContainers: [{
name: 'materialize-runtime-files',
image: controlImage,
imagePullPolicy: 'Never',
command: [
'node',
'-e',
runtimeFileMaterializationSource(),
],
securityContext: {
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
capabilities: { drop: ['ALL'] },
},
resources: {
requests: { cpu: '10m', memory: '32Mi' },
limits: { cpu: '100m', memory: '64Mi' },
},
volumeMounts: [
{
name: 'postgres-ca-projected',
mountPath: '/var/run/secrets/qinglong3/postgres-projected',
readOnly: true,
},
{
name: 'api-credential-keyring',
mountPath: '/var/run/secrets/qinglong3/api-credential',
name: 'api-credential-keyring-projected',
mountPath: '/var/run/secrets/qinglong3/api-credential-projected',
readOnly: true,
},
{
name: 'runtime-private',
mountPath: '/var/run/secrets/qinglong3/runtime',
},
],
}],
volumes: [
@@ -874,7 +925,7 @@ function clusterControlResources(controlImage) {
emptyDir: { medium: 'Memory', sizeLimit: '16Mi' },
},
{
name: 'postgres-ca',
name: 'postgres-ca-projected',
secret: {
secretName: 'ql3-postgres-ca',
defaultMode: 292,
@@ -882,7 +933,7 @@ function clusterControlResources(controlImage) {
},
},
{
name: 'api-credential-keyring',
name: 'api-credential-keyring-projected',
secret: {
secretName: CONTROL_RUNTIME_SECRET,
defaultMode: 292,
@@ -892,6 +943,10 @@ function clusterControlResources(controlImage) {
}],
},
},
{
name: 'runtime-private',
emptyDir: { medium: 'Memory', sizeLimit: '1Mi' },
},
],
},
},
@@ -2353,6 +2408,7 @@ if (require.main === module) {
module.exports = {
auditListCommand,
clusterControlResources,
runtimeFileMaterializationSource,
controlRolloutFailureEvidence,
controlTerminationFact,
credentialAuthenticationProbeSource,
test/back/ql3ClusterDeploymentAudit.test.cjs
+40 -2
View File
@@ -47,6 +47,44 @@ test('accepts the exact locked non-root multi-replica cluster deployment', () =>
]);
});
test('rejects exposing symlink-backed runtime keyring projections to Cluster Control', () => {
const withoutMaterializer = auditClusterDeployment({
root: ROOT,
readFile: intercept(
'deploy/kubernetes/ql3-cluster/base/deployment.yaml',
(source) => source.replace(
' - name: materialize-runtime-files\n',
' - name: materialize-runtime-files-disabled\n',
),
),
});
assert.equal(withoutMaterializer.compatible, false);
assert.equal(
withoutMaterializer.findings.some(
({ code }) => code === 'QL3_CLUSTER_KUBERNETES_POSTGRES_CA_BINDING',
),
true,
);
const projectedIntoRuntime = auditClusterDeployment({
root: ROOT,
readFile: intercept(
'deploy/kubernetes/ql3-cluster/base/deployment.yaml',
(source) => source.replace(
' - name: postgres-runtime-private\n mountPath: /var/run/secrets/qinglong3/postgres-runtime\n readOnly: true\n',
' - name: postgres-runtime-projected\n mountPath: /var/run/secrets/qinglong3/postgres-runtime\n readOnly: true\n',
),
),
});
assert.equal(projectedIntoRuntime.compatible, false);
assert.equal(
projectedIntoRuntime.findings.some(
({ code }) => code === 'QL3_CLUSTER_KUBERNETES_POSTGRES_CA_BINDING',
),
true,
);
});
test('keeps Cluster Copilot MCP external, digest-pinned and resource-bounded', () => {
const widened = auditClusterDeployment({
root: ROOT,
@@ -237,7 +275,7 @@ test('keeps Cluster AI optional with projected authority and an independent dige
readFile: intercept(
'deploy/kubernetes/ql3-cluster/base/deployment.yaml',
(source) =>
source.replace(
source.replaceAll(
`image: qinglong3-cluster-control:${VERSION}`,
`image: qinglong3-cluster-control-ai:${VERSION}`,
),
@@ -1261,7 +1299,7 @@ test('rejects inline runtime credentials and a privileged container', () => {
/valueFrom:\n\s+secretKeyRef:\n\s+name: ql3-cluster-control-runtime\n\s+key: postgres-runtime-url/,
'value: postgresql://inline-secret',
)
.replace(
.replaceAll(
'allowPrivilegeEscalation: false',
'allowPrivilegeEscalation: true',
),
test/back/ql3DeploymentLockContract.test.cjs
+1 -1
View File
@@ -921,7 +921,7 @@ test('source-surface audit freezes every reviewed cluster and worker authority',
schemaVersion: 1,
deploymentYamlFiles: 241,
imageOccurrences: {
control: 2,
control: 3,
'control-ai': 1,
admin: 28,
worker: 2,
test/back/ql3SecurityAdministrationKubernetesLiveContract.test.cjs
+56 -1
View File
@@ -1,6 +1,7 @@
const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { test } = require('node:test');
@@ -20,6 +21,7 @@ const {
inputAuthorityEvidenceSource,
migrationFailureEvidence,
networkPolicyReadinessSource,
runtimeFileMaterializationSource,
} = require('../../scripts/ql3-security-administration-kubernetes-live-contract.cjs');
const values = Object.freeze({
@@ -205,6 +207,21 @@ test('runs the credential ceremony against two real anti-affine control replicas
deployment.spec.template.spec.containers[0].terminationMessagePolicy,
'FallbackToLogsOnError',
);
const materializer = deployment.spec.template.spec.initContainers[0];
assert.equal(materializer.name, 'materialize-runtime-files');
assert.deepEqual(materializer.command.slice(0, 2), ['node', '-e']);
assert.match(materializer.command[2], /realpathSync/);
assert.match(materializer.command[2], /COPYFILE_EXCL/);
assert.match(materializer.command[2], /chmodSync\(output,0o400\)/);
assert.equal(materializer.securityContext.runAsNonRoot, undefined);
assert.equal(materializer.securityContext.readOnlyRootFilesystem, true);
assert.deepEqual(materializer.securityContext.capabilities.drop, ['ALL']);
assert.equal(
deployment.spec.template.spec.containers[0].volumeMounts.some(
(mount) => mount.name.includes('projected'),
),
false,
);
assert.equal(
deployment.spec.template.spec.affinity.podAntiAffinity
.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey,
@@ -215,7 +232,7 @@ test('runs the credential ceremony against two real anti-affine control replicas
environment.some(
(entry) =>
entry.name === 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE' &&
entry.value.endsWith('/keyring.json'),
entry.value === '/var/run/secrets/qinglong3/runtime/keyring.json',
),
);
assert.equal(
@@ -224,6 +241,44 @@ test('runs the credential ceremony against two real anti-affine control replicas
);
});
test('materializes kubelet symlink projections as private regular files', (context) => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-runtime-files-'));
context.after(() => fs.rmSync(directory, { recursive: true, force: true }));
const postgresDirectory = path.join(directory, 'postgres-projected');
const keyringDirectory = path.join(directory, 'keyring-projected');
const targetDirectory = path.join(directory, 'runtime');
for (const [projectedDirectory, name, value] of [
[postgresDirectory, 'ca.crt', 'test-ca'],
[keyringDirectory, 'keyring.json', '{"schemaVersion":1}'],
]) {
const generation = path.join(projectedDirectory, '..2026_08_26');
fs.mkdirSync(generation, { recursive: true });
fs.writeFileSync(path.join(generation, name), value);
fs.symlinkSync('..2026_08_26', path.join(projectedDirectory, '..data'));
}
fs.mkdirSync(targetDirectory);
const result = spawnSync(
process.execPath,
['-e', runtimeFileMaterializationSource({
postgresDirectory,
keyringDirectory,
targetDirectory,
})],
{ encoding: 'utf8' },
);
assert.equal(result.status, 0, result.stderr);
for (const [name, value] of [
['ca.crt', 'test-ca'],
['keyring.json', '{"schemaVersion":1}'],
]) {
const output = path.join(targetDirectory, name);
assert.equal(fs.lstatSync(output).isFile(), true);
assert.equal(fs.lstatSync(output).isSymbolicLink(), false);
assert.equal(fs.statSync(output).mode & 0o777, 0o400);
assert.equal(fs.readFileSync(output, 'utf8'), value);
}
});
test('keeps failed control rollout evidence bounded and content-free', () => {
const failure = JSON.stringify({
schemaVersion: 1,
test/back/ql3VersionTransition.test.cjs
+3 -3
View File
@@ -103,8 +103,8 @@ test('audits one source-derived QingLong 3 release identity', () => {
workspacePackageCount: 18,
containerRootCount: 4,
deploymentFileCount: 260,
deploymentImageReferences: 34,
deploymentVersionOccurrences: 38,
deploymentImageReferences: 35,
deploymentVersionOccurrences: 39,
compatible: true,
});
});
@@ -116,7 +116,7 @@ test('plans the exact governed version surface without touching legacy 2.x', ()
targetVersion: TARGET_VERSION,
});
assert.equal(plan.fileCount, 66);
assert.equal(plan.replacementCount, 85);
assert.equal(plan.replacementCount, 86);
assert.equal(plan.legacyRootPackageVersion, LEGACY_VERSION);
assert.equal(plan.legacyRootExcluded, true);
assert.equal(