open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-20 16:07:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): validate cluster credential key rotation

Browse Source
This commit is contained in:
whyour
2026-08-26 03:31:13 +08:00
parent b45a5e04b7
commit ac0992081e
23 changed files with 1119 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ql3-cloudnativepg-barman-live-contract.cjs
+28 -6
View File
@@ -676,7 +676,14 @@ function postgresRestoreApplicationProbeResources({
kind: 'List',
items: Object.freeze([
kubernetesSecret('ql3-dr', secretName, {
'api-credential-pepper': apiCredentialPepper,
'api-credential-pepper-keyring.json': `${JSON.stringify({
schemaVersion: 1,
activePepperKeyId: 'legacy-v1',
keys: [{
pepperKeyId: 'legacy-v1',
pepper: apiCredentialPepper,
}],
})}\n`,
}),
{
apiVersion: 'apps/v1',
@@ -753,11 +760,10 @@ function postgresRestoreApplicationProbeResources({
value: `qinglong3-dr-${suffix}`,
},
{ name: 'QL3_POSTGRES_MAX_CONNECTIONS', value: '2' },
secretValue(
'QL3_API_CREDENTIAL_PEPPER',
secretName,
'api-credential-pepper',
),
{
name: 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE',
value: '/var/run/secrets/qinglong3/api-credential/keyring.json',
},
],
ports: [{ name: 'http', containerPort: 5800 }],
startupProbe: {
@@ -794,6 +800,11 @@ function postgresRestoreApplicationProbeResources({
mountPath: '/var/run/secrets/qinglong3/postgres',
readOnly: true,
},
{
name: 'api-credential-keyring',
mountPath: '/var/run/secrets/qinglong3/api-credential',
readOnly: true,
},
],
},
],
@@ -810,6 +821,17 @@ function postgresRestoreApplicationProbeResources({
items: [{ key: 'ca.crt', path: 'ca.crt' }],
},
},
{
name: 'api-credential-keyring',
secret: {
secretName,
defaultMode: 288,
items: [{
key: 'api-credential-pepper-keyring.json',
path: 'keyring.json',
}],
},
},
],
},
},
scripts/ql3-cloudnativepg-deployment-audit.cjs
+4 -3
View File
@@ -354,13 +354,13 @@ function assertSecretBoundary(readFile, root, findings) {
if (name === 'ql3-cluster-control-runtime') {
if (
secret?.type !== 'Opaque' ||
secret?.stringData?.['api-credential-pepper'] !==
'REPLACE_WITH_CANONICAL_32_BYTE_BASE64URL'
secret?.stringData?.['api-credential-pepper-keyring.json'] !==
'{"schemaVersion":1,"activePepperKeyId":"REPLACE_WITH_ACTIVE_KEY_ID","keys":[{"pepperKeyId":"REPLACE_WITH_ACTIVE_KEY_ID","pepper":"REPLACE_WITH_CANONICAL_32_BYTE_BASE64URL"}]}\n'
) {
findings.push(
finding(
'QL3_CNPG_RUNTIME_SECRET_EXAMPLE',
'runtime Secret example may contain only the placeholder credential pepper',
'runtime Secret example must contain only the bounded placeholder credential pepper keyring',
),
);
}
@@ -503,6 +503,7 @@ function assertMigrationBinding(readFile, root, findings) {
env.has('QL3_POSTGRES_MIGRATION_URL') ||
env.has('QL3_POSTGRES_RUNTIME_URL') ||
env.has('QL3_API_CREDENTIAL_PEPPER') ||
env.has('QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE') ||
env.get('QL3_POSTGRES_MIGRATION_HOST')?.value !== PRIMARY_DNS ||
env.get('QL3_POSTGRES_MIGRATION_PORT')?.value !== '5432' ||
env.get('QL3_POSTGRES_MIGRATION_DATABASE')?.value !== 'qinglong' ||
scripts/ql3-cloudnativepg-live-contract.cjs
+5 -1
View File
@@ -760,7 +760,11 @@ async function main() {
);
}
applySecret('ql3-cluster-control-runtime', 'Opaque', {
'api-credential-pepper': randomSecret(),
'api-credential-pepper-keyring.json': `${JSON.stringify({
schemaVersion: 1,
activePepperKeyId: 'legacy-v1',
keys: [{ pepperKeyId: 'legacy-v1', pepper: randomSecret() }],
})}\n`,
});
const workerIngressTls = createWorkerIngressTls(tempDirectory);
applySecret('ql3-cluster-worker-ingress', 'Opaque', {
scripts/ql3-cluster-deployment-audit.cjs
+12 -2
View File
@@ -796,6 +796,10 @@ function assertKubernetes(readFile, root, findings) {
'QL3_POSTGRES_TLS_CA_FILE',
'/var/run/secrets/qinglong3/postgres-runtime/ca.crt',
],
[
'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE',
'/var/run/secrets/qinglong3/postgres-runtime/api-credential-pepper-keyring.json',
],
['QL3_WORKER_INGRESS_ENABLED', 'true'],
['QL3_WORKER_INGRESS_HOST', '0.0.0.0'],
['QL3_WORKER_INGRESS_PORT', '5801'],
@@ -854,7 +858,14 @@ function assertKubernetes(readFile, root, findings) {
runtimeCaVolume?.secret?.secretName !== 'ql3-cluster-control-runtime' ||
runtimeCaVolume?.secret?.defaultMode !== 0o444 ||
JSON.stringify(runtimeCaVolume?.secret?.items) !==
JSON.stringify([{ key: 'postgres-ca.crt', path: 'ca.crt' }])
JSON.stringify([
{ key: 'postgres-ca.crt', path: 'ca.crt' },
{
key: 'api-credential-pepper-keyring.json',
path: 'api-credential-pepper-keyring.json',
},
]) ||
env.has('QL3_API_CREDENTIAL_PEPPER')
) {
findings.push(
finding(
@@ -866,7 +877,6 @@ function assertKubernetes(readFile, root, findings) {
for (const [name, key] of [
['QL3_POSTGRES_RUNTIME_URL', 'postgres-runtime-url'],
['QL3_POSTGRES_TLS_SERVERNAME', 'postgres-tls-servername'],
['QL3_API_CREDENTIAL_PEPPER', 'api-credential-pepper'],
]) {
const secret = env.get(name)?.valueFrom?.secretKeyRef;
if (secret?.name !== 'ql3-cluster-control-runtime' || secret?.key !== key) {
scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs
+29 -9
View File
@@ -1483,7 +1483,14 @@ function applyRuntimeAfterRecovery(recoveryJob, migrationJobValue, secrets) {
type: 'Opaque',
stringData: {
'postgres-runtime-url': `postgresql://ql3_runtime:${secrets.runtime}@${POSTGRES_NAME}:5432/qinglong`,
'api-credential-pepper': randomBytes(32).toString('base64url'),
'api-credential-pepper-keyring.json': `${JSON.stringify({
schemaVersion: 1,
activePepperKeyId: 'legacy-v1',
keys: [{
pepperKeyId: 'legacy-v1',
pepper: randomBytes(32).toString('base64url'),
}],
})}\n`,
},
},
'create runtime-only credential after recovery success',
@@ -1535,21 +1542,34 @@ function applyRuntimeAfterRecovery(recoveryJob, migrationJobValue, secrets) {
},
},
{
name: 'QL3_API_CREDENTIAL_PEPPER',
valueFrom: {
secretKeyRef: {
name: 'ql3-cluster-control-runtime',
key: 'api-credential-pepper',
},
},
name: 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE',
value: '/var/run/secrets/qinglong3/api-credential/keyring.json',
},
];
container.volumeMounts = [
{ name: 'tmp', mountPath: '/tmp' },
{
name: 'api-credential-keyring',
mountPath: '/var/run/secrets/qinglong3/api-credential',
readOnly: true,
},
];
container.volumeMounts = [{ name: 'tmp', mountPath: '/tmp' }];
resource.spec.template.spec.volumes = [
{
name: 'tmp',
emptyDir: { medium: 'Memory', sizeLimit: '16Mi' },
},
{
name: 'api-credential-keyring',
secret: {
secretName: 'ql3-cluster-control-runtime',
defaultMode: 288,
items: [{
key: 'api-credential-pepper-keyring.json',
path: 'keyring.json',
}],
},
},
];
}
for (const resource of resources) {
scripts/ql3-security-administration-kubernetes-audit.cjs
+8 -3
View File
@@ -150,7 +150,7 @@ function auditSecurityAdministrationKubernetes(options = {}) {
`--command=${PRIVATE_ROOT}/input/command.json`,
`--assertion=${PRIVATE_ROOT}/input/assertion.jwt`,
`--keyset=${PRIVATE_ROOT}/input/keyset.json`,
`--pepper=${PRIVATE_ROOT}/input/pepper`,
`--pepper-keyring=${PRIVATE_ROOT}/input/pepper-keyring.json`,
];
if (
JSON.stringify(stager?.command) !==
@@ -181,7 +181,7 @@ function auditSecurityAdministrationKubernetes(options = {}) {
'command.json',
'assertion.jwt',
'keyset.json',
'pepper',
'pepper-keyring.json',
]) ||
privateInput?.emptyDir?.medium !== 'Memory' ||
privateInput?.emptyDir?.sizeLimit !== '1Mi' ||
@@ -241,7 +241,12 @@ function auditSecurityAdministrationKubernetes(options = {}) {
inputExample?.metadata?.name !== 'ql3-security-administration-input' ||
JSON.stringify(Object.keys(inputExample?.stringData ?? {}).sort()) !==
JSON.stringify(
['command.json', 'assertion.jwt', 'keyset.json', 'pepper'].sort(),
[
'command.json',
'assertion.jwt',
'keyset.json',
'pepper-keyring.json',
].sort(),
) ||
JSON.stringify(aggregate).includes('security-administration')
) {
scripts/ql3-security-administration-kubernetes-live-audit.cjs
+72 -15
View File
@@ -13,6 +13,7 @@ const LIMITATIONS = Object.freeze([
'CloudNativePG inside one Docker host is not infrastructure STONITH or disaster-recovery evidence',
'the local-path ReadWriteOnce volume is not encrypted production CSI custody evidence',
'a dedicated root storage-fixture Job constrains the local-path volume root before every non-root administration Job',
'the credential probe reaches the production HTTP surface inside the isolated cluster and is not external ingress TLS evidence',
]);
const BANNED_KEYS = new Set([
'assertion',
@@ -143,6 +144,7 @@ function validateSecurityAdministrationKubernetesLiveReport(report) {
'architecture',
'kubernetesImageId',
'administrationImageId',
'controlImageId',
'cniName',
'cniDistributionBinding',
'controlPlaneNodes',
@@ -154,6 +156,7 @@ function validateSecurityAdministrationKubernetesLiveReport(report) {
!['amd64', 'arm64'].includes(platform?.architecture) ||
!isSha256(platform?.kubernetesImageId) ||
!isSha256(platform?.administrationImageId) ||
!isSha256(platform?.controlImageId) ||
platform?.cniName !== 'flannel' ||
platform?.cniDistributionBinding !== 'rancher/k3s:v1.34.3-k3s1' ||
platform?.controlPlaneNodes !== 1 ||
@@ -213,6 +216,10 @@ function validateSecurityAdministrationKubernetesLiveReport(report) {
'operations',
'completedJobs',
'failedJobs',
'authenticationProbeJobs',
'controlReplicas',
'controlRollouts',
'controlReplicaAntiAffinity',
'callerDriven',
'backoffLimit',
'activeDeadlineSeconds',
@@ -221,19 +228,39 @@ function validateSecurityAdministrationKubernetesLiveReport(report) {
'serviceAccountTokenMounted',
'rbacGranted',
'responseLossReplayObserved',
'overlapGenerationCount',
'contractedGenerationCount',
'activeGenerationChanged',
'oldReferencesBeforeActivation',
'oldReferencesAfterActivation',
'oldReferencesAfterConvergence',
'oldAuthenticationBeforeActivation',
'oldAuthenticationDuringOverlap',
'newAuthenticationDuringOverlap',
'oldAuthenticationRejectedAfterConvergence',
'newAuthenticationAfterContraction',
'contractedToActiveGeneration',
'sensitiveMaterialReported',
]) ||
JSON.stringify(ceremony?.operations) !==
JSON.stringify([
'identity.register',
'audit.list',
'credential.issue',
'credential.issue.replay',
'credential.rotate',
'credential.revoke',
'credential.issue.old',
'credential.issue.old.replay',
'credential.key-references.before-activate',
'credential.issue.new',
'credential.rotate.new',
'credential.key-references.after-activate',
'credential.revoke.old',
'credential.key-references.after-converge',
]) ||
ceremony?.completedJobs !== 6 ||
ceremony?.completedJobs !== 10 ||
ceremony?.failedJobs !== 1 ||
ceremony?.authenticationProbeJobs !== 5 ||
ceremony?.controlReplicas !== 2 ||
ceremony?.controlRollouts !== 3 ||
ceremony?.controlReplicaAntiAffinity !== true ||
ceremony?.callerDriven !== true ||
ceremony?.backoffLimit !== 0 ||
ceremony?.activeDeadlineSeconds !== 300 ||
@@ -242,12 +269,24 @@ function validateSecurityAdministrationKubernetesLiveReport(report) {
ceremony?.serviceAccountTokenMounted !== false ||
ceremony?.rbacGranted !== false ||
ceremony?.responseLossReplayObserved !== true ||
ceremony?.overlapGenerationCount !== 2 ||
ceremony?.contractedGenerationCount !== 1 ||
ceremony?.activeGenerationChanged !== true ||
ceremony?.oldReferencesBeforeActivation !== 1 ||
ceremony?.oldReferencesAfterActivation !== 1 ||
ceremony?.oldReferencesAfterConvergence !== 0 ||
ceremony?.oldAuthenticationBeforeActivation !== true ||
ceremony?.oldAuthenticationDuringOverlap !== true ||
ceremony?.newAuthenticationDuringOverlap !== true ||
ceremony?.oldAuthenticationRejectedAfterConvergence !== true ||
ceremony?.newAuthenticationAfterContraction !== true ||
ceremony?.contractedToActiveGeneration !== true ||
ceremony?.sensitiveMaterialReported !== false
) {
findings.push(
finding(
'QL3_SECURITY_ADMINISTRATION_KUBERNETES_LIVE_CEREMONY',
'six serial caller-created commands plus one failed input stage must use the exact tokenless Job contract',
'ten serial caller-created commands, five content-free authentication probes and one failed input stage must prove the exact two-replica overlap, activation, convergence and contraction contract',
),
);
}
@@ -299,7 +338,7 @@ function validateSecurityAdministrationKubernetesLiveReport(report) {
delivery?.fixtureProvisionerRanAsRoot !== true ||
delivery?.privateDirectoryMode !== '0700' ||
delivery?.fileMode !== '0600' ||
delivery?.fileCount !== 2 ||
delivery?.fileCount !== 3 ||
!isSha256(delivery?.issueDigest) ||
!isSha256(delivery?.rotationDigest) ||
delivery?.issueDigest === delivery?.rotationDigest ||
@@ -342,28 +381,42 @@ function validateSecurityAdministrationKubernetesLiveReport(report) {
!exactKeys(durability, [
'identityVersion',
'identityStatus',
'credentialVersion',
'credentialState',
'oldCredentialVersion',
'oldCredentialState',
'newCredentialVersion',
'newCredentialState',
'identityMutationCount',
'credentialMutationCount',
'issueMutationCount',
'credentialVersionCount',
'oldGenerationVersionCount',
'newGenerationVersionCount',
'latestGenerationsAreNew',
'allowedAuditCount',
'authenticationDeniedAuditCount',
'authenticationRejectedAuditCount',
]) ||
durability?.identityVersion !== 1 ||
durability?.identityStatus !== 'active' ||
durability?.credentialVersion !== 3 ||
durability?.credentialState !== 'revoked' ||
durability?.oldCredentialVersion !== 2 ||
durability?.oldCredentialState !== 'revoked' ||
durability?.newCredentialVersion !== 2 ||
durability?.newCredentialState !== 'active' ||
durability?.identityMutationCount !== 1 ||
durability?.credentialMutationCount !== 3 ||
durability?.credentialMutationCount !== 4 ||
durability?.issueMutationCount !== 1 ||
durability?.credentialVersionCount !== 3 ||
durability?.allowedAuditCount !== 4
durability?.credentialVersionCount !== 4 ||
durability?.oldGenerationVersionCount !== 1 ||
durability?.newGenerationVersionCount !== 3 ||
durability?.latestGenerationsAreNew !== true ||
durability?.allowedAuditCount !== 5 ||
durability?.authenticationDeniedAuditCount !== 4 ||
durability?.authenticationRejectedAuditCount !== 1
) {
findings.push(
finding(
'QL3_SECURITY_ADMINISTRATION_KUBERNETES_LIVE_DURABILITY',
'the database must retain one identity mutation and exactly three credential generations without replay duplication',
'the database must retain one old-key version followed by three new-key versions across two credentials, exact replay and the expected authentication audits',
),
);
}
@@ -375,6 +428,9 @@ function validateSecurityAdministrationKubernetesLiveReport(report) {
'evidenceJobsDeleted',
'storageProvisionJobDeleted',
'deliveryVolumeClaimDeleted',
'controlDeploymentDeleted',
'controlServiceDeleted',
'controlRuntimeSecretDeleted',
])
) {
findings.push(
@@ -392,6 +448,7 @@ function validateSecurityAdministrationKubernetesLiveReport(report) {
'realKubeletSecretProjection',
'realAdministrationProductCommands',
'realPersistentCredentialCustody',
'realClusterControlAuthenticationRotation',
'responseLossReplay',
'failedInputStageClosed',
'leastPrivilege',
scripts/ql3-security-administration-kubernetes-live-contract.cjs
+742 -78
View File
File diff suppressed because it is too large Load Diff
scripts/ql3-worker-kubernetes-rollout-live-contract.cjs
+49 -7
View File
@@ -1617,6 +1617,25 @@ async function main(argv = process.argv.slice(2)) {
'tls.crt': pkiMaterial.oldClientCertificate,
},
});
apply({
apiVersion: 'v1',
kind: 'Secret',
metadata: {
name: 'ql3-worker-control-live-runtime',
namespace: NAMESPACE,
},
type: 'Opaque',
stringData: {
'api-credential-pepper-keyring.json': `${JSON.stringify({
schemaVersion: 1,
activePepperKeyId: 'legacy-v1',
keys: [{
pepperKeyId: 'legacy-v1',
pepper: Buffer.alloc(32, 29).toString('base64url'),
}],
})}\n`,
},
});
const workerArchitecture = remoteWorkerArchitectureForNodeRuntime(
process.arch, process.config.variables.arm_version,
);
@@ -1700,8 +1719,8 @@ async function main(argv = process.argv.slice(2)) {
{ name: 'QL3_POSTGRES_ALLOW_INSECURE', value: 'true' },
{ name: 'QL3_POSTGRES_MAX_CONNECTIONS', value: '2' },
{
name: 'QL3_API_CREDENTIAL_PEPPER',
value: Buffer.alloc(32, 29).toString('base64url'),
name: 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE',
value: '/var/run/secrets/qinglong3/api-credential/keyring.json',
},
{ name: 'QL3_WORKER_INGRESS_ENABLED', value: 'true' },
{ name: 'QL3_WORKER_INGRESS_HOST', value: '0.0.0.0' },
@@ -1738,12 +1757,35 @@ async function main(argv = process.argv.slice(2)) {
timeoutSeconds: 1,
failureThreshold: 20,
},
volumeMounts: [{ name: 'tls', mountPath: '/tls', readOnly: true }],
}],
volumes: [{
name: 'tls',
secret: { secretName: 'ql3-worker-ingress-tls-live', defaultMode: 288 },
volumeMounts: [
{ name: 'tls', mountPath: '/tls', readOnly: true },
{
name: 'api-credential-keyring',
mountPath: '/var/run/secrets/qinglong3/api-credential',
readOnly: true,
},
],
}],
volumes: [
{
name: 'tls',
secret: {
secretName: 'ql3-worker-ingress-tls-live',
defaultMode: 288,
},
},
{
name: 'api-credential-keyring',
secret: {
secretName: 'ql3-worker-control-live-runtime',
defaultMode: 288,
items: [{
key: 'api-credential-pepper-keyring.json',
path: 'keyring.json',
}],
},
},
],
},
},
},