const assert = require('node:assert/strict'); const crypto = require('node:crypto'); const { spawnSync } = require('node:child_process'); const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); const { afterEach, test } = require('node:test'); const { CRL_ANNOTATION, REVIEWED_AUTHORITY, WorkerCredentialManagementPkiRotationEvidenceError, inspectClientConfiguration, inspectClientIssuerAuthority, parseCrlInspectionOutput, parseArguments, runAfterEvidence, runBeforeEvidence, validateBeforeState, validateWorkerCredentialManagementPkiRotationEvidence, } = require('../../scripts/ql3-worker-credential-management-pki-rotation-evidence.cjs'); const NOW_MS = 1_700_000_000_000; const ISSUER = 'https://identity.production.example.org/'; const temporaryDirectories = []; function sha(character) { return `sha256:${character.repeat(64)}`; } function digest(domain, value) { return `sha256:${crypto .createHash('sha256') .update(domain) .update('\0') .update(String(value)) .digest('hex')}`; } function rawDigest(value) { return `sha256:${crypto.createHash('sha256').update(value).digest('hex')}`; } function encode(value) { return Buffer.from(JSON.stringify(value)).toString('base64url'); } function assertionToken(subject = 'operator-a') { const now = Math.floor(NOW_MS / 1000); return `${encode({ alg: 'EdDSA', kid: `key-${subject}`, typ: 'ql3-worker-credential-management+jwt', })}.${encode({ acr: 'urn:production:mfa', amr: ['pwd', 'otp'], aud: 'qinglong3-worker-credential-management', auth_time: now - 20, exp: now + 120, iat: now - 10, iss: ISSUER, jti: `session-${subject}`, ql3_purpose: 'worker-credential-management', sub: subject, })}.${Buffer.alloc(64, 7).toString('base64url')}`; } function ceremonyReport() { return { schemaVersion: 1, fixture: 'qinglong/worker-credential-management-live-ceremony@v1', observedAt: new Date(NOW_MS).toISOString(), identity: { providerKind: 'external_oidc', issuer: ISSUER, discoveryDocumentSha256: sha('c'), jwksSha256: sha('d'), audience: 'qinglong3-worker-credential-management', type: 'ql3-worker-credential-management+jwt', purpose: 'worker-credential-management', requesterSubjectSha256: digest( 'qinglong3.worker-management.subject.v1', 'operator-a', ), reviewerSubjectSha256: digest( 'qinglong3.worker-management.subject.v1', 'reviewer-b', ), requesterKeyIdSha256: sha('1'), reviewerKeyIdSha256: sha('2'), }, ceremony: { actionRefSha256: sha('3'), authorityProjectIdSha256: sha('4'), planStatus: 'created', approvalStatus: 'created', requesterSelfDecisionStatus: 403, requesterSelfDecisionCode: 'forbidden', reviewerDecisionStatus: 'decided', approvalState: 'approved', inspectionStale: false, dispatchCreated: false, approvalConsumed: false, planDigest: 'a'.repeat(64), previewDigest: 'b'.repeat(64), requestIdSha256: ['5', '6', '7', '8', '9'].map(sha), }, gates: { externalIdentity: true, workerPurposeBound: true, requesterAndReviewerDistinct: true, requesterSelfDecisionRejected: true, reviewerDecisionAccepted: true, inspectionAuthorized: true, noExecutionOrConsumption: true, passed: true, }, }; } function durableReport(ceremonyBytes, ceremony) { return { schemaVersion: 1, fixture: 'qinglong/worker-credential-management-durable-audit-evidence@v1', observedAt: new Date(NOW_MS + 30).toISOString(), source: { ceremonyReportSha256: rawDigest(ceremonyBytes), ceremonyDefinitionSha256: sha('e'), ceremonyFixture: ceremony.fixture, }, database: { postgresVersionNumber: 180004, transactionReadOnly: true, roleNameSha256: sha('f'), roleCanLogin: true, privilegedAttributesDenied: true, privilegedMembershipDenied: true, exactTargetSelect: true, ql3TableMutationDenied: true, }, durableState: { actionRefSha256: ceremony.ceremony.actionRefSha256, authorityProjectIdSha256: ceremony.ceremony.authorityProjectIdSha256, approvalRequestIdSha256: sha('0'), reviewerDecisionIdSha256: sha('1'), planDigest: ceremony.ceremony.planDigest, previewDigest: ceremony.ceremony.previewDigest, approvalVersion: 2, approvalState: 'approved', requesterSubjectSha256: ceremony.identity.requesterSubjectSha256, reviewerSubjectSha256: ceremony.identity.reviewerSubjectSha256, dispatchCreated: false, approvalConsumed: false, requesterSelfDecisionAuditAbsent: true, auditRows: [ { kind: 'proposal', eventIdSha256: sha('2'), operationId: 'approval.request', outcome: 'approval_required', subjectSha256: ceremony.identity.requesterSubjectSha256, authenticationIdSha256: sha('3'), reasonCode: 'worker_credential_review', policyFencePresent: true, }, { kind: 'reviewer_decision', eventIdSha256: sha('4'), operationId: 'approval.decide', outcome: 'allowed', subjectSha256: ceremony.identity.reviewerSubjectSha256, authenticationIdSha256: sha('5'), reasonCode: 'worker_credential_review', policyFencePresent: true, }, ], }, gates: { sourceBound: true, readOnlyEvidenceRole: true, immutablePlanObserved: true, reviewedApprovalObserved: true, requesterSelfDecisionLeftNoAudit: true, proposalAndDecisionAuditObserved: true, noExecutionOrConsumption: true, passed: true, }, }; } function writeFile(directory, name, value, raw = false) { const filePath = path.join(directory, name); fs.writeFileSync(filePath, raw ? value : `${JSON.stringify(value)}\n`, { mode: 0o600, }); return filePath; } function runOpenSsl(args) { const result = spawnSync('openssl', args, { encoding: 'utf8' }); assert.equal(result.status, 0, result.stderr); } function fixture() { const directory = fs.realpathSync( fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-pki-rotation-test-')), ); fs.chmodSync(directory, 0o700); temporaryDirectories.push(directory); const ceremony = ceremonyReport(); const ceremonyReportFile = writeFile( directory, 'ceremony-report.json', ceremony, ); const ceremonyBytes = fs.readFileSync(ceremonyReportFile); return { directory, oldConfigFile: writeFile(directory, 'old-client.json', {}), newConfigFile: writeFile(directory, 'new-client.json', {}), assertionFile: writeFile(directory, 'operator.jwt', assertionToken(), true), commandFile: writeFile(directory, 'inspect-command.json', { schemaVersion: 1, operation: 'worker-credential.inspect', }), kubernetesFile: writeFile(directory, 'kubeconfig.json', {}), issuerCaFile: writeFile(directory, 'client-issuer-ca.pem', 'issuer', true), beforeCrlFile: writeFile(directory, 'before.crl', 'before-crl', true), afterCrlFile: writeFile(directory, 'after.crl', 'after-crl', true), ceremonyReportFile, durableAuditReportFile: writeFile( directory, 'durable-report.json', durableReport(ceremonyBytes, ceremony), ), beforeFile: path.join(directory, 'before-state.json'), outputFile: path.join(directory, 'rotation-evidence.json'), }; } function kubernetesSnapshot(crlSha256, phase, options = {}) { const generation = phase === 'before' ? 12 : 13; const suffix = phase === 'before' ? 'old' : 'new'; const podUids = options.reuseOldPod ? ['pod-old-a', 'pod-new-b'] : [`pod-${suffix}-a`, `pod-${suffix}-b`]; return { clusterServerSha256: sha('6'), collectorSubjectSha256: sha('7'), authorization: REVIEWED_AUTHORITY.map((entry) => ({ ...entry, observed: options.allowSecretRead && entry.verb === 'get' && entry.resource === 'secrets' ? true : entry.allowed, })), deployment: { apiVersion: 'apps/v1', kind: 'Deployment', metadata: { name: 'ql3-worker-credential-management', namespace: 'qinglong3-system', uid: 'deployment-a', resourceVersion: `resource-${generation}`, generation: options.sameGeneration ? 12 : generation, }, spec: { replicas: 2, strategy: { type: 'RollingUpdate', rollingUpdate: { maxUnavailable: 0 }, }, template: { metadata: { annotations: { [CRL_ANNOTATION]: crlSha256 } }, }, }, status: { observedGeneration: options.sameGeneration ? 12 : generation, replicas: 2, updatedReplicas: 2, readyReplicas: 2, availableReplicas: 2, }, }, pods: { apiVersion: 'v1', kind: 'List', items: podUids.map((uid, index) => ({ metadata: { namespace: 'qinglong3-system', uid, labels: { 'app.kubernetes.io/name': 'ql3-worker-credential-management', 'app.kubernetes.io/component': 'worker-credential-management', 'pod-template-hash': `template-${suffix}`, }, }, spec: { serviceAccountName: 'ql3-worker-credential-management', automountServiceAccountToken: false, nodeName: `node-${index + 1}`, }, status: { phase: 'Running', conditions: [{ type: 'Ready', status: 'True' }], containerStatuses: [{ name: 'management', ready: true }], }, })), }, }; } function harness(options = {}) { const state = { phase: 'before', calls: [] }; const beforeCrl = { sha256: sha('8'), issuerSha256: sha('9'), number: '1000', lastUpdateMs: NOW_MS - 1_000, nextUpdateMs: NOW_MS + 600_000, }; const afterCrl = { ...beforeCrl, sha256: sha('a'), number: options.nonMonotonicCrl ? '1000' : '1001', lastUpdateMs: NOW_MS + 1, }; return { state, dependencies: { useDefaults: false, now: () => NOW_MS, normalize(command) { return command; }, inspectClient(configFile) { const client = path.basename(configFile).startsWith('old') ? 'old' : 'new'; return { endpointSha256: sha('b'), servernameSha256: sha('c'), serverTrustBundleSha256: sha('d'), serverAuthoritySha256: [sha('e')], clientCertificateSha256: client === 'old' ? sha('f') : sha('0'), certificate: { checkIssued(certificate) { return certificate.id === 'client-issuer'; }, verify(publicKey) { return publicKey.id === 'client-issuer'; }, }, }; }, inspectIssuer() { const id = options.wrongIssuer ? 'wrong-issuer' : 'client-issuer'; return { bundleSha256: sha('1'), certificateSha256: sha('2'), subjectSha256: sha('9'), certificate: { id, publicKey: { id } }, }; }, inspectAuthoritySubject() { throw new Error('unused in injected inspectors'); }, inspectCrl(bytes) { return bytes.toString() === 'before-crl' ? beforeCrl : afterCrl; }, async collectKubernetes(_file, crlSha256) { return kubernetesSnapshot(crlSha256, state.phase, options); }, async execute(paths) { const client = path.basename(paths.configFile).startsWith('old') ? 'old' : 'new'; state.calls.push(`${state.phase}:${client}`); if (state.phase === 'after' && client === 'old') { throw Object.assign(new Error('rejected'), { statusCode: 401, responseCode: 'client_certificate_required', }); } return { requestId: `request-${state.phase}-${client}` }; }, }, }; } function beforeOptions(paths) { return { oldConfigFile: paths.oldConfigFile, newConfigFile: paths.newConfigFile, assertionFile: paths.assertionFile, commandFile: paths.commandFile, kubernetesFile: paths.kubernetesFile, issuerCaFile: paths.issuerCaFile, crlFile: paths.beforeCrlFile, outputFile: paths.beforeFile, }; } function afterOptions(paths) { return { beforeFile: paths.beforeFile, oldConfigFile: paths.oldConfigFile, newConfigFile: paths.newConfigFile, assertionFile: paths.assertionFile, commandFile: paths.commandFile, kubernetesFile: paths.kubernetesFile, issuerCaFile: paths.issuerCaFile, crlFile: paths.afterCrlFile, ceremonyReportFile: paths.ceremonyReportFile, durableAuditReportFile: paths.durableAuditReportFile, outputFile: paths.outputFile, }; } afterEach(() => { for (const directory of temporaryDirectories.splice(0)) { fs.rmSync(directory, { recursive: true, force: true }); } }); test('proves a complete CRL-bound rollout and revoked certificate rejection', async () => { const paths = fixture(); const runtime = harness(); const before = await runBeforeEvidence( beforeOptions(paths), runtime.dependencies, ); assert.equal(validateBeforeState(before).compatible, true); runtime.state.phase = 'after'; const report = await runAfterEvidence( afterOptions(paths), runtime.dependencies, ); assert.deepEqual(runtime.state.calls, [ 'before:old', 'before:new', 'after:old', 'after:new', ]); assert.equal(report.transport.afterOldStatus, 401); assert.equal(report.transport.afterNewStatus, 200); assert.equal(report.schemaVersion, 2); assert.equal(report.transport.serverTrustBundleSha256, sha('d')); assert.equal(report.pki.clientIssuerCaSha256, sha('2')); assert.notEqual( report.transport.serverAuthoritySha256[0], report.pki.clientIssuerCaSha256, ); assert.equal(report.kubernetes.oldPodsFullyReplaced, true); assert.equal( validateWorkerCredentialManagementPkiRotationEvidence(report).compatible, true, ); assert.equal(fs.statSync(paths.beforeFile).mode & 0o777, 0o600); assert.equal(fs.statSync(paths.outputFile).mode & 0o777, 0o600); assert.doesNotMatch( JSON.stringify(report), /operator-a|reviewer-b|deployment-a|pod-old|pod-new|node-|BEGIN|eyJ/, ); const audit = spawnSync( process.execPath, [ path.resolve( __dirname, '../../scripts/ql3-worker-credential-management-pki-rotation-evidence-audit.cjs', ), `--report=${paths.outputFile}`, ], { encoding: 'utf8' }, ); assert.equal(audit.status, 0, audit.stderr); assert.equal(JSON.parse(audit.stdout).compatible, true); }); test('fails closed when the CRL number does not increase', async () => { const paths = fixture(); const runtime = harness({ nonMonotonicCrl: true }); await runBeforeEvidence(beforeOptions(paths), runtime.dependencies); runtime.state.phase = 'after'; await assert.rejects( runAfterEvidence(afterOptions(paths), runtime.dependencies), /CRL did not advance monotonically/, ); assert.equal(fs.existsSync(paths.outputFile), false); assert.deepEqual(runtime.state.calls, ['before:old', 'before:new']); }); test('fails closed when client certificates are not issued by the explicit client CA', async () => { const paths = fixture(); const runtime = harness({ wrongIssuer: true }); await assert.rejects( runBeforeEvidence(beforeOptions(paths), runtime.dependencies), /not bound to one explicit issuer CA/, ); assert.deepEqual(runtime.state.calls, []); assert.equal(fs.existsSync(paths.beforeFile), false); }); test('keeps management server trust independent from the client certificate issuer', () => { const paths = fixture(); const serverKey = path.join(paths.directory, 'server-ca.key'); const serverCa = path.join(paths.directory, 'server-ca.pem'); const issuerKey = path.join(paths.directory, 'client-ca.key'); const issuerCa = path.join(paths.directory, 'real-client-ca.pem'); const clientKey = path.join(paths.directory, 'client.key'); const clientCsr = path.join(paths.directory, 'client.csr'); const clientCertificate = path.join(paths.directory, 'client.pem'); const extension = writeFile( paths.directory, 'client.ext', 'basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=clientAuth\n', true, ); for (const [key, certificate, subject] of [ [serverKey, serverCa, '/CN=QL3 Management Server Trust'], [issuerKey, issuerCa, '/CN=QL3 Management Client Issuer'], ]) { runOpenSsl([ 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', key, '-out', certificate, '-subj', subject, '-days', '2', '-addext', 'basicConstraints=critical,CA:TRUE', '-addext', 'keyUsage=critical,keyCertSign,cRLSign', ]); } runOpenSsl([ 'req', '-newkey', 'rsa:2048', '-nodes', '-keyout', clientKey, '-out', clientCsr, '-subj', '/CN=QL3 Management Client', ]); runOpenSsl([ 'x509', '-req', '-in', clientCsr, '-CA', issuerCa, '-CAkey', issuerKey, '-CAcreateserial', '-out', clientCertificate, '-days', '1', '-extfile', extension, ]); for (const filePath of [serverKey, issuerKey, clientKey]) { fs.chmodSync(filePath, 0o600); } const configFile = writeFile(paths.directory, 'real-client.json', { schemaVersion: 1, endpoint: 'https://worker-management.production.example.org/api/v3/worker-credentials/management', servername: 'worker-management.production.example.org', caFile: serverCa, clientCertificateFile: clientCertificate, clientPrivateKeyFile: clientKey, requestTimeoutMs: 5_000, }); const profile = inspectClientConfiguration(configFile, Date.now()); const issuer = inspectClientIssuerAuthority(issuerCa, Date.now()); assert.notEqual( profile.serverAuthoritySha256[0], issuer.certificateSha256, ); assert.equal(profile.certificate.checkIssued(issuer.certificate), true); assert.equal(profile.certificate.verify(issuer.certificate.publicKey), true); }); test('fails closed for incomplete pod replacement or an unchanged generation', async () => { for (const changed of [{ reuseOldPod: true }, { sameGeneration: true }]) { const paths = fixture(); const runtime = harness(changed); await runBeforeEvidence(beforeOptions(paths), runtime.dependencies); runtime.state.phase = 'after'; await assert.rejects( runAfterEvidence(afterOptions(paths), runtime.dependencies), /complete old generation/, ); assert.equal(fs.existsSync(paths.outputFile), false); } }); test('rejects any widened Kubernetes evidence authority before client access', async () => { const paths = fixture(); const runtime = harness({ allowSecretRead: true }); await assert.rejects( runBeforeEvidence(beforeOptions(paths), runtime.dependencies), /authority is not exact read-only/, ); assert.deepEqual(runtime.state.calls, []); assert.equal(fs.existsSync(paths.beforeFile), false); }); test('offline audit rejects false gates, widened shape and sensitive material', async () => { const paths = fixture(); const runtime = harness(); await runBeforeEvidence(beforeOptions(paths), runtime.dependencies); runtime.state.phase = 'after'; const report = await runAfterEvidence( afterOptions(paths), runtime.dependencies, ); for (const changed of [ { ...structuredClone(report), extra: true }, { ...structuredClone(report), gates: { ...report.gates, passed: false }, }, { ...structuredClone(report), privateKey: 'not-exportable' }, ]) { assert.equal( validateWorkerCredentialManagementPkiRotationEvidence(changed).compatible, false, ); } }); test('parses only the exact before and after CLI argument sets', () => { const common = [ '--old-config=/private/old.json', '--new-config=/private/new.json', '--assertion=/private/assertion.jwt', '--command=/private/command.json', '--kubernetes=/private/kubernetes.json', '--client-issuer-ca=/private/client-issuer-ca.pem', '--crl=/private/client.crl', '--output=/private/evidence.json', ]; assert.equal(parseArguments(['--phase=before', ...common]).phase, 'before'); assert.equal( parseArguments([ '--phase=after', ...common, '--before=/private/before.json', '--ceremony-report=/private/ceremony.json', '--durable-audit-report=/private/durable.json', ]).phase, 'after', ); assert.throws( () => parseArguments(['--phase=before', ...common, '--extra=true']), WorkerCredentialManagementPkiRotationEvidenceError, ); }); test('parses the exact OpenSSL 3 SHA-256 CRL inspection labels', () => { const fingerprint = Array.from({ length: 32 }, () => 'AB').join(':'); const inspected = parseCrlInspectionOutput( [ `SHA256 Fingerprint=${fingerprint}`, 'issuer=CN=QingLong Worker Management Client CA', 'lastUpdate=Aug 1 00:00:00 2026 GMT', 'nextUpdate=Aug 31 00:00:00 2026 GMT', 'crlNumber=0x1001', ].join('\n'), ); assert.equal(inspected.sha256, `sha256:${'ab'.repeat(32)}`); assert.equal(inspected.number, '1001'); assert.equal(inspected.lastUpdateMs, Date.UTC(2026, 7, 1)); assert.equal(inspected.nextUpdateMs, Date.UTC(2026, 7, 31)); assert.throws( () => parseCrlInspectionOutput( `SHA-256 Fingerprint=${fingerprint}\nissuer=CN=CA\nlastUpdate=Aug 1 00:00:00 2026 GMT\nnextUpdate=Aug 31 00:00:00 2026 GMT\ncrlNumber=0x1001`, ), WorkerCredentialManagementPkiRotationEvidenceError, ); });