test(ql3): audit plugin recovery evidence

This commit is contained in:
whyour
2026-08-14 14:24:29 +08:00
parent 2485d61e8a
commit 1c75ac0f45
9 changed files with 952 additions and 27 deletions
@@ -0,0 +1,508 @@
#!/usr/bin/env node
'use strict';
const fs = require('node:fs');
const path = require('node:path');
const FIXTURE = 'qinglong/plugin-package-recovery-e2e-live-contract@v2';
const LIMITATIONS = Object.freeze([
'isolated PostgreSQL uses explicit TLS disable; production manifests remain verify-full',
'the authenticated HTTPS OCI Distribution fixture implements the immutable GET/referrers surface used by the resolver, not a production registry storage implementation',
'the disposable Kind control plane is single-replica; this gate proves workload ordering, not Kubernetes control-plane HA',
]);
const GATE_KEYS = Object.freeze([
'healthyInitialActivation',
'missingTransitionFailedClosed',
'invalidUpgradeRejectedBeforeActivation',
'activePointerUidUnchanged',
'activePointerResourceVersionUnchanged',
'activePointerJsonUnchanged',
'candidateRevisionAbsent',
'exactAuthenticatedOciRequests',
'recoveryRbacLeastPrivilege',
'runtimeRolledOutAfterRecovery',
'passed',
]);
const BANNED_KEYS = new Set([
'activejson',
'assertion',
'authorization',
'bearer',
'certificate',
'connectionstring',
'credential',
'dsn',
'kubeconfig',
'password',
'privatekey',
'secret',
'tlskey',
'token',
]);
const SHA256 = /^[a-f0-9]{64}$/;
const SHA256_ID = /^sha256:[a-f0-9]{64}$/;
const UUID =
/^[a-f0-9]{8}-[a-f0-9]{4}-[1-8][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/i;
function finding(code, detail) {
return Object.freeze({ code, detail });
}
function exactKeys(value, expected) {
return Boolean(
value &&
typeof value === 'object' &&
!Array.isArray(value) &&
JSON.stringify(Object.keys(value).sort()) ===
JSON.stringify([...expected].sort()),
);
}
function imageDigest(value) {
if (typeof value !== 'string') return undefined;
return value.match(/sha256:[a-f0-9]{64}$/)?.[0];
}
function safeToken(value, maximum = 256) {
return (
typeof value === 'string' &&
value.length > 0 &&
value.length <= maximum &&
/^[A-Za-z0-9][A-Za-z0-9._:/@+-]*$/.test(value)
);
}
function sensitiveMaterial(value, key = '') {
if (BANNED_KEYS.has(key.toLowerCase())) return true;
if (typeof value === 'string') {
return (
/-----BEGIN (?:CERTIFICATE|(?:RSA |EC |OPENSSH )?PRIVATE KEY)-----/.test(
value,
) ||
/postgres(?:ql)?:\/\/[^/\s]+:[^@\s]+@/i.test(value) ||
/\bBasic\s+[A-Za-z0-9+/=_-]{8,}/i.test(value) ||
/\beyJ[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{8,}\./.test(value) ||
/\bqlsecret(?::|\/\/)/i.test(value)
);
}
if (Array.isArray(value)) {
return value.some((entry) => sensitiveMaterial(entry));
}
if (value && typeof value === 'object') {
return Object.entries(value).some(([entryKey, entry]) =>
sensitiveMaterial(entry, entryKey),
);
}
return false;
}
function validIso(value) {
return (
typeof value === 'string' &&
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/.test(value) &&
Number.isFinite(Date.parse(value))
);
}
function validRuntime(report) {
const runtime = report.runtime;
const ordering = report.ordering;
return (
exactKeys(runtime, [
'replicas',
'creationTimestamp',
'recoveryJobUid',
'recoveryCompletedAt',
'nodes',
'imageIds',
]) &&
runtime.replicas === 2 &&
runtime.creationTimestamp === ordering?.runtimeCreatedAt &&
runtime.recoveryJobUid === ordering?.rejectionRecoveryJobUid &&
runtime.recoveryCompletedAt === ordering?.rejectionRecoveryCompletedAt &&
Array.isArray(runtime.nodes) &&
runtime.nodes.length === 2 &&
new Set(runtime.nodes).size === 2 &&
runtime.nodes.every((value) => safeToken(value)) &&
Array.isArray(runtime.imageIds) &&
runtime.imageIds.length === 1 &&
imageDigest(runtime.imageIds[0]) === report.images?.controlBuildId
);
}
function validatePluginPackageRecoveryE2ELiveReport(report) {
const findings = [];
if (
!exactKeys(report, [
'schema',
'observedAt',
'sourceRevision',
'passed',
'cluster',
'architecture',
'elapsedMs',
'images',
'ordering',
'failedUpgrade',
'database',
'oci',
'kubernetes',
'runtime',
'gates',
'limitations',
]) ||
report?.schema !== FIXTURE ||
report?.passed !== true ||
!validIso(report?.observedAt) ||
!/^[a-f0-9]{40}$/.test(report?.sourceRevision ?? '') ||
!/^ql3-plugin-recovery-e2e(?:-[a-z0-9](?:[-a-z0-9]{0,24}[a-z0-9])?)?$/.test(
report?.cluster ?? '',
) ||
!['amd64', 'arm64'].includes(report?.architecture) ||
!Number.isSafeInteger(report?.elapsedMs) ||
report.elapsedMs < 1 ||
report.elapsedMs > 60 * 60 * 1000
) {
findings.push(
finding('QL3_PLUGIN_RECOVERY_E2E_ENVELOPE', 'report envelope is invalid'),
);
}
const images = report?.images;
if (
!exactKeys(images, [
'adminBuildId',
'adminSourceRevision',
'controlBuildId',
'controlSourceRevision',
'postgresRepositoryDigest',
'migrationImageId',
'initialRecoveryImageId',
'stageRecoveryImageId',
'rejectionRecoveryImageId',
'postgresImageId',
]) ||
!SHA256_ID.test(images?.adminBuildId ?? '') ||
!SHA256_ID.test(images?.controlBuildId ?? '') ||
images?.adminSourceRevision !== report?.sourceRevision ||
images?.controlSourceRevision !== report?.sourceRevision ||
images?.postgresRepositoryDigest !==
'postgres@sha256:1961f96e6029a02c3812d7cb329a3b03a3ac2bb067058dec17b0f5596aca9296' ||
!imageDigest(images?.migrationImageId) ||
imageDigest(images?.initialRecoveryImageId) !==
imageDigest(images?.migrationImageId) ||
imageDigest(images?.stageRecoveryImageId) !==
imageDigest(images?.migrationImageId) ||
imageDigest(images?.rejectionRecoveryImageId) !==
imageDigest(images?.migrationImageId) ||
!imageDigest(images?.postgresImageId)
) {
findings.push(
finding('QL3_PLUGIN_RECOVERY_E2E_IMAGES', 'image provenance is invalid'),
);
}
const ordering = report?.ordering;
const timeKeys = [
'migrationCompletedAt',
'initialRecoveryCompletedAt',
'upgradeStageFailedAt',
'transitionCompletedAt',
'rejectionRecoveryCompletedAt',
'runtimeCreatedAt',
];
const uidKeys = [
'migrationJobUid',
'initialRecoveryJobUid',
'upgradeStageJobUid',
'transitionJobUid',
'rejectionRecoveryJobUid',
'runtimeBoundRecoveryJobUid',
];
const times = timeKeys.map((key) => Date.parse(ordering?.[key]));
if (
!exactKeys(ordering, [...uidKeys, ...timeKeys]) ||
uidKeys.some((key) => !UUID.test(ordering?.[key] ?? '')) ||
ordering?.runtimeBoundRecoveryJobUid !==
ordering?.rejectionRecoveryJobUid ||
timeKeys.some((key) => !validIso(ordering?.[key])) ||
times.some((value, index) => index > 0 && value < times[index - 1])
) {
findings.push(
finding(
'QL3_PLUGIN_RECOVERY_E2E_ORDERING',
'Job and rollout ordering is invalid',
),
);
}
const failed = report?.failedUpgrade;
if (
!exactKeys(failed, [
'stageFailure',
'transitionReceiptDigest',
'rejectionReason',
'candidateRevisionCount',
'activePointerUnchanged',
]) ||
!exactKeys(failed?.stageFailure, ['jobUid', 'reason', 'durableState']) ||
failed?.stageFailure?.jobUid !== ordering?.upgradeStageJobUid ||
failed?.stageFailure?.reason !==
'ClusterPluginPackageRecoveryRequiredError' ||
failed?.stageFailure?.durableState !== 'staged' ||
!SHA256.test(failed?.transitionReceiptDigest ?? '') ||
failed?.rejectionReason !== 'activation_fact_conflict' ||
failed?.candidateRevisionCount !== 0 ||
failed?.activePointerUnchanged !== true
) {
findings.push(
finding(
'QL3_PLUGIN_RECOVERY_E2E_FAILED_UPGRADE',
'failed upgrade evidence is invalid',
),
);
}
const database = report?.database;
const databaseKeys = [
'migrationCount',
'capabilityVersion',
'initialState',
'initialActiveLockDigest',
'upgradeState',
'upgradePreviousActiveLockDigest',
'upgradeActiveLockDigest',
'upgradeFailureReason',
'initialMutationCount',
'upgradeMutationCount',
'headInstallationId',
'transitionReceiptCount',
'initialRevisionCount',
'upgradeRevisionCount',
'recoverableCount',
];
if (
!exactKeys(database, databaseKeys) ||
database?.migrationCount !== 65 ||
database?.capabilityVersion !== 64 ||
database?.initialState !== 'active' ||
database?.upgradeState !== 'failed' ||
!SHA256.test(database?.initialActiveLockDigest ?? '') ||
database?.upgradePreviousActiveLockDigest !==
database?.initialActiveLockDigest ||
database?.upgradeActiveLockDigest !== database?.initialActiveLockDigest ||
database?.upgradeFailureReason !== 'activation_fact_conflict' ||
database?.initialMutationCount !== 4 ||
database?.upgradeMutationCount !== 3 ||
database?.headInstallationId !== 'install-plugin-recovery-e2e-upgrade' ||
database?.transitionReceiptCount !== 1 ||
database?.initialRevisionCount !== 1 ||
database?.upgradeRevisionCount !== 0 ||
database?.recoverableCount !== 0
) {
findings.push(
finding(
'QL3_PLUGIN_RECOVERY_E2E_DATABASE',
'durable database evidence is invalid',
),
);
}
const oci = report?.oci;
if (
!exactKeys(oci, [
'https',
'authentication',
'authenticatedRequestCount',
'requestCount',
'uniquePaths',
'initialRequestCount',
'upgradeRequestCount',
'redirects',
]) ||
oci?.https !== true ||
oci?.authentication !== 'exact-registry-basic' ||
oci?.authenticatedRequestCount !== 18 ||
oci?.requestCount !== 18 ||
oci?.uniquePaths !== 12 ||
oci?.initialRequestCount !== 6 ||
oci?.upgradeRequestCount !== 12 ||
oci?.redirects !== 0
) {
findings.push(
finding('QL3_PLUGIN_RECOVERY_E2E_OCI', 'OCI request evidence is invalid'),
);
}
const kubernetes = report?.kubernetes;
const pointer = kubernetes?.activePointer;
const rbac = kubernetes?.rbac;
if (
!exactKeys(kubernetes, ['activePointer', 'rbac']) ||
!exactKeys(pointer, [
'name',
'uid',
'resourceVersion',
'activeJsonDigest',
'intentDigest',
'activationRef',
]) ||
!safeToken(pointer?.name) ||
!UUID.test(pointer?.uid ?? '') ||
!/^[1-9][0-9]*$/.test(pointer?.resourceVersion ?? '') ||
!SHA256.test(pointer?.activeJsonDigest ?? '') ||
!SHA256.test(pointer?.intentDigest ?? '') ||
!safeToken(pointer?.activationRef) ||
!exactKeys(rbac, [
'getConfigMaps',
'createConfigMaps',
'updateConfigMaps',
'listConfigMaps',
'deleteConfigMaps',
'getSecrets',
]) ||
rbac?.getConfigMaps !== true ||
rbac?.createConfigMaps !== true ||
rbac?.updateConfigMaps !== true ||
rbac?.listConfigMaps !== false ||
rbac?.deleteConfigMaps !== false ||
rbac?.getSecrets !== false
) {
findings.push(
finding(
'QL3_PLUGIN_RECOVERY_E2E_KUBERNETES',
'active pointer or RBAC evidence is invalid',
),
);
}
if (!validRuntime(report ?? {})) {
findings.push(
finding(
'QL3_PLUGIN_RECOVERY_E2E_RUNTIME',
'runtime rollout evidence is invalid',
),
);
}
if (
!exactKeys(report?.gates, GATE_KEYS) ||
GATE_KEYS.some((key) => report?.gates?.[key] !== true)
) {
findings.push(
finding(
'QL3_PLUGIN_RECOVERY_E2E_GATES',
'every gate must be explicitly true',
),
);
}
if (
!Array.isArray(report?.limitations) ||
JSON.stringify([...report.limitations].sort()) !==
JSON.stringify([...LIMITATIONS].sort())
) {
findings.push(
finding(
'QL3_PLUGIN_RECOVERY_E2E_LIMITATIONS',
'limitations must remain exact',
),
);
}
if (sensitiveMaterial(report)) {
findings.push(
finding(
'QL3_PLUGIN_RECOVERY_E2E_SENSITIVE',
'report contains a forbidden key or sensitive material',
),
);
}
return Object.freeze({
schemaVersion: 1,
fixture: FIXTURE,
findings: Object.freeze(findings),
compatible: findings.length === 0,
});
}
function readPrivateReport(filePath) {
if (!path.isAbsolute(filePath) || path.resolve(filePath) !== filePath) {
throw new Error('report path must be absolute and canonical');
}
const before = fs.lstatSync(filePath);
if (
!before.isFile() ||
before.isSymbolicLink() ||
before.size < 2 ||
before.size > 1024 * 1024 ||
(before.mode & 0o777) !== 0o600
) {
throw new Error(
'report must be an owner-private regular file between 2 bytes and 1 MiB',
);
}
const descriptor = fs.openSync(
filePath,
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
);
try {
const opened = fs.fstatSync(descriptor);
if (
opened.dev !== before.dev ||
opened.ino !== before.ino ||
opened.size !== before.size ||
(opened.mode & 0o777) !== 0o600
) {
throw new Error('report identity or mode changed while opening');
}
const text = fs.readFileSync(descriptor, 'utf8');
const after = fs.fstatSync(descriptor);
const currentPath = fs.lstatSync(filePath);
if (
after.size !== opened.size ||
after.mtimeMs !== opened.mtimeMs ||
currentPath.dev !== opened.dev ||
currentPath.ino !== opened.ino ||
currentPath.size !== opened.size ||
(currentPath.mode & 0o777) !== 0o600
) {
throw new Error('report changed while reading');
}
return JSON.parse(text);
} finally {
fs.closeSync(descriptor);
}
}
function main(argv) {
if (argv.length !== 1 || !argv[0].startsWith('--report=')) {
throw new Error(
'usage: ql3-plugin-package-recovery-e2e-live-audit --report=/absolute/report.json',
);
}
const result = validatePluginPackageRecoveryE2ELiveReport(
readPrivateReport(argv[0].slice('--report='.length)),
);
process.stdout.write(`${JSON.stringify(result)}\n`);
if (!result.compatible) process.exitCode = 1;
}
if (require.main === module) {
try {
main(process.argv.slice(2));
} catch (error) {
process.stderr.write(
`QL3 Plugin Package recovery E2E live audit failed: ${
error instanceof Error ? error.message : String(error)
}\n`,
);
process.exitCode = 2;
}
}
module.exports = {
FIXTURE,
GATE_KEYS,
LIMITATIONS,
readPrivateReport,
validatePluginPackageRecoveryE2ELiveReport,
};
@@ -6,7 +6,7 @@ const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { randomBytes } = require('node:crypto');
const { createHash, randomBytes } = require('node:crypto');
const { spawnSync } = require('node:child_process');
const yaml = require('js-yaml');
@@ -181,7 +181,62 @@ function imageExists(image) {
);
}
function buildImages() {
function sourceRevision() {
const value = process.env.QL3_SOURCE_REVISION ?? '';
if (!/^[a-f0-9]{40}$/.test(value)) {
fail(
'QL3_SOURCE_REVISION must be the exact lowercase 40-hex source revision',
);
}
return value;
}
function privateReportPath(argv) {
if (
argv.length !== 1 ||
!argv[0].startsWith('--report=') ||
!path.isAbsolute(argv[0].slice('--report='.length))
) {
fail(
'usage: ql3-plugin-package-recovery-e2e-live-contract ' +
'--report=/absolute/private-report.json',
);
}
const reportFile = argv[0].slice('--report='.length);
if (path.resolve(reportFile) !== reportFile) {
fail('Plugin Package recovery E2E report path must be canonical');
}
if (fs.existsSync(reportFile)) {
fail('refusing to overwrite the Plugin Package recovery E2E report');
}
const parent = fs.lstatSync(path.dirname(reportFile));
if (!parent.isDirectory() || parent.isSymbolicLink()) {
fail('Plugin Package recovery E2E report parent must be a real directory');
}
return reportFile;
}
function writePrivateReport(reportFile, report) {
const temporaryReport = path.join(
path.dirname(reportFile),
`.${path.basename(reportFile)}.${process.pid}.` +
`${randomBytes(6).toString('hex')}.tmp`,
);
let descriptor;
try {
descriptor = fs.openSync(temporaryReport, 'wx', 0o600);
fs.writeFileSync(descriptor, `${JSON.stringify(report, null, 2)}\n`);
fs.fsyncSync(descriptor);
fs.closeSync(descriptor);
descriptor = undefined;
fs.linkSync(temporaryReport, reportFile);
} finally {
if (descriptor !== undefined) fs.closeSync(descriptor);
fs.rmSync(temporaryReport, { force: true });
}
}
function buildImages(revision) {
if (process.env.QL3_SKIP_IMAGE_BUILD === '1') {
assert.equal(imageExists(ADMIN_IMAGE), true, `${ADMIN_IMAGE} is absent`);
assert.equal(
@@ -200,7 +255,7 @@ function buildImages() {
'--tag',
ADMIN_IMAGE,
'--build-arg',
'SOURCE_REVISION=ql3-plugin-recovery-e2e-live',
`SOURCE_REVISION=${revision}`,
'.',
],
{ label: 'build current QingLong 3.0 cluster-admin image' },
@@ -214,7 +269,7 @@ function buildImages() {
'--tag',
CONTROL_IMAGE,
'--build-arg',
'SOURCE_REVISION=ql3-plugin-recovery-e2e-live',
`SOURCE_REVISION=${revision}`,
'.',
],
{ label: 'build current QingLong 3.0 cluster-control image' },
@@ -248,6 +303,15 @@ function imageId(image) {
)[0].Id;
}
function imageSourceRevision(image) {
return JSON.parse(
run(DOCKER, ['image', 'inspect', image], {
capture: true,
quiet: true,
}).stdout,
)[0].Config?.Labels?.['org.opencontainers.image.revision'];
}
function createRegistryCertificate(root) {
const caKey = path.join(root, 'registry-ca.key');
const caCert = path.join(root, 'registry-ca.crt');
@@ -1264,6 +1328,19 @@ function activePointerEvidence(fixture) {
});
}
function reportActivePointer(pointer) {
return Object.freeze({
name: pointer.name,
uid: pointer.uid,
resourceVersion: pointer.resourceVersion,
activeJsonDigest: createHash('sha256')
.update(pointer.activeJson, 'utf8')
.digest('hex'),
intentDigest: pointer.intentDigest,
activationRef: pointer.activationRef,
});
}
function registryEvidence(fixture) {
const output = kubectl(
['-n', NAMESPACE, 'logs', REGISTRY_NAME, '-c', 'registry'],
@@ -1542,12 +1619,14 @@ function diagnostics() {
if (snapshot.stdout) process.stderr.write(`${snapshot.stdout}\n`);
}
async function main() {
async function main(argv = process.argv.slice(2)) {
const reportFile = privateReportPath(argv);
if (process.env.QL3_PLUGIN_PACKAGE_RECOVERY_E2E_LIVE !== '1') {
fail(
'Refusing to create a live cluster without QL3_PLUGIN_PACKAGE_RECOVERY_E2E_LIVE=1',
);
}
const revision = sourceRevision();
const clusterName = exactClusterName();
const existing = kind(['get', 'clusters'], {
capture: true,
@@ -1605,7 +1684,13 @@ async function main() {
let created = false;
const startedAt = Date.now();
try {
buildImages();
buildImages(revision);
const adminSourceRevision = imageSourceRevision(ADMIN_IMAGE);
const controlSourceRevision = imageSourceRevision(CONTROL_IMAGE);
assert.equal(adminSourceRevision, revision);
assert.equal(controlSourceRevision, revision);
const adminBuildId = imageId(ADMIN_IMAGE);
const controlBuildId = imageId(CONTROL_IMAGE);
ensurePostgresImage();
fs.writeFileSync(
kindConfig,
@@ -1829,13 +1914,17 @@ async function main() {
]);
const report = Object.freeze({
schema: REPORT_SCHEMA,
observedAt: new Date().toISOString(),
sourceRevision: revision,
passed: true,
cluster: clusterName,
architecture: fixtureArchitecture,
elapsedMs: Date.now() - startedAt,
images: Object.freeze({
adminBuildId: imageId(ADMIN_IMAGE),
controlBuildId: imageId(CONTROL_IMAGE),
adminBuildId,
adminSourceRevision,
controlBuildId,
controlSourceRevision,
postgresRepositoryDigest: POSTGRES_REPOSITORY_DIGEST,
migrationImageId,
initialRecoveryImageId,
@@ -1871,17 +1960,38 @@ async function main() {
database,
oci,
kubernetes: Object.freeze({
activePointer: pointerAfterRejection,
activePointer: reportActivePointer(pointerAfterRejection),
rbac,
}),
runtime,
gates: Object.freeze({
healthyInitialActivation: true,
missingTransitionFailedClosed: true,
invalidUpgradeRejectedBeforeActivation: true,
activePointerUidUnchanged: true,
activePointerResourceVersionUnchanged: true,
activePointerJsonUnchanged: true,
candidateRevisionAbsent: true,
exactAuthenticatedOciRequests: true,
recoveryRbacLeastPrivilege: true,
runtimeRolledOutAfterRecovery: true,
passed: true,
}),
limitations: Object.freeze([
'isolated PostgreSQL uses explicit TLS disable; production manifests remain verify-full',
'the authenticated HTTPS OCI Distribution fixture implements the immutable GET/referrers surface used by the resolver, not a production registry storage implementation',
'the disposable Kind control plane is single-replica; this gate proves workload ordering, not Kubernetes control-plane HA',
]),
});
process.stdout.write(`${JSON.stringify(report)}\n`);
writePrivateReport(reportFile, report);
process.stdout.write(
`${JSON.stringify({
schemaVersion: 1,
fixture: REPORT_SCHEMA,
reportWritten: true,
passed: true,
})}\n`,
);
} catch (error) {
diagnostics();
throw error;
@@ -1902,13 +2012,21 @@ async function main() {
}
}
main().catch((error) => {
process.stderr.write(
`${JSON.stringify({
schema: 'qinglong/plugin-package-recovery-e2e-live-failure@v1',
name: error?.name ?? 'Error',
message: error?.message ?? 'unknown failure',
})}\n`,
);
process.exitCode = 1;
});
if (require.main === module) {
main().catch((error) => {
process.stderr.write(
`${JSON.stringify({
schema: 'qinglong/plugin-package-recovery-e2e-live-failure@v1',
name: error?.name ?? 'Error',
message: error?.message ?? 'unknown failure',
})}\n`,
);
process.exitCode = 1;
});
}
module.exports = {
REPORT_SCHEMA,
privateReportPath,
writePrivateReport,
};