mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
test(ql3): audit plugin recovery evidence
This commit is contained in:
@@ -0,0 +1,508 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
'use strict';
|
||||
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const FIXTURE = 'qinglong/plugin-package-recovery-e2e-live-contract@v2';
|
||||
const LIMITATIONS = Object.freeze([
|
||||
'isolated PostgreSQL uses explicit TLS disable; production manifests remain verify-full',
|
||||
'the authenticated HTTPS OCI Distribution fixture implements the immutable GET/referrers surface used by the resolver, not a production registry storage implementation',
|
||||
'the disposable Kind control plane is single-replica; this gate proves workload ordering, not Kubernetes control-plane HA',
|
||||
]);
|
||||
const GATE_KEYS = Object.freeze([
|
||||
'healthyInitialActivation',
|
||||
'missingTransitionFailedClosed',
|
||||
'invalidUpgradeRejectedBeforeActivation',
|
||||
'activePointerUidUnchanged',
|
||||
'activePointerResourceVersionUnchanged',
|
||||
'activePointerJsonUnchanged',
|
||||
'candidateRevisionAbsent',
|
||||
'exactAuthenticatedOciRequests',
|
||||
'recoveryRbacLeastPrivilege',
|
||||
'runtimeRolledOutAfterRecovery',
|
||||
'passed',
|
||||
]);
|
||||
const BANNED_KEYS = new Set([
|
||||
'activejson',
|
||||
'assertion',
|
||||
'authorization',
|
||||
'bearer',
|
||||
'certificate',
|
||||
'connectionstring',
|
||||
'credential',
|
||||
'dsn',
|
||||
'kubeconfig',
|
||||
'password',
|
||||
'privatekey',
|
||||
'secret',
|
||||
'tlskey',
|
||||
'token',
|
||||
]);
|
||||
const SHA256 = /^[a-f0-9]{64}$/;
|
||||
const SHA256_ID = /^sha256:[a-f0-9]{64}$/;
|
||||
const UUID =
|
||||
/^[a-f0-9]{8}-[a-f0-9]{4}-[1-8][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/i;
|
||||
|
||||
function finding(code, detail) {
|
||||
return Object.freeze({ code, detail });
|
||||
}
|
||||
|
||||
function exactKeys(value, expected) {
|
||||
return Boolean(
|
||||
value &&
|
||||
typeof value === 'object' &&
|
||||
!Array.isArray(value) &&
|
||||
JSON.stringify(Object.keys(value).sort()) ===
|
||||
JSON.stringify([...expected].sort()),
|
||||
);
|
||||
}
|
||||
|
||||
function imageDigest(value) {
|
||||
if (typeof value !== 'string') return undefined;
|
||||
return value.match(/sha256:[a-f0-9]{64}$/)?.[0];
|
||||
}
|
||||
|
||||
function safeToken(value, maximum = 256) {
|
||||
return (
|
||||
typeof value === 'string' &&
|
||||
value.length > 0 &&
|
||||
value.length <= maximum &&
|
||||
/^[A-Za-z0-9][A-Za-z0-9._:/@+-]*$/.test(value)
|
||||
);
|
||||
}
|
||||
|
||||
function sensitiveMaterial(value, key = '') {
|
||||
if (BANNED_KEYS.has(key.toLowerCase())) return true;
|
||||
if (typeof value === 'string') {
|
||||
return (
|
||||
/-----BEGIN (?:CERTIFICATE|(?:RSA |EC |OPENSSH )?PRIVATE KEY)-----/.test(
|
||||
value,
|
||||
) ||
|
||||
/postgres(?:ql)?:\/\/[^/\s]+:[^@\s]+@/i.test(value) ||
|
||||
/\bBasic\s+[A-Za-z0-9+/=_-]{8,}/i.test(value) ||
|
||||
/\beyJ[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{8,}\./.test(value) ||
|
||||
/\bqlsecret(?::|\/\/)/i.test(value)
|
||||
);
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.some((entry) => sensitiveMaterial(entry));
|
||||
}
|
||||
if (value && typeof value === 'object') {
|
||||
return Object.entries(value).some(([entryKey, entry]) =>
|
||||
sensitiveMaterial(entry, entryKey),
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function validIso(value) {
|
||||
return (
|
||||
typeof value === 'string' &&
|
||||
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/.test(value) &&
|
||||
Number.isFinite(Date.parse(value))
|
||||
);
|
||||
}
|
||||
|
||||
function validRuntime(report) {
|
||||
const runtime = report.runtime;
|
||||
const ordering = report.ordering;
|
||||
return (
|
||||
exactKeys(runtime, [
|
||||
'replicas',
|
||||
'creationTimestamp',
|
||||
'recoveryJobUid',
|
||||
'recoveryCompletedAt',
|
||||
'nodes',
|
||||
'imageIds',
|
||||
]) &&
|
||||
runtime.replicas === 2 &&
|
||||
runtime.creationTimestamp === ordering?.runtimeCreatedAt &&
|
||||
runtime.recoveryJobUid === ordering?.rejectionRecoveryJobUid &&
|
||||
runtime.recoveryCompletedAt === ordering?.rejectionRecoveryCompletedAt &&
|
||||
Array.isArray(runtime.nodes) &&
|
||||
runtime.nodes.length === 2 &&
|
||||
new Set(runtime.nodes).size === 2 &&
|
||||
runtime.nodes.every((value) => safeToken(value)) &&
|
||||
Array.isArray(runtime.imageIds) &&
|
||||
runtime.imageIds.length === 1 &&
|
||||
imageDigest(runtime.imageIds[0]) === report.images?.controlBuildId
|
||||
);
|
||||
}
|
||||
|
||||
function validatePluginPackageRecoveryE2ELiveReport(report) {
|
||||
const findings = [];
|
||||
if (
|
||||
!exactKeys(report, [
|
||||
'schema',
|
||||
'observedAt',
|
||||
'sourceRevision',
|
||||
'passed',
|
||||
'cluster',
|
||||
'architecture',
|
||||
'elapsedMs',
|
||||
'images',
|
||||
'ordering',
|
||||
'failedUpgrade',
|
||||
'database',
|
||||
'oci',
|
||||
'kubernetes',
|
||||
'runtime',
|
||||
'gates',
|
||||
'limitations',
|
||||
]) ||
|
||||
report?.schema !== FIXTURE ||
|
||||
report?.passed !== true ||
|
||||
!validIso(report?.observedAt) ||
|
||||
!/^[a-f0-9]{40}$/.test(report?.sourceRevision ?? '') ||
|
||||
!/^ql3-plugin-recovery-e2e(?:-[a-z0-9](?:[-a-z0-9]{0,24}[a-z0-9])?)?$/.test(
|
||||
report?.cluster ?? '',
|
||||
) ||
|
||||
!['amd64', 'arm64'].includes(report?.architecture) ||
|
||||
!Number.isSafeInteger(report?.elapsedMs) ||
|
||||
report.elapsedMs < 1 ||
|
||||
report.elapsedMs > 60 * 60 * 1000
|
||||
) {
|
||||
findings.push(
|
||||
finding('QL3_PLUGIN_RECOVERY_E2E_ENVELOPE', 'report envelope is invalid'),
|
||||
);
|
||||
}
|
||||
|
||||
const images = report?.images;
|
||||
if (
|
||||
!exactKeys(images, [
|
||||
'adminBuildId',
|
||||
'adminSourceRevision',
|
||||
'controlBuildId',
|
||||
'controlSourceRevision',
|
||||
'postgresRepositoryDigest',
|
||||
'migrationImageId',
|
||||
'initialRecoveryImageId',
|
||||
'stageRecoveryImageId',
|
||||
'rejectionRecoveryImageId',
|
||||
'postgresImageId',
|
||||
]) ||
|
||||
!SHA256_ID.test(images?.adminBuildId ?? '') ||
|
||||
!SHA256_ID.test(images?.controlBuildId ?? '') ||
|
||||
images?.adminSourceRevision !== report?.sourceRevision ||
|
||||
images?.controlSourceRevision !== report?.sourceRevision ||
|
||||
images?.postgresRepositoryDigest !==
|
||||
'postgres@sha256:1961f96e6029a02c3812d7cb329a3b03a3ac2bb067058dec17b0f5596aca9296' ||
|
||||
!imageDigest(images?.migrationImageId) ||
|
||||
imageDigest(images?.initialRecoveryImageId) !==
|
||||
imageDigest(images?.migrationImageId) ||
|
||||
imageDigest(images?.stageRecoveryImageId) !==
|
||||
imageDigest(images?.migrationImageId) ||
|
||||
imageDigest(images?.rejectionRecoveryImageId) !==
|
||||
imageDigest(images?.migrationImageId) ||
|
||||
!imageDigest(images?.postgresImageId)
|
||||
) {
|
||||
findings.push(
|
||||
finding('QL3_PLUGIN_RECOVERY_E2E_IMAGES', 'image provenance is invalid'),
|
||||
);
|
||||
}
|
||||
|
||||
const ordering = report?.ordering;
|
||||
const timeKeys = [
|
||||
'migrationCompletedAt',
|
||||
'initialRecoveryCompletedAt',
|
||||
'upgradeStageFailedAt',
|
||||
'transitionCompletedAt',
|
||||
'rejectionRecoveryCompletedAt',
|
||||
'runtimeCreatedAt',
|
||||
];
|
||||
const uidKeys = [
|
||||
'migrationJobUid',
|
||||
'initialRecoveryJobUid',
|
||||
'upgradeStageJobUid',
|
||||
'transitionJobUid',
|
||||
'rejectionRecoveryJobUid',
|
||||
'runtimeBoundRecoveryJobUid',
|
||||
];
|
||||
const times = timeKeys.map((key) => Date.parse(ordering?.[key]));
|
||||
if (
|
||||
!exactKeys(ordering, [...uidKeys, ...timeKeys]) ||
|
||||
uidKeys.some((key) => !UUID.test(ordering?.[key] ?? '')) ||
|
||||
ordering?.runtimeBoundRecoveryJobUid !==
|
||||
ordering?.rejectionRecoveryJobUid ||
|
||||
timeKeys.some((key) => !validIso(ordering?.[key])) ||
|
||||
times.some((value, index) => index > 0 && value < times[index - 1])
|
||||
) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_PLUGIN_RECOVERY_E2E_ORDERING',
|
||||
'Job and rollout ordering is invalid',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const failed = report?.failedUpgrade;
|
||||
if (
|
||||
!exactKeys(failed, [
|
||||
'stageFailure',
|
||||
'transitionReceiptDigest',
|
||||
'rejectionReason',
|
||||
'candidateRevisionCount',
|
||||
'activePointerUnchanged',
|
||||
]) ||
|
||||
!exactKeys(failed?.stageFailure, ['jobUid', 'reason', 'durableState']) ||
|
||||
failed?.stageFailure?.jobUid !== ordering?.upgradeStageJobUid ||
|
||||
failed?.stageFailure?.reason !==
|
||||
'ClusterPluginPackageRecoveryRequiredError' ||
|
||||
failed?.stageFailure?.durableState !== 'staged' ||
|
||||
!SHA256.test(failed?.transitionReceiptDigest ?? '') ||
|
||||
failed?.rejectionReason !== 'activation_fact_conflict' ||
|
||||
failed?.candidateRevisionCount !== 0 ||
|
||||
failed?.activePointerUnchanged !== true
|
||||
) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_PLUGIN_RECOVERY_E2E_FAILED_UPGRADE',
|
||||
'failed upgrade evidence is invalid',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const database = report?.database;
|
||||
const databaseKeys = [
|
||||
'migrationCount',
|
||||
'capabilityVersion',
|
||||
'initialState',
|
||||
'initialActiveLockDigest',
|
||||
'upgradeState',
|
||||
'upgradePreviousActiveLockDigest',
|
||||
'upgradeActiveLockDigest',
|
||||
'upgradeFailureReason',
|
||||
'initialMutationCount',
|
||||
'upgradeMutationCount',
|
||||
'headInstallationId',
|
||||
'transitionReceiptCount',
|
||||
'initialRevisionCount',
|
||||
'upgradeRevisionCount',
|
||||
'recoverableCount',
|
||||
];
|
||||
if (
|
||||
!exactKeys(database, databaseKeys) ||
|
||||
database?.migrationCount !== 65 ||
|
||||
database?.capabilityVersion !== 64 ||
|
||||
database?.initialState !== 'active' ||
|
||||
database?.upgradeState !== 'failed' ||
|
||||
!SHA256.test(database?.initialActiveLockDigest ?? '') ||
|
||||
database?.upgradePreviousActiveLockDigest !==
|
||||
database?.initialActiveLockDigest ||
|
||||
database?.upgradeActiveLockDigest !== database?.initialActiveLockDigest ||
|
||||
database?.upgradeFailureReason !== 'activation_fact_conflict' ||
|
||||
database?.initialMutationCount !== 4 ||
|
||||
database?.upgradeMutationCount !== 3 ||
|
||||
database?.headInstallationId !== 'install-plugin-recovery-e2e-upgrade' ||
|
||||
database?.transitionReceiptCount !== 1 ||
|
||||
database?.initialRevisionCount !== 1 ||
|
||||
database?.upgradeRevisionCount !== 0 ||
|
||||
database?.recoverableCount !== 0
|
||||
) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_PLUGIN_RECOVERY_E2E_DATABASE',
|
||||
'durable database evidence is invalid',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const oci = report?.oci;
|
||||
if (
|
||||
!exactKeys(oci, [
|
||||
'https',
|
||||
'authentication',
|
||||
'authenticatedRequestCount',
|
||||
'requestCount',
|
||||
'uniquePaths',
|
||||
'initialRequestCount',
|
||||
'upgradeRequestCount',
|
||||
'redirects',
|
||||
]) ||
|
||||
oci?.https !== true ||
|
||||
oci?.authentication !== 'exact-registry-basic' ||
|
||||
oci?.authenticatedRequestCount !== 18 ||
|
||||
oci?.requestCount !== 18 ||
|
||||
oci?.uniquePaths !== 12 ||
|
||||
oci?.initialRequestCount !== 6 ||
|
||||
oci?.upgradeRequestCount !== 12 ||
|
||||
oci?.redirects !== 0
|
||||
) {
|
||||
findings.push(
|
||||
finding('QL3_PLUGIN_RECOVERY_E2E_OCI', 'OCI request evidence is invalid'),
|
||||
);
|
||||
}
|
||||
|
||||
const kubernetes = report?.kubernetes;
|
||||
const pointer = kubernetes?.activePointer;
|
||||
const rbac = kubernetes?.rbac;
|
||||
if (
|
||||
!exactKeys(kubernetes, ['activePointer', 'rbac']) ||
|
||||
!exactKeys(pointer, [
|
||||
'name',
|
||||
'uid',
|
||||
'resourceVersion',
|
||||
'activeJsonDigest',
|
||||
'intentDigest',
|
||||
'activationRef',
|
||||
]) ||
|
||||
!safeToken(pointer?.name) ||
|
||||
!UUID.test(pointer?.uid ?? '') ||
|
||||
!/^[1-9][0-9]*$/.test(pointer?.resourceVersion ?? '') ||
|
||||
!SHA256.test(pointer?.activeJsonDigest ?? '') ||
|
||||
!SHA256.test(pointer?.intentDigest ?? '') ||
|
||||
!safeToken(pointer?.activationRef) ||
|
||||
!exactKeys(rbac, [
|
||||
'getConfigMaps',
|
||||
'createConfigMaps',
|
||||
'updateConfigMaps',
|
||||
'listConfigMaps',
|
||||
'deleteConfigMaps',
|
||||
'getSecrets',
|
||||
]) ||
|
||||
rbac?.getConfigMaps !== true ||
|
||||
rbac?.createConfigMaps !== true ||
|
||||
rbac?.updateConfigMaps !== true ||
|
||||
rbac?.listConfigMaps !== false ||
|
||||
rbac?.deleteConfigMaps !== false ||
|
||||
rbac?.getSecrets !== false
|
||||
) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_PLUGIN_RECOVERY_E2E_KUBERNETES',
|
||||
'active pointer or RBAC evidence is invalid',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
if (!validRuntime(report ?? {})) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_PLUGIN_RECOVERY_E2E_RUNTIME',
|
||||
'runtime rollout evidence is invalid',
|
||||
),
|
||||
);
|
||||
}
|
||||
if (
|
||||
!exactKeys(report?.gates, GATE_KEYS) ||
|
||||
GATE_KEYS.some((key) => report?.gates?.[key] !== true)
|
||||
) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_PLUGIN_RECOVERY_E2E_GATES',
|
||||
'every gate must be explicitly true',
|
||||
),
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Array.isArray(report?.limitations) ||
|
||||
JSON.stringify([...report.limitations].sort()) !==
|
||||
JSON.stringify([...LIMITATIONS].sort())
|
||||
) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_PLUGIN_RECOVERY_E2E_LIMITATIONS',
|
||||
'limitations must remain exact',
|
||||
),
|
||||
);
|
||||
}
|
||||
if (sensitiveMaterial(report)) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_PLUGIN_RECOVERY_E2E_SENSITIVE',
|
||||
'report contains a forbidden key or sensitive material',
|
||||
),
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
fixture: FIXTURE,
|
||||
findings: Object.freeze(findings),
|
||||
compatible: findings.length === 0,
|
||||
});
|
||||
}
|
||||
|
||||
function readPrivateReport(filePath) {
|
||||
if (!path.isAbsolute(filePath) || path.resolve(filePath) !== filePath) {
|
||||
throw new Error('report path must be absolute and canonical');
|
||||
}
|
||||
const before = fs.lstatSync(filePath);
|
||||
if (
|
||||
!before.isFile() ||
|
||||
before.isSymbolicLink() ||
|
||||
before.size < 2 ||
|
||||
before.size > 1024 * 1024 ||
|
||||
(before.mode & 0o777) !== 0o600
|
||||
) {
|
||||
throw new Error(
|
||||
'report must be an owner-private regular file between 2 bytes and 1 MiB',
|
||||
);
|
||||
}
|
||||
const descriptor = fs.openSync(
|
||||
filePath,
|
||||
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
try {
|
||||
const opened = fs.fstatSync(descriptor);
|
||||
if (
|
||||
opened.dev !== before.dev ||
|
||||
opened.ino !== before.ino ||
|
||||
opened.size !== before.size ||
|
||||
(opened.mode & 0o777) !== 0o600
|
||||
) {
|
||||
throw new Error('report identity or mode changed while opening');
|
||||
}
|
||||
const text = fs.readFileSync(descriptor, 'utf8');
|
||||
const after = fs.fstatSync(descriptor);
|
||||
const currentPath = fs.lstatSync(filePath);
|
||||
if (
|
||||
after.size !== opened.size ||
|
||||
after.mtimeMs !== opened.mtimeMs ||
|
||||
currentPath.dev !== opened.dev ||
|
||||
currentPath.ino !== opened.ino ||
|
||||
currentPath.size !== opened.size ||
|
||||
(currentPath.mode & 0o777) !== 0o600
|
||||
) {
|
||||
throw new Error('report changed while reading');
|
||||
}
|
||||
return JSON.parse(text);
|
||||
} finally {
|
||||
fs.closeSync(descriptor);
|
||||
}
|
||||
}
|
||||
|
||||
function main(argv) {
|
||||
if (argv.length !== 1 || !argv[0].startsWith('--report=')) {
|
||||
throw new Error(
|
||||
'usage: ql3-plugin-package-recovery-e2e-live-audit --report=/absolute/report.json',
|
||||
);
|
||||
}
|
||||
const result = validatePluginPackageRecoveryE2ELiveReport(
|
||||
readPrivateReport(argv[0].slice('--report='.length)),
|
||||
);
|
||||
process.stdout.write(`${JSON.stringify(result)}\n`);
|
||||
if (!result.compatible) process.exitCode = 1;
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
try {
|
||||
main(process.argv.slice(2));
|
||||
} catch (error) {
|
||||
process.stderr.write(
|
||||
`QL3 Plugin Package recovery E2E live audit failed: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}\n`,
|
||||
);
|
||||
process.exitCode = 2;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
FIXTURE,
|
||||
GATE_KEYS,
|
||||
LIMITATIONS,
|
||||
readPrivateReport,
|
||||
validatePluginPackageRecoveryE2ELiveReport,
|
||||
};
|
||||
@@ -6,7 +6,7 @@ const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { randomBytes } = require('node:crypto');
|
||||
const { createHash, randomBytes } = require('node:crypto');
|
||||
const { spawnSync } = require('node:child_process');
|
||||
const yaml = require('js-yaml');
|
||||
|
||||
@@ -181,7 +181,62 @@ function imageExists(image) {
|
||||
);
|
||||
}
|
||||
|
||||
function buildImages() {
|
||||
function sourceRevision() {
|
||||
const value = process.env.QL3_SOURCE_REVISION ?? '';
|
||||
if (!/^[a-f0-9]{40}$/.test(value)) {
|
||||
fail(
|
||||
'QL3_SOURCE_REVISION must be the exact lowercase 40-hex source revision',
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function privateReportPath(argv) {
|
||||
if (
|
||||
argv.length !== 1 ||
|
||||
!argv[0].startsWith('--report=') ||
|
||||
!path.isAbsolute(argv[0].slice('--report='.length))
|
||||
) {
|
||||
fail(
|
||||
'usage: ql3-plugin-package-recovery-e2e-live-contract ' +
|
||||
'--report=/absolute/private-report.json',
|
||||
);
|
||||
}
|
||||
const reportFile = argv[0].slice('--report='.length);
|
||||
if (path.resolve(reportFile) !== reportFile) {
|
||||
fail('Plugin Package recovery E2E report path must be canonical');
|
||||
}
|
||||
if (fs.existsSync(reportFile)) {
|
||||
fail('refusing to overwrite the Plugin Package recovery E2E report');
|
||||
}
|
||||
const parent = fs.lstatSync(path.dirname(reportFile));
|
||||
if (!parent.isDirectory() || parent.isSymbolicLink()) {
|
||||
fail('Plugin Package recovery E2E report parent must be a real directory');
|
||||
}
|
||||
return reportFile;
|
||||
}
|
||||
|
||||
function writePrivateReport(reportFile, report) {
|
||||
const temporaryReport = path.join(
|
||||
path.dirname(reportFile),
|
||||
`.${path.basename(reportFile)}.${process.pid}.` +
|
||||
`${randomBytes(6).toString('hex')}.tmp`,
|
||||
);
|
||||
let descriptor;
|
||||
try {
|
||||
descriptor = fs.openSync(temporaryReport, 'wx', 0o600);
|
||||
fs.writeFileSync(descriptor, `${JSON.stringify(report, null, 2)}\n`);
|
||||
fs.fsyncSync(descriptor);
|
||||
fs.closeSync(descriptor);
|
||||
descriptor = undefined;
|
||||
fs.linkSync(temporaryReport, reportFile);
|
||||
} finally {
|
||||
if (descriptor !== undefined) fs.closeSync(descriptor);
|
||||
fs.rmSync(temporaryReport, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function buildImages(revision) {
|
||||
if (process.env.QL3_SKIP_IMAGE_BUILD === '1') {
|
||||
assert.equal(imageExists(ADMIN_IMAGE), true, `${ADMIN_IMAGE} is absent`);
|
||||
assert.equal(
|
||||
@@ -200,7 +255,7 @@ function buildImages() {
|
||||
'--tag',
|
||||
ADMIN_IMAGE,
|
||||
'--build-arg',
|
||||
'SOURCE_REVISION=ql3-plugin-recovery-e2e-live',
|
||||
`SOURCE_REVISION=${revision}`,
|
||||
'.',
|
||||
],
|
||||
{ label: 'build current QingLong 3.0 cluster-admin image' },
|
||||
@@ -214,7 +269,7 @@ function buildImages() {
|
||||
'--tag',
|
||||
CONTROL_IMAGE,
|
||||
'--build-arg',
|
||||
'SOURCE_REVISION=ql3-plugin-recovery-e2e-live',
|
||||
`SOURCE_REVISION=${revision}`,
|
||||
'.',
|
||||
],
|
||||
{ label: 'build current QingLong 3.0 cluster-control image' },
|
||||
@@ -248,6 +303,15 @@ function imageId(image) {
|
||||
)[0].Id;
|
||||
}
|
||||
|
||||
function imageSourceRevision(image) {
|
||||
return JSON.parse(
|
||||
run(DOCKER, ['image', 'inspect', image], {
|
||||
capture: true,
|
||||
quiet: true,
|
||||
}).stdout,
|
||||
)[0].Config?.Labels?.['org.opencontainers.image.revision'];
|
||||
}
|
||||
|
||||
function createRegistryCertificate(root) {
|
||||
const caKey = path.join(root, 'registry-ca.key');
|
||||
const caCert = path.join(root, 'registry-ca.crt');
|
||||
@@ -1264,6 +1328,19 @@ function activePointerEvidence(fixture) {
|
||||
});
|
||||
}
|
||||
|
||||
function reportActivePointer(pointer) {
|
||||
return Object.freeze({
|
||||
name: pointer.name,
|
||||
uid: pointer.uid,
|
||||
resourceVersion: pointer.resourceVersion,
|
||||
activeJsonDigest: createHash('sha256')
|
||||
.update(pointer.activeJson, 'utf8')
|
||||
.digest('hex'),
|
||||
intentDigest: pointer.intentDigest,
|
||||
activationRef: pointer.activationRef,
|
||||
});
|
||||
}
|
||||
|
||||
function registryEvidence(fixture) {
|
||||
const output = kubectl(
|
||||
['-n', NAMESPACE, 'logs', REGISTRY_NAME, '-c', 'registry'],
|
||||
@@ -1542,12 +1619,14 @@ function diagnostics() {
|
||||
if (snapshot.stdout) process.stderr.write(`${snapshot.stdout}\n`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
async function main(argv = process.argv.slice(2)) {
|
||||
const reportFile = privateReportPath(argv);
|
||||
if (process.env.QL3_PLUGIN_PACKAGE_RECOVERY_E2E_LIVE !== '1') {
|
||||
fail(
|
||||
'Refusing to create a live cluster without QL3_PLUGIN_PACKAGE_RECOVERY_E2E_LIVE=1',
|
||||
);
|
||||
}
|
||||
const revision = sourceRevision();
|
||||
const clusterName = exactClusterName();
|
||||
const existing = kind(['get', 'clusters'], {
|
||||
capture: true,
|
||||
@@ -1605,7 +1684,13 @@ async function main() {
|
||||
let created = false;
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
buildImages();
|
||||
buildImages(revision);
|
||||
const adminSourceRevision = imageSourceRevision(ADMIN_IMAGE);
|
||||
const controlSourceRevision = imageSourceRevision(CONTROL_IMAGE);
|
||||
assert.equal(adminSourceRevision, revision);
|
||||
assert.equal(controlSourceRevision, revision);
|
||||
const adminBuildId = imageId(ADMIN_IMAGE);
|
||||
const controlBuildId = imageId(CONTROL_IMAGE);
|
||||
ensurePostgresImage();
|
||||
fs.writeFileSync(
|
||||
kindConfig,
|
||||
@@ -1829,13 +1914,17 @@ async function main() {
|
||||
]);
|
||||
const report = Object.freeze({
|
||||
schema: REPORT_SCHEMA,
|
||||
observedAt: new Date().toISOString(),
|
||||
sourceRevision: revision,
|
||||
passed: true,
|
||||
cluster: clusterName,
|
||||
architecture: fixtureArchitecture,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
images: Object.freeze({
|
||||
adminBuildId: imageId(ADMIN_IMAGE),
|
||||
controlBuildId: imageId(CONTROL_IMAGE),
|
||||
adminBuildId,
|
||||
adminSourceRevision,
|
||||
controlBuildId,
|
||||
controlSourceRevision,
|
||||
postgresRepositoryDigest: POSTGRES_REPOSITORY_DIGEST,
|
||||
migrationImageId,
|
||||
initialRecoveryImageId,
|
||||
@@ -1871,17 +1960,38 @@ async function main() {
|
||||
database,
|
||||
oci,
|
||||
kubernetes: Object.freeze({
|
||||
activePointer: pointerAfterRejection,
|
||||
activePointer: reportActivePointer(pointerAfterRejection),
|
||||
rbac,
|
||||
}),
|
||||
runtime,
|
||||
gates: Object.freeze({
|
||||
healthyInitialActivation: true,
|
||||
missingTransitionFailedClosed: true,
|
||||
invalidUpgradeRejectedBeforeActivation: true,
|
||||
activePointerUidUnchanged: true,
|
||||
activePointerResourceVersionUnchanged: true,
|
||||
activePointerJsonUnchanged: true,
|
||||
candidateRevisionAbsent: true,
|
||||
exactAuthenticatedOciRequests: true,
|
||||
recoveryRbacLeastPrivilege: true,
|
||||
runtimeRolledOutAfterRecovery: true,
|
||||
passed: true,
|
||||
}),
|
||||
limitations: Object.freeze([
|
||||
'isolated PostgreSQL uses explicit TLS disable; production manifests remain verify-full',
|
||||
'the authenticated HTTPS OCI Distribution fixture implements the immutable GET/referrers surface used by the resolver, not a production registry storage implementation',
|
||||
'the disposable Kind control plane is single-replica; this gate proves workload ordering, not Kubernetes control-plane HA',
|
||||
]),
|
||||
});
|
||||
process.stdout.write(`${JSON.stringify(report)}\n`);
|
||||
writePrivateReport(reportFile, report);
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
fixture: REPORT_SCHEMA,
|
||||
reportWritten: true,
|
||||
passed: true,
|
||||
})}\n`,
|
||||
);
|
||||
} catch (error) {
|
||||
diagnostics();
|
||||
throw error;
|
||||
@@ -1902,13 +2012,21 @@ async function main() {
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
process.stderr.write(
|
||||
`${JSON.stringify({
|
||||
schema: 'qinglong/plugin-package-recovery-e2e-live-failure@v1',
|
||||
name: error?.name ?? 'Error',
|
||||
message: error?.message ?? 'unknown failure',
|
||||
})}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
if (require.main === module) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(
|
||||
`${JSON.stringify({
|
||||
schema: 'qinglong/plugin-package-recovery-e2e-live-failure@v1',
|
||||
name: error?.name ?? 'Error',
|
||||
message: error?.message ?? 'unknown failure',
|
||||
})}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
REPORT_SCHEMA,
|
||||
privateReportPath,
|
||||
writePrivateReport,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user