mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 00:17:47 +08:00
3459 lines
106 KiB
JavaScript
3459 lines
106 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
'use strict';
|
|
|
|
const assert = require('node:assert/strict');
|
|
const crypto = require('node:crypto');
|
|
const fs = require('node:fs');
|
|
const os = require('node:os');
|
|
const path = require('node:path');
|
|
const { spawnSync } = require('node:child_process');
|
|
const {
|
|
validateCloudNativePgDrEvidence,
|
|
} = require('./ql3-cloudnativepg-dr-evidence-audit.cjs');
|
|
|
|
const ROOT = path.resolve(__dirname, '..');
|
|
const K3S_IMAGE =
|
|
'rancher/k3s:v1.32.8-k3s1@sha256:f9f125ef9c662a231a98c507afdd3ba9a94d5f02631f946d1d34000ef67f7263';
|
|
const REGISTRY_IMAGE =
|
|
'registry:2@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373';
|
|
const SKOPEO_IMAGE =
|
|
'quay.io/skopeo/stable:v1.20.0@sha256:47853bb9fb24202af9110531ebd6e43c5f97701254ca290596640290d17942f4';
|
|
const K3S_VERSION = '1.32.8';
|
|
const K3S_EVICTION_HARD =
|
|
'memory.available<100Mi,nodefs.available<64Mi,imagefs.available<64Mi,nodefs.inodesFree<1%';
|
|
const DOCKER_SCOPE_LABEL =
|
|
'io.qinglong.ql3.live=cloudnativepg-barman-disaster-recovery';
|
|
const DOCKER_RUN_LABEL_KEY = 'io.qinglong.ql3.run';
|
|
const MINIMUM_DR_FREE_BYTES = 35n * 1024n * 1024n * 1024n;
|
|
const CERT_MANAGER_VERSION = '1.20.3';
|
|
const BARMAN_VERSION = '0.13.0';
|
|
const CNPG_VERSION = '1.30.0';
|
|
const CERT_MANAGER_MANIFEST_URL =
|
|
'https://github.com/cert-manager/cert-manager/releases/download/v1.20.3/cert-manager.yaml';
|
|
const CERT_MANAGER_MANIFEST_SHA256 =
|
|
'7ee74ba06845213e96d8ceaff3d20dd51e682765c1418eddda4e8780ba082261';
|
|
const BARMAN_MANIFEST_URL =
|
|
'https://github.com/cloudnative-pg/plugin-barman-cloud/releases/download/v0.13.0/manifest.yaml';
|
|
const BARMAN_MANIFEST_SHA256 =
|
|
'd2e71e7b06822448f1a421f05781846cfdb9cc621e7ef32eef5e20c5133213b0';
|
|
const CNPG_MANIFEST_URL =
|
|
'https://github.com/cloudnative-pg/cloudnative-pg/releases/download/v1.30.0/cnpg-1.30.0.yaml';
|
|
const CNPG_MANIFEST_SHA256 =
|
|
'f8bede43fe4ee0d478c2355b204a36876b2ae4faac60f2a9452280b293da3b88';
|
|
const POSTGRES_DATABASE_ROLE_MANIFEST = path.join(
|
|
ROOT,
|
|
'deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/database-roles.yaml',
|
|
);
|
|
const POSTGRES_ROLE_NAMES = Object.freeze([
|
|
'ql3_admin',
|
|
'ql3_ai_credential_manager',
|
|
'ql3_ai_credential_tester',
|
|
'ql3_ai_maintenance',
|
|
'ql3_approval_manager',
|
|
'ql3_automation_manager',
|
|
'ql3_migration',
|
|
'ql3_package_executor',
|
|
'ql3_package_manager',
|
|
'ql3_run_manager',
|
|
'ql3_runtime',
|
|
'ql3_worker_credential_executor',
|
|
'ql3_worker_credential_manager',
|
|
'ql3_worker_ingress',
|
|
]);
|
|
const REGISTRY_DATA_TARGETS = Object.freeze(['/var/lib/registry']);
|
|
const K3S_DATA_TARGETS = Object.freeze([
|
|
'/var/lib/cni',
|
|
'/var/lib/kubelet',
|
|
'/var/lib/rancher/k3s',
|
|
'/var/log',
|
|
]);
|
|
const REVIEWED_DOCKER_DATA_TARGETS = new Set([
|
|
...REGISTRY_DATA_TARGETS,
|
|
...K3S_DATA_TARGETS,
|
|
]);
|
|
|
|
function assertCloudNativePgDrRunnerCapacity(
|
|
statfs = fs.statfsSync,
|
|
filesystemPath = os.tmpdir(),
|
|
) {
|
|
assert.equal(typeof statfs, 'function');
|
|
assert.equal(path.isAbsolute(filesystemPath), true);
|
|
const stats = statfs(filesystemPath, { bigint: true });
|
|
assert.ok(stats && typeof stats === 'object');
|
|
assert.equal(typeof stats.bavail, 'bigint');
|
|
assert.equal(typeof stats.bsize, 'bigint');
|
|
assert.ok(stats.bavail >= 0n && stats.bsize > 0n);
|
|
const availableBytes = stats.bavail * stats.bsize;
|
|
assert.ok(
|
|
availableBytes >= MINIMUM_DR_FREE_BYTES,
|
|
`CloudNativePG DR live gate requires at least 35 GiB free; found ${availableBytes} bytes`,
|
|
);
|
|
return Object.freeze({
|
|
minimumBytes: MINIMUM_DR_FREE_BYTES,
|
|
availableBytes,
|
|
});
|
|
}
|
|
|
|
const IMAGES = Object.freeze({
|
|
cnpg: 'ghcr.io/cloudnative-pg/cloudnative-pg:1.30.0@sha256:a2701eb97cdd2a34b1fdb2cb51987f544b706e40bec72ae7146cd8580efefebb',
|
|
postgres:
|
|
'ghcr.io/cloudnative-pg/postgresql:18.4-minimal-trixie@sha256:24d229d801663f95b584416f8ebdfad4849b1a3fa4cfcf95a7f026df7aa6e22d',
|
|
barman:
|
|
'ghcr.io/cloudnative-pg/plugin-barman-cloud:v0.13.0@sha256:71589dbac582333442812b07b31f7ea4d00324a8358aac7ca507dabf9f4b6c96',
|
|
barmanSidecar:
|
|
'ghcr.io/cloudnative-pg/plugin-barman-cloud-sidecar:v0.13.0@sha256:990361af3319f9e23aafa0f6d7981f99bf1f69b4e6a85cf1bc7d71d6f09bb288',
|
|
certManager:
|
|
'quay.io/jetstack/cert-manager-controller:v1.20.3@sha256:6c13d61e0348a5bc3477f8ea9a928624300b30d19b1c72a7d2b90372fc713db4',
|
|
certManagerCainjector:
|
|
'quay.io/jetstack/cert-manager-cainjector:v1.20.3@sha256:06ad347fe0dc2eb84cc355c26f6752e05e87dceb6447f5cd29b963dd66dfd8bd',
|
|
certManagerWebhook:
|
|
'quay.io/jetstack/cert-manager-webhook:v1.20.3@sha256:a61e817632cebed3bb59a189327e786fa3fdd7597167d994a1848d98fd55848f',
|
|
minio:
|
|
'docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e',
|
|
minioClient:
|
|
'docker.io/minio/mc@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727',
|
|
});
|
|
|
|
const PLATFORM_DIGESTS = Object.freeze({
|
|
amd64: Object.freeze({
|
|
cnpg: 'sha256:091d306935cfdf646debfe78010d59ebfb572150eb6eb922b0203873c0c68841',
|
|
barman:
|
|
'sha256:417449fe4f6f0a56acdeb30e4131930815f2b46b9afeb808059b57aa8b4c2ef5',
|
|
barmanSidecar:
|
|
'sha256:15cb1a01e7c5235eedac2061cab8208e5f7c39dbda292f9c2d4ddaa0c1f211e6',
|
|
certManager:
|
|
'sha256:1e4af57beb469cc3bb0fb48b9201caea2723819b9ffd3c3ea98568f55b4dd38b',
|
|
certManagerCainjector:
|
|
'sha256:a2b12d27950d1603d2c8168c3ccd95d07b93ce6ec4b530316196a31db592a9c0',
|
|
certManagerWebhook:
|
|
'sha256:953a97df613f7da7eda8ce4b1c8d8e6b50963db0800fab595d040db6eb5cb060',
|
|
minio:
|
|
'sha256:a1a8bd4ac40ad7881a245bab97323e18f971e4d4cba2c2007ec1bedd21cbaba2',
|
|
minioClient:
|
|
'sha256:eb4ea9884b77704230e2423e9004d2fa738dc272876b9cc41a297d29443b8780',
|
|
postgres:
|
|
'sha256:67e56bcbe50a58e60509815dd89e58effc5f331ab844f66331d945cd42131e8d',
|
|
}),
|
|
arm64: Object.freeze({
|
|
cnpg: 'sha256:6c7926147fd23a053dea6605d61c013d43bfd411be3532e7500ef2d2b68bb98c',
|
|
barman:
|
|
'sha256:de612e3ad8633a198b91ffbea53848407424155daf2183d656490d843a83b100',
|
|
barmanSidecar:
|
|
'sha256:f53e168e341661cd76334215ead9dfd69f06117685d3232206192cf25218da71',
|
|
certManager:
|
|
'sha256:af62a025ae4f8fd03209b5e0760868296bad5a9370aab0c91ad3b5476bcb282d',
|
|
certManagerCainjector:
|
|
'sha256:3c052c134ad1b93122b957f4d214aaa9d85a37b5ff15acc5b4d86f50e3ed822e',
|
|
certManagerWebhook:
|
|
'sha256:7c510875e038f79f7fba707b5f86d8736777a4dfefcd42179b08844ee75e685b',
|
|
minio:
|
|
'sha256:9966a92a734f9411e32f4f41d7d9d826fcdc0f68c4e20b70295bd4e7c11f8a2f',
|
|
minioClient:
|
|
'sha256:37d109dddbbb2c95873f5fc81ac93f37023264770fc580a7564148892087b1b7',
|
|
postgres:
|
|
'sha256:2adbb634a1c0af8cb036e81200aaa7c8b62517bf5e501699926b337bd9f863f1',
|
|
}),
|
|
});
|
|
|
|
function run(binary, args, options = {}) {
|
|
if (!options.quiet) {
|
|
process.stderr.write(`+ ${path.basename(binary)} ${args.join(' ')}\n`);
|
|
}
|
|
const result = spawnSync(binary, args, {
|
|
cwd: options.cwd || ROOT,
|
|
env: options.env || process.env,
|
|
input: options.input,
|
|
encoding: 'utf8',
|
|
maxBuffer: 64 * 1024 * 1024,
|
|
stdio: options.capture
|
|
? ['pipe', 'pipe', 'pipe']
|
|
: [
|
|
options.input === undefined ? 'inherit' : 'pipe',
|
|
'inherit',
|
|
'inherit',
|
|
],
|
|
});
|
|
if (result.error) throw result.error;
|
|
if (result.status !== 0 && !options.allowFailure) {
|
|
throw new Error(
|
|
`${path.basename(binary)} failed with ${String(result.status)}: ${
|
|
result.stderr || result.stdout || ''
|
|
}`,
|
|
);
|
|
}
|
|
return Object.freeze({
|
|
status: result.status,
|
|
stdout: options.capture ? result.stdout.trim() : '',
|
|
stderr: options.capture ? result.stderr.trim() : '',
|
|
});
|
|
}
|
|
|
|
function sleep(milliseconds) {
|
|
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
}
|
|
|
|
async function waitFor(description, timeoutMs, inspect) {
|
|
const startedAt = Date.now();
|
|
let last = 'not observed';
|
|
while (Date.now() - startedAt < timeoutMs) {
|
|
try {
|
|
const result = inspect();
|
|
if (result?.ready) {
|
|
return Object.freeze({
|
|
elapsedMs: Date.now() - startedAt,
|
|
value: result.value,
|
|
});
|
|
}
|
|
if (result?.fact) last = result.fact;
|
|
} catch (error) {
|
|
last = error instanceof Error ? error.message : String(error);
|
|
}
|
|
await sleep(1_000);
|
|
}
|
|
throw new Error(`${description} timed out: ${last}`);
|
|
}
|
|
|
|
function sha256(value) {
|
|
return crypto.createHash('sha256').update(value).digest('hex');
|
|
}
|
|
|
|
function normalizeArchitecture(value) {
|
|
if (value === 'x86_64' || value === 'x64') return 'amd64';
|
|
if (value === 'aarch64') return 'arm64';
|
|
return value;
|
|
}
|
|
|
|
function privateDockerDataBindArgs(root, identity, targets) {
|
|
assert.ok(path.isAbsolute(root), 'Docker data root must be absolute');
|
|
assert.match(identity, /^[a-z0-9][a-z0-9-]{0,127}$/);
|
|
assert.ok(Array.isArray(targets) && targets.length > 0);
|
|
assert.equal(new Set(targets).size, targets.length);
|
|
assert.ok(!root.includes(','), 'Docker data root cannot contain a comma');
|
|
|
|
return targets.flatMap((target, index) => {
|
|
assert.ok(
|
|
REVIEWED_DOCKER_DATA_TARGETS.has(target),
|
|
`unreviewed Docker data target ${target}`,
|
|
);
|
|
const directory = path.join(
|
|
root,
|
|
identity,
|
|
`${String(index).padStart(2, '0')}-${path.basename(target)}`,
|
|
);
|
|
fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
fs.chmodSync(directory, 0o700);
|
|
return ['--mount', `type=bind,src=${directory},dst=${target}`];
|
|
});
|
|
}
|
|
|
|
function redactRuntimeText(value, sensitiveValues) {
|
|
return sensitiveValues.reduce((redacted, sensitive) => {
|
|
if (typeof sensitive !== 'string' || sensitive.length === 0)
|
|
return redacted;
|
|
return redacted.split(sensitive).join('[REDACTED]');
|
|
}, String(value));
|
|
}
|
|
|
|
function downloadManifest(curl, target, url) {
|
|
run(curl, [
|
|
'--http1.1',
|
|
'--fail',
|
|
'--location',
|
|
'--silent',
|
|
'--show-error',
|
|
'--retry',
|
|
'4',
|
|
'--retry-all-errors',
|
|
'--connect-timeout',
|
|
'15',
|
|
'--max-time',
|
|
'300',
|
|
'--output',
|
|
target,
|
|
url,
|
|
]);
|
|
fs.chmodSync(target, 0o600);
|
|
}
|
|
|
|
function manifestSource(environmentKey, fallback, curl, target, url) {
|
|
const selected = process.env[environmentKey] || fallback;
|
|
if (selected && fs.existsSync(selected)) return path.resolve(selected);
|
|
downloadManifest(curl, target, url);
|
|
return target;
|
|
}
|
|
|
|
function replaceExactlyOnce(source, from, to) {
|
|
return replaceExactly(source, from, to, 1);
|
|
}
|
|
|
|
function replaceExactly(source, from, to, expectedCount) {
|
|
assert.ok(Number.isSafeInteger(expectedCount) && expectedCount >= 1);
|
|
const occurrences = source.split(from).length - 1;
|
|
assert.equal(
|
|
occurrences,
|
|
expectedCount,
|
|
`expected exactly ${String(expectedCount)} manifest references ${from}`,
|
|
);
|
|
return source.split(from).join(to);
|
|
}
|
|
|
|
function reviewedManifest(source, target, expectedSha256, replacements) {
|
|
const stat = fs.lstatSync(source);
|
|
assert.ok(
|
|
stat.isFile() && !stat.isSymbolicLink() && stat.size >= 1024,
|
|
'release manifest must be a non-symlink regular file',
|
|
);
|
|
const bytes = fs.readFileSync(source);
|
|
assert.equal(sha256(bytes), expectedSha256, 'release manifest digest drift');
|
|
let text = bytes.toString('utf8');
|
|
for (const [from, to, expectedCount = 1] of replacements) {
|
|
text = replaceExactly(text, from, to, expectedCount);
|
|
}
|
|
fs.writeFileSync(target, text, { mode: 0o600, flag: 'wx' });
|
|
return Object.freeze({
|
|
sourceSha256: expectedSha256,
|
|
pinnedSha256: sha256(Buffer.from(text)),
|
|
target,
|
|
});
|
|
}
|
|
|
|
function imageIdDigest(value) {
|
|
const match = String(value || '').match(/sha256:[a-f0-9]{64}$/);
|
|
assert.ok(match, `invalid Kubernetes imageID ${String(value)}`);
|
|
return match[0];
|
|
}
|
|
|
|
function platformDigestFromImageIndex(index, architecture) {
|
|
const manifests = Array.isArray(index?.manifests) ? index.manifests : [];
|
|
const matching = manifests.filter(
|
|
(manifest) =>
|
|
manifest?.platform?.os === 'linux' &&
|
|
normalizeArchitecture(manifest.platform.architecture) === architecture,
|
|
);
|
|
assert.equal(
|
|
matching.length,
|
|
1,
|
|
`expected one linux/${architecture} manifest in the reviewed image index`,
|
|
);
|
|
return imageIdDigest(matching[0].digest);
|
|
}
|
|
|
|
function digestOnlyReference(image) {
|
|
const at = image.lastIndexOf('@');
|
|
assert.ok(at > 0, `reviewed image must contain a digest: ${image}`);
|
|
const digest = image.slice(at + 1);
|
|
imageIdDigest(digest);
|
|
const named = image.slice(0, at);
|
|
const slash = named.lastIndexOf('/');
|
|
const colon = named.lastIndexOf(':');
|
|
const repository = colon > slash ? named.slice(0, colon) : named;
|
|
return `${repository}@${digest}`;
|
|
}
|
|
|
|
function runtimeImageEvidence(
|
|
pod,
|
|
containerName,
|
|
reviewedImage,
|
|
expectedPlatformDigest,
|
|
) {
|
|
const status = (pod?.status?.containerStatuses || []).find(
|
|
(candidate) => candidate.name === containerName,
|
|
);
|
|
assert.ok(status, `missing runtime container status ${containerName}`);
|
|
const imageId = imageIdDigest(status.imageID);
|
|
const indexDigest = imageIdDigest(reviewedImage);
|
|
assert.equal(imageId, expectedPlatformDigest);
|
|
return Object.freeze({
|
|
imageId,
|
|
indexDigest,
|
|
platformDigest: expectedPlatformDigest,
|
|
});
|
|
}
|
|
|
|
function mirrorPlatformImage(
|
|
docker,
|
|
registry,
|
|
sourceImage,
|
|
repository,
|
|
architecture,
|
|
expectedPlatformDigest,
|
|
) {
|
|
assert.match(repository, /^[a-z][a-z0-9-]*$/);
|
|
const localDestinationTag = `localhost:5000/ql3/${repository}:reviewed-${architecture}`;
|
|
run(
|
|
docker,
|
|
[
|
|
'run',
|
|
'--rm',
|
|
'--network',
|
|
`container:${registry}`,
|
|
SKOPEO_IMAGE,
|
|
'copy',
|
|
'--retry-times',
|
|
'4',
|
|
'--retry-delay',
|
|
'2s',
|
|
'--preserve-digests',
|
|
'--override-os',
|
|
'linux',
|
|
'--override-arch',
|
|
architecture,
|
|
'--dest-tls-verify=false',
|
|
`docker://${digestOnlyReference(sourceImage)}`,
|
|
`docker://${localDestinationTag}`,
|
|
],
|
|
{ capture: true, quiet: true },
|
|
);
|
|
const inspection = JSON.parse(
|
|
run(
|
|
docker,
|
|
[
|
|
'run',
|
|
'--rm',
|
|
'--network',
|
|
`container:${registry}`,
|
|
SKOPEO_IMAGE,
|
|
'inspect',
|
|
'--tls-verify=false',
|
|
`docker://${localDestinationTag}`,
|
|
],
|
|
{ capture: true, quiet: true },
|
|
).stdout,
|
|
);
|
|
assert.equal(
|
|
inspection.Digest,
|
|
expectedPlatformDigest,
|
|
`local registry digest drift for ${repository}`,
|
|
);
|
|
return `${registry}:5000/ql3/${repository}@${expectedPlatformDigest}`;
|
|
}
|
|
|
|
function buildClusterControlImage({
|
|
docker,
|
|
registry,
|
|
registryPort,
|
|
architecture,
|
|
suffix,
|
|
}) {
|
|
assert.match(registry, /^ql3-barman-dr-[0-9]+-[a-f0-9]{6}-registry$/);
|
|
assert.match(registryPort, /^127\.0\.0\.1:[0-9]+$/);
|
|
assert.match(architecture, /^(?:amd64|arm64)$/);
|
|
assert.match(suffix, /^[0-9]+-[a-f0-9]{6}$/);
|
|
const repository = `${registryPort}/ql3/cluster-control`;
|
|
const localTag = `${repository}:dr-${suffix}`;
|
|
run(docker, [
|
|
'build',
|
|
'--file',
|
|
'deploy/containers/ql3-cluster-control/Dockerfile',
|
|
'--platform',
|
|
`linux/${architecture}`,
|
|
'--provenance=false',
|
|
'--target',
|
|
'runtime',
|
|
'--tag',
|
|
localTag,
|
|
'--build-arg',
|
|
'SOURCE_REVISION=cloudnativepg-barman-live',
|
|
'.',
|
|
]);
|
|
run(docker, ['push', localTag]);
|
|
const inspection = JSON.parse(
|
|
run(docker, ['image', 'inspect', localTag], {
|
|
capture: true,
|
|
quiet: true,
|
|
}).stdout,
|
|
);
|
|
assert.equal(inspection.length, 1);
|
|
const prefix = `${repository}@`;
|
|
const matching = (inspection[0].RepoDigests || []).filter((reference) =>
|
|
reference.startsWith(prefix),
|
|
);
|
|
assert.equal(
|
|
matching.length,
|
|
1,
|
|
'missing exact local image repository digest',
|
|
);
|
|
const platformDigest = imageIdDigest(matching[0]);
|
|
return Object.freeze({
|
|
localTag,
|
|
platformDigest,
|
|
runtimeImage: `${registry}:5000/ql3/cluster-control@${platformDigest}`,
|
|
});
|
|
}
|
|
|
|
function kubernetesSecret(namespace, name, values) {
|
|
return {
|
|
apiVersion: 'v1',
|
|
kind: 'Secret',
|
|
metadata: { namespace, name },
|
|
type: 'Opaque',
|
|
data: Object.fromEntries(
|
|
Object.entries(values).map(([key, value]) => [
|
|
key,
|
|
Buffer.from(value, 'utf8').toString('base64'),
|
|
]),
|
|
),
|
|
};
|
|
}
|
|
|
|
function postgresRoleSecretResources(credentials) {
|
|
assert.ok(credentials && typeof credentials === 'object');
|
|
assert.deepEqual(Object.keys(credentials).sort(), POSTGRES_ROLE_NAMES);
|
|
const items = POSTGRES_ROLE_NAMES.map((role) => {
|
|
const password = credentials[role];
|
|
assert.match(password, /^[A-Za-z0-9_-]{32,128}$/);
|
|
return Object.freeze({
|
|
apiVersion: 'v1',
|
|
kind: 'Secret',
|
|
metadata: {
|
|
namespace: 'ql3-dr',
|
|
name: `ql3-postgres-${role
|
|
.replace(/^ql3_/, '')
|
|
.replaceAll('_', '-')}-auth`,
|
|
},
|
|
type: 'kubernetes.io/basic-auth',
|
|
data: {
|
|
username: Buffer.from(role, 'utf8').toString('base64'),
|
|
password: Buffer.from(password, 'utf8').toString('base64'),
|
|
},
|
|
});
|
|
});
|
|
return Object.freeze({
|
|
apiVersion: 'v1',
|
|
kind: 'List',
|
|
items: Object.freeze(items),
|
|
});
|
|
}
|
|
|
|
function postgresMigrationJobResource({ controlImage }) {
|
|
imageIdDigest(controlImage);
|
|
assert.match(
|
|
controlImage,
|
|
/^ql3-barman-dr-[0-9]+-[a-f0-9]{6}-registry:5000\/ql3\/cluster-control@sha256:[a-f0-9]{64}$/,
|
|
);
|
|
const roleSecret = 'ql3-postgres-migration-auth';
|
|
const secretValue = (name, key) => ({
|
|
name,
|
|
valueFrom: { secretKeyRef: { name: roleSecret, key } },
|
|
});
|
|
return Object.freeze({
|
|
apiVersion: 'batch/v1',
|
|
kind: 'Job',
|
|
metadata: {
|
|
namespace: 'ql3-dr',
|
|
name: 'ql3-cluster-migration',
|
|
labels: {
|
|
'app.kubernetes.io/name': 'ql3-cluster-migration',
|
|
'app.kubernetes.io/component': 'database-migration',
|
|
'app.kubernetes.io/part-of': 'qinglong3',
|
|
},
|
|
},
|
|
spec: {
|
|
backoffLimit: 0,
|
|
activeDeadlineSeconds: 600,
|
|
template: {
|
|
metadata: {
|
|
labels: {
|
|
'app.kubernetes.io/name': 'ql3-cluster-migration',
|
|
'app.kubernetes.io/component': 'database-migration',
|
|
'app.kubernetes.io/part-of': 'qinglong3',
|
|
},
|
|
},
|
|
spec: {
|
|
automountServiceAccountToken: false,
|
|
restartPolicy: 'Never',
|
|
securityContext: {
|
|
runAsNonRoot: true,
|
|
runAsUser: 10001,
|
|
runAsGroup: 10001,
|
|
fsGroup: 10001,
|
|
seccompProfile: { type: 'RuntimeDefault' },
|
|
},
|
|
containers: [
|
|
{
|
|
name: 'migration',
|
|
image: controlImage,
|
|
imagePullPolicy: 'IfNotPresent',
|
|
command: [
|
|
'node',
|
|
'/opt/qinglong/node_modules/@qinglong/cluster-postgres/dist/migration/migrationCli.js',
|
|
],
|
|
env: [
|
|
{ name: 'QL3_POSTGRES_TLS_MODE', value: 'verify-full' },
|
|
{
|
|
name: 'QL3_POSTGRES_TLS_CA_FILE',
|
|
value: '/var/run/secrets/qinglong3/postgres/ca.crt',
|
|
},
|
|
{
|
|
name: 'QL3_POSTGRES_TLS_SERVERNAME',
|
|
value: 'ql3-postgres-rw.ql3-dr.svc',
|
|
},
|
|
{
|
|
name: 'QL3_POSTGRES_MIGRATION_HOST',
|
|
value: 'ql3-postgres-rw.ql3-dr.svc',
|
|
},
|
|
{ name: 'QL3_POSTGRES_MIGRATION_PORT', value: '5432' },
|
|
{
|
|
name: 'QL3_POSTGRES_MIGRATION_DATABASE',
|
|
value: 'qinglong',
|
|
},
|
|
secretValue('QL3_POSTGRES_MIGRATION_USER', 'username'),
|
|
secretValue('QL3_POSTGRES_MIGRATION_PASSWORD', 'password'),
|
|
{
|
|
name: 'QL3_POSTGRES_APPLICATION_NAME',
|
|
value: 'qinglong3-cluster-migration',
|
|
},
|
|
],
|
|
resources: {
|
|
requests: { cpu: '100m', memory: '128Mi' },
|
|
limits: { cpu: '1', memory: '256Mi' },
|
|
},
|
|
securityContext: {
|
|
allowPrivilegeEscalation: false,
|
|
readOnlyRootFilesystem: true,
|
|
capabilities: { drop: ['ALL'] },
|
|
},
|
|
volumeMounts: [
|
|
{ name: 'tmp', mountPath: '/tmp' },
|
|
{
|
|
name: 'postgres-ca',
|
|
mountPath: '/var/run/secrets/qinglong3/postgres',
|
|
readOnly: true,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
volumes: [
|
|
{ name: 'tmp', emptyDir: { medium: 'Memory', sizeLimit: '16Mi' } },
|
|
{
|
|
name: 'postgres-ca',
|
|
secret: {
|
|
secretName: 'ql3-postgres-ca',
|
|
defaultMode: 292,
|
|
items: [{ key: 'ca.crt', path: 'ca.crt' }],
|
|
},
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
});
|
|
}
|
|
|
|
function postgresRestoreApplicationProbeResources({
|
|
clusterName,
|
|
controlImage,
|
|
apiCredentialPepper,
|
|
}) {
|
|
assert.match(clusterName, /^ql3-postgres-restore-(?:latest|pitr)$/);
|
|
imageIdDigest(controlImage);
|
|
assert.match(
|
|
controlImage,
|
|
/^ql3-barman-dr-[0-9]+-[a-f0-9]{6}-registry:5000\/ql3\/cluster-control@sha256:[a-f0-9]{64}$/,
|
|
);
|
|
assert.match(apiCredentialPepper, /^[A-Za-z0-9_-]{43}$/);
|
|
assert.equal(
|
|
Buffer.from(apiCredentialPepper, 'base64url').toString('base64url'),
|
|
apiCredentialPepper,
|
|
);
|
|
const suffix = clusterName.endsWith('-latest') ? 'latest' : 'pitr';
|
|
const name = `ql3-dr-application-${suffix}`;
|
|
const secretName = `${name}-security`;
|
|
const databaseSecret = 'ql3-postgres-runtime-auth';
|
|
const secretValue = (name, secret, key) => ({
|
|
name,
|
|
valueFrom: { secretKeyRef: { name: secret, key } },
|
|
});
|
|
const labels = Object.freeze({
|
|
'app.kubernetes.io/name': name,
|
|
'app.kubernetes.io/component': 'restore-application-probe',
|
|
'app.kubernetes.io/part-of': 'qinglong3',
|
|
'ql3.cloud/restore-cluster': clusterName,
|
|
});
|
|
return Object.freeze({
|
|
apiVersion: 'v1',
|
|
kind: 'List',
|
|
items: Object.freeze([
|
|
kubernetesSecret('ql3-dr', secretName, {
|
|
'api-credential-pepper': apiCredentialPepper,
|
|
}),
|
|
{
|
|
apiVersion: 'apps/v1',
|
|
kind: 'Deployment',
|
|
metadata: { namespace: 'ql3-dr', name, labels },
|
|
spec: {
|
|
replicas: 1,
|
|
minReadySeconds: 2,
|
|
progressDeadlineSeconds: 300,
|
|
strategy: { type: 'Recreate' },
|
|
selector: { matchLabels: labels },
|
|
template: {
|
|
metadata: { labels },
|
|
spec: {
|
|
automountServiceAccountToken: false,
|
|
terminationGracePeriodSeconds: 15,
|
|
securityContext: {
|
|
runAsNonRoot: true,
|
|
runAsUser: 10001,
|
|
runAsGroup: 10001,
|
|
fsGroup: 10001,
|
|
seccompProfile: { type: 'RuntimeDefault' },
|
|
},
|
|
containers: [
|
|
{
|
|
name: 'cluster-control',
|
|
image: controlImage,
|
|
imagePullPolicy: 'IfNotPresent',
|
|
env: [
|
|
{ name: 'QL_DEPLOYMENT_PROFILE', value: 'cluster-control' },
|
|
{ name: 'QL3_CLUSTER_CONTROL_ENABLED', value: 'true' },
|
|
{ name: 'QL3_WORKER_INGRESS_ENABLED', value: 'false' },
|
|
{ name: 'QL3_CLUSTER_HTTP_HOST', value: '0.0.0.0' },
|
|
{ name: 'QL3_CLUSTER_HTTP_PORT', value: '5800' },
|
|
{
|
|
name: 'QL3_CLUSTER_REPLICA_ID',
|
|
valueFrom: {
|
|
fieldRef: {
|
|
apiVersion: 'v1',
|
|
fieldPath: 'metadata.name',
|
|
},
|
|
},
|
|
},
|
|
{ name: 'QL3_POSTGRES_TLS_MODE', value: 'verify-full' },
|
|
{
|
|
name: 'QL3_POSTGRES_TLS_CA_FILE',
|
|
value: '/var/run/secrets/qinglong3/postgres/ca.crt',
|
|
},
|
|
{
|
|
name: 'QL3_POSTGRES_TLS_SERVERNAME',
|
|
value: `${clusterName}-rw.ql3-dr.svc`,
|
|
},
|
|
{
|
|
name: 'QL3_POSTGRES_RUNTIME_HOST',
|
|
value: `${clusterName}-rw.ql3-dr.svc`,
|
|
},
|
|
{ name: 'QL3_POSTGRES_RUNTIME_PORT', value: '5432' },
|
|
{
|
|
name: 'QL3_POSTGRES_RUNTIME_DATABASE',
|
|
value: 'qinglong',
|
|
},
|
|
secretValue(
|
|
'QL3_POSTGRES_RUNTIME_USER',
|
|
databaseSecret,
|
|
'username',
|
|
),
|
|
secretValue(
|
|
'QL3_POSTGRES_RUNTIME_PASSWORD',
|
|
databaseSecret,
|
|
'password',
|
|
),
|
|
{
|
|
name: 'QL3_POSTGRES_APPLICATION_NAME',
|
|
value: `qinglong3-dr-${suffix}`,
|
|
},
|
|
{ name: 'QL3_POSTGRES_MAX_CONNECTIONS', value: '2' },
|
|
secretValue(
|
|
'QL3_API_CREDENTIAL_PEPPER',
|
|
secretName,
|
|
'api-credential-pepper',
|
|
),
|
|
],
|
|
ports: [{ name: 'http', containerPort: 5800 }],
|
|
startupProbe: {
|
|
httpGet: { path: '/livez', port: 'http' },
|
|
periodSeconds: 2,
|
|
timeoutSeconds: 1,
|
|
failureThreshold: 30,
|
|
},
|
|
readinessProbe: {
|
|
httpGet: { path: '/readyz', port: 'http' },
|
|
periodSeconds: 2,
|
|
timeoutSeconds: 1,
|
|
failureThreshold: 5,
|
|
},
|
|
livenessProbe: {
|
|
httpGet: { path: '/livez', port: 'http' },
|
|
periodSeconds: 10,
|
|
timeoutSeconds: 2,
|
|
failureThreshold: 3,
|
|
},
|
|
resources: {
|
|
requests: { cpu: '50m', memory: '128Mi' },
|
|
limits: { cpu: '1', memory: '512Mi' },
|
|
},
|
|
securityContext: {
|
|
allowPrivilegeEscalation: false,
|
|
readOnlyRootFilesystem: true,
|
|
capabilities: { drop: ['ALL'] },
|
|
},
|
|
volumeMounts: [
|
|
{ name: 'tmp', mountPath: '/tmp' },
|
|
{
|
|
name: 'postgres-ca',
|
|
mountPath: '/var/run/secrets/qinglong3/postgres',
|
|
readOnly: true,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
volumes: [
|
|
{
|
|
name: 'tmp',
|
|
emptyDir: { medium: 'Memory', sizeLimit: '16Mi' },
|
|
},
|
|
{
|
|
name: 'postgres-ca',
|
|
secret: {
|
|
secretName: `${clusterName}-ca`,
|
|
defaultMode: 292,
|
|
items: [{ key: 'ca.crt', path: 'ca.crt' }],
|
|
},
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
]),
|
|
});
|
|
}
|
|
|
|
function minioFixtureResources({ minioImage, clientImage, credentials }) {
|
|
const namespace = 'ql3-dr';
|
|
assert.notEqual(credentials.writer.accessKey, credentials.recovery.accessKey);
|
|
assert.notEqual(credentials.writer.secretKey, credentials.recovery.secretKey);
|
|
const writerPolicy = JSON.stringify({
|
|
Version: '2012-10-17',
|
|
Statement: [
|
|
{
|
|
Effect: 'Allow',
|
|
Action: [
|
|
's3:GetBucketLocation',
|
|
's3:GetBucketVersioning',
|
|
's3:GetObjectLockConfiguration',
|
|
's3:ListBucket',
|
|
's3:ListBucketMultipartUploads',
|
|
's3:ListBucketVersions',
|
|
],
|
|
Resource: ['arn:aws:s3:::ql3-dr'],
|
|
},
|
|
{
|
|
Effect: 'Allow',
|
|
Action: [
|
|
's3:AbortMultipartUpload',
|
|
's3:DeleteObject',
|
|
's3:DeleteObjectVersion',
|
|
's3:GetObject',
|
|
's3:GetObjectVersion',
|
|
's3:ListMultipartUploadParts',
|
|
's3:PutObject',
|
|
],
|
|
Resource: ['arn:aws:s3:::ql3-dr/*'],
|
|
},
|
|
],
|
|
});
|
|
const recoveryPolicy = JSON.stringify({
|
|
Version: '2012-10-17',
|
|
Statement: [
|
|
{
|
|
Effect: 'Allow',
|
|
Action: [
|
|
's3:GetBucketLocation',
|
|
's3:GetBucketVersioning',
|
|
's3:GetObjectLockConfiguration',
|
|
's3:ListBucket',
|
|
's3:ListBucketVersions',
|
|
],
|
|
Resource: ['arn:aws:s3:::ql3-dr'],
|
|
},
|
|
{
|
|
Effect: 'Allow',
|
|
Action: ['s3:GetObject', 's3:GetObjectVersion'],
|
|
Resource: ['arn:aws:s3:::ql3-dr/*'],
|
|
},
|
|
],
|
|
});
|
|
const credentialEnv = (name, secretName, key) => ({
|
|
name,
|
|
valueFrom: { secretKeyRef: { name: secretName, key } },
|
|
});
|
|
const clientEnvironment = [
|
|
{ name: 'MC_CONFIG_DIR', value: '/tmp/mc' },
|
|
{ name: 'SSL_CERT_FILE', value: '/certs/ca.crt' },
|
|
credentialEnv('MINIO_ROOT_USER', 'ql3-minio-root', 'ACCESS_KEY_ID'),
|
|
credentialEnv('MINIO_ROOT_PASSWORD', 'ql3-minio-root', 'ACCESS_SECRET_KEY'),
|
|
credentialEnv(
|
|
'WRITER_ACCESS_KEY',
|
|
'ql3-object-store-writer',
|
|
'ACCESS_KEY_ID',
|
|
),
|
|
credentialEnv(
|
|
'WRITER_SECRET_KEY',
|
|
'ql3-object-store-writer',
|
|
'ACCESS_SECRET_KEY',
|
|
),
|
|
credentialEnv(
|
|
'RECOVERY_ACCESS_KEY',
|
|
'ql3-object-store-recovery',
|
|
'ACCESS_KEY_ID',
|
|
),
|
|
credentialEnv(
|
|
'RECOVERY_SECRET_KEY',
|
|
'ql3-object-store-recovery',
|
|
'ACCESS_SECRET_KEY',
|
|
),
|
|
];
|
|
const clientVolumes = [
|
|
{
|
|
name: 'ca',
|
|
secret: {
|
|
secretName: 'ql3-minio-ca',
|
|
items: [{ key: 'tls.crt', path: 'ca.crt' }],
|
|
},
|
|
},
|
|
{ name: 'tmp', emptyDir: {} },
|
|
];
|
|
const clientMounts = [
|
|
{ name: 'ca', mountPath: '/certs', readOnly: true },
|
|
{ name: 'tmp', mountPath: '/tmp' },
|
|
];
|
|
const clientSecurityContext = {
|
|
allowPrivilegeEscalation: false,
|
|
capabilities: { drop: ['ALL'] },
|
|
readOnlyRootFilesystem: true,
|
|
runAsNonRoot: true,
|
|
runAsUser: 1000,
|
|
seccompProfile: { type: 'RuntimeDefault' },
|
|
};
|
|
const bootstrapScript = `set -eu
|
|
until mc alias set ql3 https://ql3-minio:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" --api S3v4 --path on >/dev/null 2>&1; do sleep 2; done
|
|
mc mb --with-lock --ignore-existing ql3/ql3-dr >/dev/null
|
|
mc retention set --default governance 30d ql3/ql3-dr/ >/dev/null
|
|
mc ilm rule add --expire-days 45 --noncurrent-expire-days 45 ql3/ql3-dr/ >/dev/null
|
|
printf '%s' '${writerPolicy}' > /tmp/writer-policy.json
|
|
printf '%s' '${recoveryPolicy}' > /tmp/recovery-policy.json
|
|
mc admin policy create ql3 ql3-writer /tmp/writer-policy.json >/dev/null
|
|
mc admin user add ql3 "$WRITER_ACCESS_KEY" "$WRITER_SECRET_KEY" >/dev/null
|
|
mc admin policy attach ql3 ql3-writer --user "$WRITER_ACCESS_KEY" >/dev/null
|
|
mc admin policy create ql3 ql3-recovery /tmp/recovery-policy.json >/dev/null
|
|
mc admin user add ql3 "$RECOVERY_ACCESS_KEY" "$RECOVERY_SECRET_KEY" >/dev/null
|
|
mc admin policy attach ql3 ql3-recovery --user "$RECOVERY_ACCESS_KEY" >/dev/null
|
|
mc alias set writer https://ql3-minio:9000 "$WRITER_ACCESS_KEY" "$WRITER_SECRET_KEY" --api S3v4 --path on >/dev/null
|
|
printf 'ql3-object-store-authority\n' > /tmp/probe
|
|
mc cp /tmp/probe writer/ql3-dr/authority/probe >/dev/null
|
|
mc version info ql3/ql3-dr >/dev/null
|
|
mc retention info ql3/ql3-dr/ >/dev/null
|
|
mc ilm rule ls ql3/ql3-dr/ >/dev/null
|
|
echo QL3_MINIO_AUTHORITY_READY`;
|
|
const verifierScript = `set -eu
|
|
mc alias set recovery https://ql3-minio:9000 "$RECOVERY_ACCESS_KEY" "$RECOVERY_SECRET_KEY" --api S3v4 --path on >/dev/null
|
|
mc cat recovery/ql3-dr/authority/probe > /tmp/readback
|
|
grep -qx ql3-object-store-authority /tmp/readback
|
|
printf 'unauthorized\n' > /tmp/unauthorized
|
|
if mc cp /tmp/unauthorized recovery/ql3-dr/authority/unauthorized >/dev/null 2>&1; then exit 51; fi
|
|
if mc rm recovery/ql3-dr/authority/probe >/dev/null 2>&1; then exit 52; fi
|
|
mc ls recovery/ql3-dr/ >/dev/null
|
|
echo QL3_RECOVERY_READ_ONLY_READY`;
|
|
const job = (name, script, backoffLimit) => ({
|
|
apiVersion: 'batch/v1',
|
|
kind: 'Job',
|
|
metadata: { namespace, name },
|
|
spec: {
|
|
backoffLimit,
|
|
template: {
|
|
metadata: { labels: { app: name } },
|
|
spec: {
|
|
restartPolicy: 'Never',
|
|
securityContext: { fsGroup: 1000 },
|
|
containers: [
|
|
{
|
|
name: 'mc',
|
|
image: clientImage,
|
|
imagePullPolicy: 'IfNotPresent',
|
|
command: ['/bin/sh', '-ec', script],
|
|
env: clientEnvironment,
|
|
resources: {
|
|
requests: { cpu: '10m', memory: '32Mi' },
|
|
limits: { cpu: '250m', memory: '128Mi' },
|
|
},
|
|
securityContext: clientSecurityContext,
|
|
volumeMounts: clientMounts,
|
|
},
|
|
],
|
|
volumes: clientVolumes,
|
|
},
|
|
},
|
|
},
|
|
});
|
|
const items = [
|
|
{
|
|
apiVersion: 'v1',
|
|
kind: 'Namespace',
|
|
metadata: { name: namespace },
|
|
},
|
|
kubernetesSecret(namespace, 'ql3-minio-root', {
|
|
ACCESS_KEY_ID: credentials.root.accessKey,
|
|
ACCESS_SECRET_KEY: credentials.root.secretKey,
|
|
}),
|
|
kubernetesSecret(namespace, 'ql3-object-store-writer', {
|
|
ACCESS_KEY_ID: credentials.writer.accessKey,
|
|
ACCESS_SECRET_KEY: credentials.writer.secretKey,
|
|
}),
|
|
kubernetesSecret(namespace, 'ql3-object-store-recovery', {
|
|
ACCESS_KEY_ID: credentials.recovery.accessKey,
|
|
ACCESS_SECRET_KEY: credentials.recovery.secretKey,
|
|
}),
|
|
{
|
|
apiVersion: 'cert-manager.io/v1',
|
|
kind: 'Issuer',
|
|
metadata: { namespace, name: 'ql3-minio-selfsigned' },
|
|
spec: { selfSigned: {} },
|
|
},
|
|
{
|
|
apiVersion: 'cert-manager.io/v1',
|
|
kind: 'Certificate',
|
|
metadata: { namespace, name: 'ql3-minio-ca' },
|
|
spec: {
|
|
isCA: true,
|
|
commonName: 'ql3-minio-ca',
|
|
secretName: 'ql3-minio-ca',
|
|
duration: '24h',
|
|
renewBefore: '1h',
|
|
issuerRef: { name: 'ql3-minio-selfsigned', kind: 'Issuer' },
|
|
},
|
|
},
|
|
{
|
|
apiVersion: 'cert-manager.io/v1',
|
|
kind: 'Issuer',
|
|
metadata: { namespace, name: 'ql3-minio-ca' },
|
|
spec: { ca: { secretName: 'ql3-minio-ca' } },
|
|
},
|
|
{
|
|
apiVersion: 'cert-manager.io/v1',
|
|
kind: 'Certificate',
|
|
metadata: { namespace, name: 'ql3-minio-server' },
|
|
spec: {
|
|
secretName: 'ql3-minio-tls',
|
|
duration: '12h',
|
|
renewBefore: '1h',
|
|
issuerRef: { name: 'ql3-minio-ca', kind: 'Issuer' },
|
|
dnsNames: [
|
|
'ql3-minio',
|
|
'ql3-minio.ql3-dr',
|
|
'ql3-minio.ql3-dr.svc',
|
|
'ql3-minio.ql3-dr.svc.cluster.local',
|
|
],
|
|
},
|
|
},
|
|
{
|
|
apiVersion: 'v1',
|
|
kind: 'PersistentVolumeClaim',
|
|
metadata: { namespace, name: 'ql3-minio-data' },
|
|
spec: {
|
|
accessModes: ['ReadWriteOnce'],
|
|
storageClassName: 'local-path',
|
|
resources: { requests: { storage: '2Gi' } },
|
|
},
|
|
},
|
|
{
|
|
apiVersion: 'v1',
|
|
kind: 'Service',
|
|
metadata: { namespace, name: 'ql3-minio' },
|
|
spec: {
|
|
selector: { app: 'ql3-minio' },
|
|
ports: [
|
|
{ name: 'api', port: 9000, targetPort: 9000 },
|
|
{ name: 'console', port: 9001, targetPort: 9001 },
|
|
],
|
|
},
|
|
},
|
|
{
|
|
apiVersion: 'apps/v1',
|
|
kind: 'Deployment',
|
|
metadata: { namespace, name: 'ql3-minio' },
|
|
spec: {
|
|
replicas: 1,
|
|
selector: { matchLabels: { app: 'ql3-minio' } },
|
|
template: {
|
|
metadata: { labels: { app: 'ql3-minio' } },
|
|
spec: {
|
|
securityContext: {
|
|
fsGroup: 1000,
|
|
fsGroupChangePolicy: 'OnRootMismatch',
|
|
},
|
|
containers: [
|
|
{
|
|
name: 'minio',
|
|
image: minioImage,
|
|
imagePullPolicy: 'IfNotPresent',
|
|
args: [
|
|
'server',
|
|
'--certs-dir',
|
|
'/certs',
|
|
'--console-address',
|
|
':9001',
|
|
'/data',
|
|
],
|
|
env: [
|
|
{ name: 'HOME', value: '/tmp' },
|
|
credentialEnv(
|
|
'MINIO_ROOT_USER',
|
|
'ql3-minio-root',
|
|
'ACCESS_KEY_ID',
|
|
),
|
|
credentialEnv(
|
|
'MINIO_ROOT_PASSWORD',
|
|
'ql3-minio-root',
|
|
'ACCESS_SECRET_KEY',
|
|
),
|
|
],
|
|
ports: [
|
|
{ name: 'api', containerPort: 9000 },
|
|
{ name: 'console', containerPort: 9001 },
|
|
],
|
|
readinessProbe: {
|
|
httpGet: {
|
|
path: '/minio/health/ready',
|
|
port: 'api',
|
|
scheme: 'HTTPS',
|
|
},
|
|
periodSeconds: 2,
|
|
timeoutSeconds: 2,
|
|
failureThreshold: 30,
|
|
},
|
|
resources: {
|
|
requests: { cpu: '25m', memory: '128Mi' },
|
|
limits: { cpu: '500m', memory: '512Mi' },
|
|
},
|
|
securityContext: clientSecurityContext,
|
|
volumeMounts: [
|
|
{ name: 'data', mountPath: '/data' },
|
|
{ name: 'certs', mountPath: '/certs', readOnly: true },
|
|
{ name: 'tmp', mountPath: '/tmp' },
|
|
],
|
|
},
|
|
],
|
|
volumes: [
|
|
{
|
|
name: 'data',
|
|
persistentVolumeClaim: { claimName: 'ql3-minio-data' },
|
|
},
|
|
{
|
|
name: 'certs',
|
|
secret: {
|
|
secretName: 'ql3-minio-tls',
|
|
items: [
|
|
{ key: 'tls.crt', path: 'public.crt' },
|
|
{ key: 'tls.key', path: 'private.key' },
|
|
],
|
|
},
|
|
},
|
|
{ name: 'tmp', emptyDir: {} },
|
|
],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
apiVersion: 'barmancloud.cnpg.io/v1',
|
|
kind: 'ObjectStore',
|
|
metadata: { namespace, name: 'ql3-postgres-backup' },
|
|
spec: {
|
|
retentionPolicy: '30d',
|
|
configuration: {
|
|
destinationPath: 's3://ql3-dr/qinglong3/ql3-postgres',
|
|
endpointURL: 'https://ql3-minio.ql3-dr.svc:9000',
|
|
endpointCA: { name: 'ql3-minio-ca', key: 'tls.crt' },
|
|
s3Credentials: {
|
|
accessKeyId: {
|
|
name: 'ql3-object-store-writer',
|
|
key: 'ACCESS_KEY_ID',
|
|
},
|
|
secretAccessKey: {
|
|
name: 'ql3-object-store-writer',
|
|
key: 'ACCESS_SECRET_KEY',
|
|
},
|
|
},
|
|
wal: { compression: 'lz4', encryption: 'AES256', maxParallel: 2 },
|
|
data: { compression: 'lz4', encryption: 'AES256', jobs: 2 },
|
|
},
|
|
},
|
|
},
|
|
{
|
|
apiVersion: 'barmancloud.cnpg.io/v1',
|
|
kind: 'ObjectStore',
|
|
metadata: { namespace, name: 'ql3-postgres-recovery-source' },
|
|
spec: {
|
|
configuration: {
|
|
destinationPath: 's3://ql3-dr/qinglong3/ql3-postgres',
|
|
endpointURL: 'https://ql3-minio.ql3-dr.svc:9000',
|
|
endpointCA: { name: 'ql3-minio-ca', key: 'tls.crt' },
|
|
s3Credentials: {
|
|
accessKeyId: {
|
|
name: 'ql3-object-store-recovery',
|
|
key: 'ACCESS_KEY_ID',
|
|
},
|
|
secretAccessKey: {
|
|
name: 'ql3-object-store-recovery',
|
|
key: 'ACCESS_SECRET_KEY',
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
job('ql3-minio-bootstrap', bootstrapScript, 3),
|
|
];
|
|
return Object.freeze({
|
|
core: Object.freeze({ apiVersion: 'v1', kind: 'List', items }),
|
|
verifier: Object.freeze({
|
|
apiVersion: 'v1',
|
|
kind: 'List',
|
|
items: [job('ql3-minio-recovery-authority', verifierScript, 0)],
|
|
}),
|
|
});
|
|
}
|
|
|
|
function postgresSourceFixtureResources({ postgresImage }) {
|
|
assert.match(
|
|
postgresImage,
|
|
/^[a-z0-9.-]+(?::[0-9]+)?\/[a-z0-9/_-]+@sha256:[a-f0-9]{64}$/,
|
|
);
|
|
const namespace = 'ql3-dr';
|
|
const clusterName = 'ql3-postgres';
|
|
const pluginName = 'barman-cloud.cloudnative-pg.io';
|
|
const cluster = {
|
|
apiVersion: 'postgresql.cnpg.io/v1',
|
|
kind: 'Cluster',
|
|
metadata: {
|
|
namespace,
|
|
name: clusterName,
|
|
labels: {
|
|
'app.kubernetes.io/name': clusterName,
|
|
'app.kubernetes.io/component': 'database',
|
|
'app.kubernetes.io/part-of': 'qinglong3',
|
|
},
|
|
},
|
|
spec: {
|
|
instances: 3,
|
|
imageName: postgresImage,
|
|
imagePullPolicy: 'IfNotPresent',
|
|
enableSuperuserAccess: false,
|
|
primaryUpdateMethod: 'switchover',
|
|
failoverDelay: 0,
|
|
switchoverDelay: 60,
|
|
smartShutdownTimeout: 60,
|
|
stopDelay: 300,
|
|
bootstrap: {
|
|
initdb: { database: 'qinglong', owner: 'qinglong' },
|
|
},
|
|
plugins: [
|
|
{
|
|
name: pluginName,
|
|
isWALArchiver: true,
|
|
parameters: { barmanObjectName: 'ql3-postgres-backup' },
|
|
},
|
|
],
|
|
postgresql: {
|
|
parameters: {
|
|
max_connections: '100',
|
|
password_encryption: 'scram-sha-256',
|
|
shared_buffers: '64MB',
|
|
synchronous_commit: 'remote_apply',
|
|
},
|
|
synchronous: {
|
|
method: 'any',
|
|
number: 1,
|
|
dataDurability: 'required',
|
|
failoverQuorum: true,
|
|
},
|
|
},
|
|
affinity: {
|
|
enablePodAntiAffinity: true,
|
|
podAntiAffinityType: 'required',
|
|
topologyKey: 'kubernetes.io/hostname',
|
|
},
|
|
managed: { services: { disabledDefaultServices: ['r', 'ro'] } },
|
|
resources: {
|
|
requests: { cpu: '100m', memory: '256Mi' },
|
|
limits: { cpu: '1', memory: '512Mi' },
|
|
},
|
|
storage: { storageClass: 'local-path', size: '1Gi' },
|
|
walStorage: { storageClass: 'local-path', size: '512Mi' },
|
|
},
|
|
};
|
|
const backup = {
|
|
apiVersion: 'postgresql.cnpg.io/v1',
|
|
kind: 'Backup',
|
|
metadata: { namespace, name: 'ql3-postgres-base-backup' },
|
|
spec: {
|
|
cluster: { name: clusterName },
|
|
method: 'plugin',
|
|
pluginConfiguration: { name: pluginName },
|
|
target: 'prefer-standby',
|
|
},
|
|
};
|
|
return Object.freeze({
|
|
cluster: Object.freeze(cluster),
|
|
backup: Object.freeze(backup),
|
|
});
|
|
}
|
|
|
|
function postgresRestoreFixtureResources({
|
|
postgresImage,
|
|
clusterName,
|
|
targetTime,
|
|
}) {
|
|
assert.match(clusterName, /^ql3-postgres-restore-(?:latest|pitr)$/);
|
|
if (targetTime !== undefined) {
|
|
assert.match(
|
|
targetTime,
|
|
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/,
|
|
);
|
|
assert.ok(Number.isFinite(Date.parse(targetTime)));
|
|
}
|
|
assert.equal(clusterName.endsWith('-pitr'), targetTime !== undefined);
|
|
const source = postgresSourceFixtureResources({ postgresImage }).cluster;
|
|
const { bootstrap: _bootstrap, plugins: _plugins, ...haSpec } = source.spec;
|
|
const recovery = { source: 'ql3-postgres-origin' };
|
|
if (targetTime !== undefined) {
|
|
recovery.recoveryTarget = { targetTime };
|
|
}
|
|
return Object.freeze({
|
|
apiVersion: 'postgresql.cnpg.io/v1',
|
|
kind: 'Cluster',
|
|
metadata: {
|
|
namespace: 'ql3-dr',
|
|
name: clusterName,
|
|
labels: {
|
|
'app.kubernetes.io/name': clusterName,
|
|
'app.kubernetes.io/component': 'database-restore-drill',
|
|
'app.kubernetes.io/part-of': 'qinglong3',
|
|
},
|
|
},
|
|
spec: {
|
|
...haSpec,
|
|
bootstrap: { recovery },
|
|
externalClusters: [
|
|
{
|
|
name: 'ql3-postgres-origin',
|
|
plugin: {
|
|
name: 'barman-cloud.cloudnative-pg.io',
|
|
parameters: {
|
|
barmanObjectName: 'ql3-postgres-recovery-source',
|
|
serverName: 'ql3-postgres',
|
|
},
|
|
},
|
|
},
|
|
],
|
|
},
|
|
});
|
|
}
|
|
|
|
function podReady(pod) {
|
|
return pod?.status?.conditions?.some(
|
|
(condition) => condition.type === 'Ready' && condition.status === 'True',
|
|
);
|
|
}
|
|
|
|
function postgresClusterRuntimeEvidence(cluster, pods) {
|
|
const clusterName = cluster?.metadata?.name;
|
|
assert.match(
|
|
clusterName || '',
|
|
/^ql3-postgres(?:-restore-(?:latest|pitr))?$/,
|
|
);
|
|
assert.ok(Array.isArray(pods));
|
|
const clusterPods = pods.filter(
|
|
(pod) => pod?.metadata?.labels?.['cnpg.io/cluster'] === clusterName,
|
|
);
|
|
const readyPods = clusterPods.filter(podReady);
|
|
const nodes = [...new Set(readyPods.map((pod) => pod.spec?.nodeName))].filter(
|
|
Boolean,
|
|
);
|
|
const primary = cluster?.status?.currentPrimary;
|
|
const ready =
|
|
cluster?.spec?.instances === 3 &&
|
|
Number(cluster?.status?.readyInstances) === 3 &&
|
|
typeof primary === 'string' &&
|
|
readyPods.some((pod) => pod.metadata?.name === primary) &&
|
|
readyPods.length === 3 &&
|
|
nodes.length === 3;
|
|
return Object.freeze({
|
|
ready,
|
|
value: ready
|
|
? Object.freeze({
|
|
primary,
|
|
pods: Object.freeze(readyPods),
|
|
nodes: Object.freeze(nodes.sort()),
|
|
})
|
|
: undefined,
|
|
fact: `${readyPods.length}/3 ready Pods across ${
|
|
nodes.length
|
|
}/3 nodes; primary=${primary || 'none'}; statusReady=${String(
|
|
cluster?.status?.readyInstances ?? 0,
|
|
)}`,
|
|
});
|
|
}
|
|
|
|
function postgresRestoreApplicationRuntimeEvidence(
|
|
deployment,
|
|
pods,
|
|
clusterName,
|
|
) {
|
|
assert.match(clusterName, /^ql3-postgres-restore-(?:latest|pitr)$/);
|
|
assert.ok(Array.isArray(pods));
|
|
const expectedName = `ql3-dr-application-${
|
|
clusterName.endsWith('-latest') ? 'latest' : 'pitr'
|
|
}`;
|
|
const matchingPods = pods.filter(
|
|
(pod) =>
|
|
pod?.metadata?.labels?.['app.kubernetes.io/name'] === expectedName &&
|
|
pod?.metadata?.labels?.['ql3.cloud/restore-cluster'] === clusterName,
|
|
);
|
|
const readyPods = matchingPods.filter(podReady);
|
|
const ready =
|
|
deployment?.metadata?.name === expectedName &&
|
|
deployment?.spec?.replicas === 1 &&
|
|
deployment?.status?.availableReplicas === 1 &&
|
|
deployment?.status?.readyReplicas === 1 &&
|
|
readyPods.length === 1;
|
|
return Object.freeze({
|
|
ready,
|
|
value: ready ? Object.freeze({ pod: readyPods[0] }) : undefined,
|
|
fact: `${String(
|
|
readyPods.length,
|
|
)}/1 ready application Pods; available=${String(
|
|
deployment?.status?.availableReplicas ?? 0,
|
|
)}; readyReplicas=${String(deployment?.status?.readyReplicas ?? 0)}`,
|
|
});
|
|
}
|
|
|
|
function backupRuntimeEvidence(backup) {
|
|
const status = backup?.status || {};
|
|
const phase = String(status.phase || '').toLowerCase();
|
|
const startedAt = status.startedAt;
|
|
const stoppedAt = status.stoppedAt;
|
|
const evidence = Object.freeze({
|
|
method: status.method,
|
|
phase,
|
|
targetPod: status.instanceID?.podName,
|
|
backupId: status.backupId,
|
|
beginWal: status.beginWal,
|
|
endWal: status.endWal,
|
|
startedAt,
|
|
stoppedAt,
|
|
});
|
|
const complete =
|
|
phase === 'completed' &&
|
|
status.method === 'plugin' &&
|
|
/^ql3-postgres-[1-9][0-9]*$/.test(status.instanceID?.podName || '') &&
|
|
typeof status.backupId === 'string' &&
|
|
status.backupId.length > 0 &&
|
|
/^[0-9A-F]{24}$/.test(status.beginWal || '') &&
|
|
/^[0-9A-F]{24}$/.test(status.endWal || '') &&
|
|
Number.isFinite(Date.parse(startedAt)) &&
|
|
Number.isFinite(Date.parse(stoppedAt)) &&
|
|
Date.parse(stoppedAt) >= Date.parse(startedAt);
|
|
return Object.freeze({
|
|
ready: complete,
|
|
value: complete ? evidence : undefined,
|
|
fact: `phase=${phase || 'missing'} method=${
|
|
status.method || 'missing'
|
|
} backupId=${
|
|
typeof status.backupId === 'string' && status.backupId.length > 0
|
|
? 'present'
|
|
: 'missing'
|
|
} beginWal=${status.beginWal || 'missing'} endWal=${
|
|
status.endWal || 'missing'
|
|
}`,
|
|
});
|
|
}
|
|
|
|
function postgresArchiverEvidence(row) {
|
|
const evidence = Object.freeze({
|
|
archivedCount: Number(row?.archivedCount),
|
|
failedCount: Number(row?.failedCount),
|
|
lastArchivedWal: row?.lastArchivedWal,
|
|
lastArchivedTime: row?.lastArchivedTime,
|
|
});
|
|
assert.ok(Number.isSafeInteger(evidence.archivedCount));
|
|
assert.ok(evidence.archivedCount >= 1);
|
|
assert.equal(evidence.failedCount, 0);
|
|
assert.match(evidence.lastArchivedWal || '', /^[0-9A-F]{24}$/);
|
|
assert.ok(Number.isFinite(Date.parse(evidence.lastArchivedTime)));
|
|
return evidence;
|
|
}
|
|
|
|
function postgresSql(kubectl, namespace, podName, sql) {
|
|
assert.match(namespace, /^[a-z0-9][a-z0-9-]*$/);
|
|
assert.match(
|
|
podName,
|
|
/^ql3-postgres(?:-restore-(?:latest|pitr))?-[1-9][0-9]*$/,
|
|
);
|
|
assert.equal(typeof sql, 'string');
|
|
assert.ok(sql.length > 0 && !sql.includes('\0'));
|
|
return kubectl(
|
|
[
|
|
'-n',
|
|
namespace,
|
|
'exec',
|
|
podName,
|
|
'-c',
|
|
'postgres',
|
|
'--',
|
|
'psql',
|
|
'--username',
|
|
'postgres',
|
|
'--dbname',
|
|
'qinglong',
|
|
'--no-psqlrc',
|
|
'--tuples-only',
|
|
'--no-align',
|
|
'--set',
|
|
'ON_ERROR_STOP=1',
|
|
'--command',
|
|
sql,
|
|
],
|
|
{ capture: true, quiet: true },
|
|
).stdout;
|
|
}
|
|
|
|
function postgresQueryJson(kubectl, namespace, podName, sql) {
|
|
const output = postgresSql(kubectl, namespace, podName, sql);
|
|
assert.ok(output.length > 0, 'PostgreSQL JSON query returned no rows');
|
|
return JSON.parse(output);
|
|
}
|
|
|
|
function restoreMarkerEvidence(row, expectedAfterMarker) {
|
|
assert.equal(typeof row?.beforeMarkerPresent, 'boolean');
|
|
assert.equal(typeof row?.afterMarkerPresent, 'boolean');
|
|
assert.equal(row.beforeMarkerPresent, true);
|
|
assert.equal(row.afterMarkerPresent, expectedAfterMarker);
|
|
return Object.freeze({
|
|
beforeMarkerPresent: row.beforeMarkerPresent,
|
|
afterMarkerPresent: row.afterMarkerPresent,
|
|
});
|
|
}
|
|
|
|
function postgresMarkerEvidence(row) {
|
|
assert.match(
|
|
row?.id || '',
|
|
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/,
|
|
);
|
|
assert.match(
|
|
row?.createdAt || '',
|
|
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/,
|
|
);
|
|
assert.ok(Number.isFinite(Date.parse(row.createdAt)));
|
|
assert.match(row?.wal || '', /^[0-9A-F]{24}$/);
|
|
return Object.freeze({ id: row.id, createdAt: row.createdAt, wal: row.wal });
|
|
}
|
|
|
|
function auditedCloudNativePgDrEvidence(report) {
|
|
const validation = validateCloudNativePgDrEvidence(report);
|
|
assert.equal(
|
|
validation.compatible,
|
|
true,
|
|
`generated DR evidence is incompatible: ${JSON.stringify(
|
|
validation.findings,
|
|
)}`,
|
|
);
|
|
return Object.freeze(report);
|
|
}
|
|
|
|
function parseEvidenceReportPath(argv) {
|
|
assert.ok(Array.isArray(argv));
|
|
if (argv.length === 0) return undefined;
|
|
assert.equal(
|
|
argv.length,
|
|
1,
|
|
'usage: ql3-cloudnativepg-barman-live-contract [--report=/absolute/path]',
|
|
);
|
|
assert.match(
|
|
argv[0],
|
|
/^--report=\/.{1,4085}$/,
|
|
'report must use --report=/absolute/path',
|
|
);
|
|
const target = argv[0].slice('--report='.length);
|
|
assert.equal(path.isAbsolute(target), true);
|
|
assert.ok(Buffer.byteLength(target, 'utf8') <= 4096);
|
|
assert.notEqual(path.basename(target), '.');
|
|
assert.notEqual(path.basename(target), '..');
|
|
return path.normalize(target);
|
|
}
|
|
|
|
function preflightPrivateEvidenceReportPath(target) {
|
|
assert.equal(path.isAbsolute(target), true);
|
|
const directory = fs.realpathSync(path.dirname(target));
|
|
const directoryStat = fs.lstatSync(directory);
|
|
assert.ok(directoryStat.isDirectory() && !directoryStat.isSymbolicLink());
|
|
const publishedTarget = path.join(directory, path.basename(target));
|
|
try {
|
|
fs.lstatSync(publishedTarget);
|
|
} catch (error) {
|
|
if (error?.code === 'ENOENT') return publishedTarget;
|
|
throw error;
|
|
}
|
|
throw new Error('refusing to overwrite an existing DR evidence report');
|
|
}
|
|
|
|
function writePrivateEvidenceReport(target, report) {
|
|
assert.equal(path.isAbsolute(target), true);
|
|
const directory = fs.realpathSync(path.dirname(target));
|
|
const directoryStat = fs.lstatSync(directory);
|
|
assert.ok(directoryStat.isDirectory() && !directoryStat.isSymbolicLink());
|
|
const publishedTarget = path.join(directory, path.basename(target));
|
|
const bytes = Buffer.from(`${JSON.stringify(report)}\n`, 'utf8');
|
|
assert.ok(bytes.length >= 2 && bytes.length <= 1024 * 1024);
|
|
const temporary = path.join(
|
|
directory,
|
|
`.${path.basename(target)}.${process.pid}-${crypto
|
|
.randomBytes(6)
|
|
.toString('hex')}.tmp`,
|
|
);
|
|
let descriptor;
|
|
try {
|
|
descriptor = fs.openSync(temporary, 'wx', 0o600);
|
|
fs.writeFileSync(descriptor, bytes);
|
|
fs.fsyncSync(descriptor);
|
|
fs.closeSync(descriptor);
|
|
descriptor = undefined;
|
|
fs.linkSync(temporary, publishedTarget);
|
|
fs.unlinkSync(temporary);
|
|
const published = fs.lstatSync(publishedTarget);
|
|
assert.ok(published.isFile() && !published.isSymbolicLink());
|
|
assert.equal(published.size, bytes.length);
|
|
assert.equal(published.mode & 0o077, 0);
|
|
return Object.freeze({
|
|
path: publishedTarget,
|
|
bytes: published.size,
|
|
sha256: `sha256:${sha256(bytes)}`,
|
|
});
|
|
} finally {
|
|
if (descriptor !== undefined) fs.closeSync(descriptor);
|
|
try {
|
|
fs.unlinkSync(temporary);
|
|
} catch (error) {
|
|
if (error?.code !== 'ENOENT') throw error;
|
|
}
|
|
}
|
|
}
|
|
|
|
function postgresRoleEvidence(rows) {
|
|
assert.ok(Array.isArray(rows));
|
|
assert.deepEqual(
|
|
rows.map((role) => role?.name),
|
|
POSTGRES_ROLE_NAMES,
|
|
);
|
|
return Object.freeze(
|
|
rows.map((role) => {
|
|
assert.equal(role.login, true);
|
|
for (const attribute of [
|
|
'superuser',
|
|
'createdb',
|
|
'createrole',
|
|
'replication',
|
|
'bypassrls',
|
|
]) {
|
|
assert.equal(role[attribute], false, `${role.name}.${attribute}`);
|
|
}
|
|
return Object.freeze({
|
|
name: role.name,
|
|
superuser: false,
|
|
createdb: false,
|
|
createrole: false,
|
|
replication: false,
|
|
bypassrls: false,
|
|
});
|
|
}),
|
|
);
|
|
}
|
|
|
|
function postgresDatabaseContractEvidence(row) {
|
|
const migrationCount = Number(row?.migrationCount);
|
|
const controlCoreCapability = Number(row?.controlCoreCapability);
|
|
const postgresVersionNumber = Number(row?.postgresVersionNumber);
|
|
assert.equal(migrationCount, 52);
|
|
assert.equal(controlCoreCapability, 51);
|
|
assert.equal(row?.databaseOwner, 'ql3_migration');
|
|
assert.equal(postgresVersionNumber, 180004);
|
|
return Object.freeze({
|
|
migrationCount,
|
|
controlCoreCapability,
|
|
databaseOwner: row.databaseOwner,
|
|
postgresVersionNumber,
|
|
roles: postgresRoleEvidence(row.roles),
|
|
});
|
|
}
|
|
|
|
function certificateEvidence(secret) {
|
|
const encoded = secret?.data?.['tls.crt'];
|
|
assert.equal(typeof encoded, 'string');
|
|
const certificate = new crypto.X509Certificate(
|
|
Buffer.from(encoded, 'base64'),
|
|
);
|
|
return Object.freeze({
|
|
serialSha256: `sha256:${sha256(
|
|
Buffer.from(certificate.serialNumber, 'ascii'),
|
|
)}`,
|
|
subjectSha256: `sha256:${sha256(Buffer.from(certificate.subject, 'utf8'))}`,
|
|
subjectAltNameSha256: `sha256:${sha256(
|
|
Buffer.from(certificate.subjectAltName || '', 'utf8'),
|
|
)}`,
|
|
resourceVersion: secret.metadata.resourceVersion,
|
|
validFrom: new Date(certificate.validFrom).toISOString(),
|
|
validTo: new Date(certificate.validTo).toISOString(),
|
|
});
|
|
}
|
|
|
|
function certificateRotationEvidence(
|
|
previous,
|
|
current,
|
|
previousRevision,
|
|
currentRevision,
|
|
) {
|
|
assert.match(previous?.serialSha256 || '', /^sha256:[a-f0-9]{64}$/);
|
|
assert.match(current?.serialSha256 || '', /^sha256:[a-f0-9]{64}$/);
|
|
assert.notEqual(previous.serialSha256, current.serialSha256);
|
|
assert.equal(typeof previous.resourceVersion, 'string');
|
|
assert.equal(typeof current.resourceVersion, 'string');
|
|
assert.notEqual(previous.resourceVersion, current.resourceVersion);
|
|
assert.ok(Number.isSafeInteger(previousRevision) && previousRevision >= 1);
|
|
assert.ok(
|
|
Number.isSafeInteger(currentRevision) && currentRevision > previousRevision,
|
|
);
|
|
return Object.freeze({
|
|
previousSerialSha256: previous.serialSha256,
|
|
currentSerialSha256: current.serialSha256,
|
|
previousSecretResourceVersion: previous.resourceVersion,
|
|
currentSecretResourceVersion: current.resourceVersion,
|
|
});
|
|
}
|
|
|
|
function webhookConfigurationHasCaBundle(configuration) {
|
|
const webhooks = configuration?.webhooks;
|
|
return (
|
|
Array.isArray(webhooks) &&
|
|
webhooks.length > 0 &&
|
|
webhooks.every((webhook) => {
|
|
const caBundle = webhook?.clientConfig?.caBundle;
|
|
return (
|
|
typeof caBundle === 'string' &&
|
|
caBundle.length > 0 &&
|
|
/^[A-Za-z0-9+/]+={0,2}$/.test(caBundle) &&
|
|
Buffer.from(caBundle, 'base64').length > 0
|
|
);
|
|
})
|
|
);
|
|
}
|
|
|
|
async function main(argv = process.argv.slice(2)) {
|
|
const requestedEvidenceReportPath = parseEvidenceReportPath(argv);
|
|
const evidenceReportPath =
|
|
requestedEvidenceReportPath === undefined
|
|
? undefined
|
|
: preflightPrivateEvidenceReportPath(requestedEvidenceReportPath);
|
|
if (process.env.QL3_CLOUDNATIVEPG_BARMAN_LIVE !== '1') {
|
|
throw new Error(
|
|
'Refusing Docker/Kubernetes mutation without QL3_CLOUDNATIVEPG_BARMAN_LIVE=1',
|
|
);
|
|
}
|
|
const sourceRevision = process.env.QL3_SOURCE_REVISION;
|
|
assert.match(
|
|
sourceRevision || '',
|
|
/^[a-f0-9]{40,64}$/,
|
|
'QL3_SOURCE_REVISION must bind the exact source commit',
|
|
);
|
|
assertCloudNativePgDrRunnerCapacity();
|
|
|
|
const docker = process.env.QL3_DOCKER_BIN || 'docker';
|
|
const kubectlBinary = process.env.QL3_KUBECTL_BIN || 'kubectl';
|
|
const curl = process.env.QL3_CURL_BIN || 'curl';
|
|
const suffix = `${process.pid}-${crypto.randomBytes(3).toString('hex')}`;
|
|
const prefix = `ql3-barman-dr-${suffix}`;
|
|
assert.match(prefix, /^ql3-barman-dr-[0-9]+-[a-f0-9]{6}$/);
|
|
const dockerLabels = [
|
|
'--label',
|
|
DOCKER_SCOPE_LABEL,
|
|
'--label',
|
|
`${DOCKER_RUN_LABEL_KEY}=${prefix}`,
|
|
];
|
|
const network = `${prefix}-network`;
|
|
const registry = `${prefix}-registry`;
|
|
const server = `${prefix}-server`;
|
|
const agents = [
|
|
`${prefix}-agent-a`,
|
|
`${prefix}-agent-b`,
|
|
`${prefix}-agent-c`,
|
|
];
|
|
const nodes = [server, ...agents];
|
|
const containers = [registry, ...nodes];
|
|
const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-barman-dr-'));
|
|
fs.chmodSync(temporary, 0o700);
|
|
const dockerDataRoot = path.join(temporary, 'docker-data');
|
|
const registryDataBindArgs = privateDockerDataBindArgs(
|
|
dockerDataRoot,
|
|
registry,
|
|
REGISTRY_DATA_TARGETS,
|
|
);
|
|
const nodeDataBindArgs = new Map(
|
|
nodes.map((node) => [
|
|
node,
|
|
privateDockerDataBindArgs(dockerDataRoot, node, K3S_DATA_TARGETS),
|
|
]),
|
|
);
|
|
const kubeconfig = path.join(temporary, 'kubeconfig');
|
|
const registriesConfig = path.join(temporary, 'registries.yaml');
|
|
const noProxy = [
|
|
registry,
|
|
'localhost',
|
|
'127.0.0.1',
|
|
'10.0.0.0/8',
|
|
'172.16.0.0/12',
|
|
'192.168.0.0/16',
|
|
'.svc',
|
|
'.cluster.local',
|
|
].join(',');
|
|
fs.writeFileSync(
|
|
registriesConfig,
|
|
`mirrors:\n "${registry}:5000":\n endpoint:\n - "http://${registry}:5000"\n`,
|
|
{ mode: 0o600, flag: 'wx' },
|
|
);
|
|
let certManagerManifest;
|
|
let barmanManifest;
|
|
let cnpgManifest;
|
|
let clusterControlImage;
|
|
|
|
const clusterToken = crypto.randomBytes(32).toString('base64url');
|
|
const createdContainers = new Set();
|
|
const builtImages = new Set();
|
|
let networkCreated = false;
|
|
const keep = process.env.QL3_KEEP_BARMAN_DR_CLUSTER === '1';
|
|
try {
|
|
const certManagerSource = manifestSource(
|
|
'QL3_CERT_MANAGER_MANIFEST_FILE',
|
|
'/private/tmp/ql3-cert-manager-v1.20.3-release-manifest.yaml',
|
|
curl,
|
|
path.join(temporary, 'cert-manager-source.yaml'),
|
|
CERT_MANAGER_MANIFEST_URL,
|
|
);
|
|
const barmanSource = manifestSource(
|
|
'QL3_BARMAN_MANIFEST_FILE',
|
|
'/private/tmp/ql3-barman-cloud-v0.13.0-release-manifest.yaml',
|
|
curl,
|
|
path.join(temporary, 'barman-source.yaml'),
|
|
BARMAN_MANIFEST_URL,
|
|
);
|
|
const cnpgSource = manifestSource(
|
|
'QL3_CNPG_OPERATOR_MANIFEST_FILE',
|
|
'',
|
|
curl,
|
|
path.join(temporary, 'cnpg-source.yaml'),
|
|
CNPG_MANIFEST_URL,
|
|
);
|
|
run(docker, ['version'], { capture: true, quiet: true });
|
|
const architecture = normalizeArchitecture(
|
|
run(docker, ['info', '--format', '{{.Architecture}}'], {
|
|
capture: true,
|
|
quiet: true,
|
|
}).stdout,
|
|
);
|
|
const expected = PLATFORM_DIGESTS[architecture];
|
|
assert.ok(expected, `unsupported Docker architecture ${architecture}`);
|
|
const runtimeImages = Object.freeze({
|
|
cnpg: `${registry}:5000/ql3/cloudnative-pg@${expected.cnpg}`,
|
|
barman: `${registry}:5000/ql3/barman-cloud@${expected.barman}`,
|
|
barmanSidecar: `${registry}:5000/ql3/barman-cloud-sidecar@${expected.barmanSidecar}`,
|
|
certManager: `${registry}:5000/ql3/cert-manager-controller@${expected.certManager}`,
|
|
certManagerCainjector: `${registry}:5000/ql3/cert-manager-cainjector@${expected.certManagerCainjector}`,
|
|
certManagerWebhook: `${registry}:5000/ql3/cert-manager-webhook@${expected.certManagerWebhook}`,
|
|
minio: `${registry}:5000/ql3/minio@${expected.minio}`,
|
|
minioClient: `${registry}:5000/ql3/minio-client@${expected.minioClient}`,
|
|
postgres: `${registry}:5000/ql3/postgresql@${expected.postgres}`,
|
|
});
|
|
certManagerManifest = reviewedManifest(
|
|
certManagerSource,
|
|
path.join(temporary, 'cert-manager-pinned.yaml'),
|
|
CERT_MANAGER_MANIFEST_SHA256,
|
|
[
|
|
[
|
|
'quay.io/jetstack/cert-manager-controller:v1.20.3',
|
|
runtimeImages.certManager,
|
|
],
|
|
[
|
|
'quay.io/jetstack/cert-manager-cainjector:v1.20.3',
|
|
runtimeImages.certManagerCainjector,
|
|
],
|
|
[
|
|
'quay.io/jetstack/cert-manager-webhook:v1.20.3',
|
|
runtimeImages.certManagerWebhook,
|
|
],
|
|
],
|
|
);
|
|
barmanManifest = reviewedManifest(
|
|
barmanSource,
|
|
path.join(temporary, 'barman-pinned.yaml'),
|
|
BARMAN_MANIFEST_SHA256,
|
|
[
|
|
[
|
|
'ghcr.io/cloudnative-pg/plugin-barman-cloud:v0.13.0',
|
|
runtimeImages.barman,
|
|
],
|
|
[
|
|
' Z2hjci5pby9jbG91ZG5hdGl2ZS1wZy9wbHVnaW4tYmFybWFuLWNsb3VkLXNpZGVjYXI6dj\n AuMTMuMA==',
|
|
` ${Buffer.from(runtimeImages.barmanSidecar).toString('base64')}`,
|
|
],
|
|
],
|
|
);
|
|
cnpgManifest = reviewedManifest(
|
|
cnpgSource,
|
|
path.join(temporary, 'cnpg-pinned.yaml'),
|
|
CNPG_MANIFEST_SHA256,
|
|
[
|
|
['ghcr.io/cloudnative-pg/cloudnative-pg:1.30.0', runtimeImages.cnpg, 2],
|
|
[
|
|
' imagePullPolicy: Always',
|
|
' imagePullPolicy: IfNotPresent',
|
|
],
|
|
],
|
|
);
|
|
|
|
for (const container of containers) {
|
|
assert.equal(
|
|
run(docker, ['inspect', container], {
|
|
capture: true,
|
|
quiet: true,
|
|
allowFailure: true,
|
|
}).status,
|
|
1,
|
|
`refusing to reuse Docker container ${container}`,
|
|
);
|
|
}
|
|
assert.equal(
|
|
run(docker, ['network', 'inspect', network], {
|
|
capture: true,
|
|
quiet: true,
|
|
allowFailure: true,
|
|
}).status,
|
|
1,
|
|
`refusing to reuse Docker network ${network}`,
|
|
);
|
|
|
|
run(docker, ['network', 'create', ...dockerLabels, network], {
|
|
capture: true,
|
|
quiet: true,
|
|
});
|
|
networkCreated = true;
|
|
run(
|
|
docker,
|
|
[
|
|
'run',
|
|
'-d',
|
|
'--network',
|
|
network,
|
|
...dockerLabels,
|
|
'--name',
|
|
registry,
|
|
'-p',
|
|
'127.0.0.1::5000',
|
|
...registryDataBindArgs,
|
|
REGISTRY_IMAGE,
|
|
],
|
|
{ capture: true, quiet: true },
|
|
);
|
|
createdContainers.add(registry);
|
|
const registryPort = run(docker, ['port', registry, '5000/tcp'], {
|
|
capture: true,
|
|
quiet: true,
|
|
}).stdout;
|
|
assert.match(registryPort, /^127\.0\.0\.1:\d+$/);
|
|
await waitFor('local OCI registry readiness', 60_000, () => {
|
|
const result = run(
|
|
curl,
|
|
['--fail', '--silent', `http://${registryPort}/v2/`],
|
|
{
|
|
capture: true,
|
|
quiet: true,
|
|
allowFailure: true,
|
|
},
|
|
);
|
|
return result.status === 0 && result.stdout === '{}'
|
|
? { ready: true, value: true }
|
|
: { ready: false, fact: result.stderr || result.stdout };
|
|
});
|
|
const clusterControlLocalTag = `${registryPort}/ql3/cluster-control:dr-${suffix}`;
|
|
builtImages.add(clusterControlLocalTag);
|
|
clusterControlImage = buildClusterControlImage({
|
|
docker,
|
|
registry,
|
|
registryPort,
|
|
architecture,
|
|
suffix,
|
|
});
|
|
assert.equal(clusterControlImage.localTag, clusterControlLocalTag);
|
|
run(
|
|
docker,
|
|
[
|
|
'run',
|
|
'-d',
|
|
'--privileged',
|
|
'--network',
|
|
network,
|
|
...dockerLabels,
|
|
'--name',
|
|
server,
|
|
'--env',
|
|
`NO_PROXY=${noProxy}`,
|
|
'--env',
|
|
`no_proxy=${noProxy}`,
|
|
'--volume',
|
|
`${registriesConfig}:/etc/rancher/k3s/registries.yaml:ro`,
|
|
...nodeDataBindArgs.get(server),
|
|
'-p',
|
|
'127.0.0.1::6443',
|
|
K3S_IMAGE,
|
|
'server',
|
|
'--token',
|
|
clusterToken,
|
|
'--node-name',
|
|
server,
|
|
'--disable=traefik',
|
|
'--disable=servicelb',
|
|
'--kubelet-arg',
|
|
`eviction-hard=${K3S_EVICTION_HARD}`,
|
|
'--write-kubeconfig-mode=600',
|
|
'--tls-san=127.0.0.1',
|
|
],
|
|
{ capture: true, quiet: true },
|
|
);
|
|
createdContainers.add(server);
|
|
await waitFor('K3s API readiness', 180_000, () => {
|
|
const result = run(
|
|
docker,
|
|
['exec', server, 'kubectl', 'get', '--raw=/readyz'],
|
|
{ capture: true, quiet: true, allowFailure: true },
|
|
);
|
|
return result.status === 0 && result.stdout === 'ok'
|
|
? { ready: true, value: true }
|
|
: { ready: false, fact: result.stderr || result.stdout };
|
|
});
|
|
|
|
for (const agent of agents) {
|
|
run(
|
|
docker,
|
|
[
|
|
'run',
|
|
'-d',
|
|
'--privileged',
|
|
'--network',
|
|
network,
|
|
...dockerLabels,
|
|
'--name',
|
|
agent,
|
|
'--env',
|
|
`NO_PROXY=${noProxy}`,
|
|
'--env',
|
|
`no_proxy=${noProxy}`,
|
|
'--volume',
|
|
`${registriesConfig}:/etc/rancher/k3s/registries.yaml:ro`,
|
|
...nodeDataBindArgs.get(agent),
|
|
K3S_IMAGE,
|
|
'agent',
|
|
'--server',
|
|
`https://${server}:6443`,
|
|
'--token',
|
|
clusterToken,
|
|
'--node-name',
|
|
agent,
|
|
'--kubelet-arg',
|
|
`eviction-hard=${K3S_EVICTION_HARD}`,
|
|
],
|
|
{ capture: true, quiet: true },
|
|
);
|
|
createdContainers.add(agent);
|
|
}
|
|
|
|
const port = run(docker, ['port', server, '6443/tcp'], {
|
|
capture: true,
|
|
quiet: true,
|
|
}).stdout;
|
|
assert.match(port, /^127\.0\.0\.1:\d+$/);
|
|
const config = run(
|
|
docker,
|
|
['exec', server, 'cat', '/etc/rancher/k3s/k3s.yaml'],
|
|
{ capture: true, quiet: true },
|
|
).stdout.replace('https://127.0.0.1:6443', `https://${port}`);
|
|
fs.writeFileSync(kubeconfig, `${config}\n`, { mode: 0o600, flag: 'wx' });
|
|
|
|
const kubectl = (args, options = {}) =>
|
|
run(kubectlBinary, ['--kubeconfig', kubeconfig, ...args], options);
|
|
const kubectlJson = (args) =>
|
|
JSON.parse(
|
|
kubectl([...args, '-o', 'json'], {
|
|
capture: true,
|
|
quiet: true,
|
|
}).stdout,
|
|
);
|
|
|
|
const nodesReady = await waitFor('four K3s nodes', 240_000, () => {
|
|
const nodes = kubectlJson(['get', 'nodes']).items || [];
|
|
const ready = nodes.filter((node) =>
|
|
node.status.conditions?.some(
|
|
(condition) =>
|
|
condition.type === 'Ready' && condition.status === 'True',
|
|
),
|
|
);
|
|
return ready.length === 4
|
|
? { ready: true, value: ready }
|
|
: { ready: false, fact: `${ready.length}/4 Ready nodes` };
|
|
});
|
|
const versions = new Set(
|
|
nodesReady.value.map((node) => node.status.nodeInfo.kubeletVersion),
|
|
);
|
|
assert.equal(versions.size, 1);
|
|
assert.match([...versions][0], /^v1\.32\.8\+k3s1$/);
|
|
|
|
assert.equal(
|
|
mirrorPlatformImage(
|
|
docker,
|
|
registry,
|
|
IMAGES.certManager,
|
|
'cert-manager-controller',
|
|
architecture,
|
|
expected.certManager,
|
|
),
|
|
runtimeImages.certManager,
|
|
);
|
|
assert.equal(
|
|
mirrorPlatformImage(
|
|
docker,
|
|
registry,
|
|
IMAGES.certManagerCainjector,
|
|
'cert-manager-cainjector',
|
|
architecture,
|
|
expected.certManagerCainjector,
|
|
),
|
|
runtimeImages.certManagerCainjector,
|
|
);
|
|
assert.equal(
|
|
mirrorPlatformImage(
|
|
docker,
|
|
registry,
|
|
IMAGES.certManagerWebhook,
|
|
'cert-manager-webhook',
|
|
architecture,
|
|
expected.certManagerWebhook,
|
|
),
|
|
runtimeImages.certManagerWebhook,
|
|
);
|
|
kubectl(['apply', '--server-side', '-f', certManagerManifest.target]);
|
|
kubectl([
|
|
'wait',
|
|
'--for=condition=Established',
|
|
'crd/certificates.cert-manager.io',
|
|
'crd/issuers.cert-manager.io',
|
|
'--timeout=5m',
|
|
]);
|
|
for (const deployment of [
|
|
'cert-manager',
|
|
'cert-manager-cainjector',
|
|
'cert-manager-webhook',
|
|
]) {
|
|
kubectl([
|
|
'-n',
|
|
'cert-manager',
|
|
'rollout',
|
|
'status',
|
|
`deployment/${deployment}`,
|
|
'--timeout=10m',
|
|
]);
|
|
}
|
|
await waitFor('cert-manager webhook CA injection', 300_000, () => {
|
|
const configurations = [
|
|
kubectlJson([
|
|
'get',
|
|
'mutatingwebhookconfiguration',
|
|
'cert-manager-webhook',
|
|
]),
|
|
kubectlJson([
|
|
'get',
|
|
'validatingwebhookconfiguration',
|
|
'cert-manager-webhook',
|
|
]),
|
|
];
|
|
const ready = configurations.filter(webhookConfigurationHasCaBundle);
|
|
return ready.length === configurations.length
|
|
? { ready: true, value: ready.length }
|
|
: {
|
|
ready: false,
|
|
fact: `${String(ready.length)}/${String(
|
|
configurations.length,
|
|
)} webhook configurations have CA bundles`,
|
|
};
|
|
});
|
|
|
|
assert.equal(
|
|
mirrorPlatformImage(
|
|
docker,
|
|
registry,
|
|
IMAGES.cnpg,
|
|
'cloudnative-pg',
|
|
architecture,
|
|
expected.cnpg,
|
|
),
|
|
runtimeImages.cnpg,
|
|
);
|
|
kubectl(['apply', '--server-side', '-f', cnpgManifest.target]);
|
|
kubectl([
|
|
'wait',
|
|
'--for=condition=Established',
|
|
'crd/clusters.postgresql.cnpg.io',
|
|
'--timeout=5m',
|
|
]);
|
|
kubectl([
|
|
'-n',
|
|
'cnpg-system',
|
|
'rollout',
|
|
'status',
|
|
'deployment/cnpg-controller-manager',
|
|
'--timeout=10m',
|
|
]);
|
|
|
|
assert.equal(
|
|
mirrorPlatformImage(
|
|
docker,
|
|
registry,
|
|
IMAGES.barman,
|
|
'barman-cloud',
|
|
architecture,
|
|
expected.barman,
|
|
),
|
|
runtimeImages.barman,
|
|
);
|
|
assert.equal(
|
|
mirrorPlatformImage(
|
|
docker,
|
|
registry,
|
|
IMAGES.barmanSidecar,
|
|
'barman-cloud-sidecar',
|
|
architecture,
|
|
expected.barmanSidecar,
|
|
),
|
|
runtimeImages.barmanSidecar,
|
|
);
|
|
kubectl(['apply', '--server-side', '-f', barmanManifest.target]);
|
|
kubectl([
|
|
'wait',
|
|
'--for=condition=Established',
|
|
'crd/objectstores.barmancloud.cnpg.io',
|
|
'--timeout=5m',
|
|
]);
|
|
kubectl([
|
|
'-n',
|
|
'cnpg-system',
|
|
'rollout',
|
|
'status',
|
|
'deployment/barman-cloud',
|
|
'--timeout=10m',
|
|
]);
|
|
kubectl([
|
|
'-n',
|
|
'cnpg-system',
|
|
'wait',
|
|
'--for=condition=Ready',
|
|
'certificate/barman-cloud-client',
|
|
'certificate/barman-cloud-server',
|
|
'--timeout=5m',
|
|
]);
|
|
|
|
assert.equal(
|
|
mirrorPlatformImage(
|
|
docker,
|
|
registry,
|
|
IMAGES.postgres,
|
|
'postgresql',
|
|
architecture,
|
|
expected.postgres,
|
|
),
|
|
runtimeImages.postgres,
|
|
);
|
|
assert.equal(
|
|
mirrorPlatformImage(
|
|
docker,
|
|
registry,
|
|
IMAGES.minio,
|
|
'minio',
|
|
architecture,
|
|
expected.minio,
|
|
),
|
|
runtimeImages.minio,
|
|
);
|
|
assert.equal(
|
|
mirrorPlatformImage(
|
|
docker,
|
|
registry,
|
|
IMAGES.minioClient,
|
|
'minio-client',
|
|
architecture,
|
|
expected.minioClient,
|
|
),
|
|
runtimeImages.minioClient,
|
|
);
|
|
const minioCredentials = Object.freeze({
|
|
root: Object.freeze({
|
|
accessKey: `QL3ROOT${crypto
|
|
.randomBytes(4)
|
|
.toString('hex')
|
|
.toUpperCase()}`,
|
|
secretKey: crypto.randomBytes(32).toString('base64url'),
|
|
}),
|
|
writer: Object.freeze({
|
|
accessKey: `QL3W${crypto.randomBytes(5).toString('hex').toUpperCase()}`,
|
|
secretKey: crypto.randomBytes(32).toString('base64url'),
|
|
}),
|
|
recovery: Object.freeze({
|
|
accessKey: `QL3R${crypto.randomBytes(5).toString('hex').toUpperCase()}`,
|
|
secretKey: crypto.randomBytes(32).toString('base64url'),
|
|
}),
|
|
});
|
|
const minioFixture = minioFixtureResources({
|
|
minioImage: runtimeImages.minio,
|
|
clientImage: runtimeImages.minioClient,
|
|
credentials: minioCredentials,
|
|
});
|
|
kubectl(['apply', '-f', '-'], {
|
|
input: `${JSON.stringify(minioFixture.core)}\n`,
|
|
});
|
|
kubectl([
|
|
'-n',
|
|
'ql3-dr',
|
|
'wait',
|
|
'--for=condition=Ready',
|
|
'certificate/ql3-minio-ca',
|
|
'certificate/ql3-minio-server',
|
|
'--timeout=5m',
|
|
]);
|
|
kubectl([
|
|
'-n',
|
|
'ql3-dr',
|
|
'rollout',
|
|
'status',
|
|
'deployment/ql3-minio',
|
|
'--timeout=10m',
|
|
]);
|
|
kubectl([
|
|
'-n',
|
|
'ql3-dr',
|
|
'wait',
|
|
'--for=condition=Complete',
|
|
'job/ql3-minio-bootstrap',
|
|
'--timeout=10m',
|
|
]);
|
|
const bootstrapLog = kubectl(
|
|
['-n', 'ql3-dr', 'logs', 'job/ql3-minio-bootstrap'],
|
|
{ capture: true, quiet: true },
|
|
).stdout;
|
|
assert.match(bootstrapLog, /(?:^|\n)QL3_MINIO_AUTHORITY_READY(?:\n|$)/);
|
|
kubectl(['apply', '-f', '-'], {
|
|
input: `${JSON.stringify(minioFixture.verifier)}\n`,
|
|
});
|
|
kubectl([
|
|
'-n',
|
|
'ql3-dr',
|
|
'wait',
|
|
'--for=condition=Complete',
|
|
'job/ql3-minio-recovery-authority',
|
|
'--timeout=5m',
|
|
]);
|
|
const recoveryAuthorityLog = kubectl(
|
|
['-n', 'ql3-dr', 'logs', 'job/ql3-minio-recovery-authority'],
|
|
{ capture: true, quiet: true },
|
|
).stdout;
|
|
assert.match(
|
|
recoveryAuthorityLog,
|
|
/(?:^|\n)QL3_RECOVERY_READ_ONLY_READY(?:\n|$)/,
|
|
);
|
|
const writerObjectStore = kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
'objectstore/ql3-postgres-backup',
|
|
]);
|
|
const recoveryObjectStore = kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
'objectstore/ql3-postgres-recovery-source',
|
|
]);
|
|
assert.equal(writerObjectStore.spec.retentionPolicy, '30d');
|
|
assert.equal(
|
|
writerObjectStore.spec.configuration.s3Credentials.accessKeyId.name,
|
|
'ql3-object-store-writer',
|
|
);
|
|
assert.equal(
|
|
recoveryObjectStore.spec.configuration.s3Credentials.accessKeyId.name,
|
|
'ql3-object-store-recovery',
|
|
);
|
|
assert.equal(recoveryObjectStore.spec.configuration.serverName, undefined);
|
|
assert.equal(
|
|
writerObjectStore.spec.configuration.endpointURL,
|
|
'https://ql3-minio.ql3-dr.svc:9000',
|
|
);
|
|
|
|
const sourceFixture = postgresSourceFixtureResources({
|
|
postgresImage: runtimeImages.postgres,
|
|
});
|
|
kubectl(['apply', '-f', '-'], {
|
|
input: `${JSON.stringify(sourceFixture.cluster)}\n`,
|
|
});
|
|
const sourceReady = await waitFor(
|
|
'three-node PostgreSQL source cluster',
|
|
15 * 60_000,
|
|
() =>
|
|
postgresClusterRuntimeEvidence(
|
|
kubectlJson(['-n', 'ql3-dr', 'get', 'cluster/ql3-postgres']),
|
|
kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
'pods',
|
|
'-l',
|
|
'cnpg.io/cluster=ql3-postgres',
|
|
]).items,
|
|
),
|
|
);
|
|
const sourcePrimary = sourceReady.value.primary;
|
|
const sourcePostgresImages = sourceReady.value.pods.map((pod) =>
|
|
runtimeImageEvidence(pod, 'postgres', IMAGES.postgres, expected.postgres),
|
|
);
|
|
const sourceBarmanSidecarImages = sourceReady.value.pods.map((pod) =>
|
|
runtimeImageEvidence(
|
|
pod,
|
|
'plugin-barman-cloud',
|
|
IMAGES.barmanSidecar,
|
|
expected.barmanSidecar,
|
|
),
|
|
);
|
|
assert.ok(clusterControlImage);
|
|
const postgresRoleCredentials = Object.freeze(
|
|
Object.fromEntries(
|
|
POSTGRES_ROLE_NAMES.map((role) => [
|
|
role,
|
|
crypto.randomBytes(32).toString('base64url'),
|
|
]),
|
|
),
|
|
);
|
|
kubectl(['apply', '-f', '-'], {
|
|
input: `${JSON.stringify(
|
|
postgresRoleSecretResources(postgresRoleCredentials),
|
|
)}\n`,
|
|
});
|
|
kubectl(['-n', 'ql3-dr', 'apply', '-f', POSTGRES_DATABASE_ROLE_MANIFEST]);
|
|
const roleNamesSql = POSTGRES_ROLE_NAMES.map((role) => `'${role}'`).join(
|
|
', ',
|
|
);
|
|
const roleCatalogSql = `SELECT json_build_object(
|
|
'roles', COALESCE((
|
|
SELECT json_agg(json_build_object(
|
|
'name', rolname,
|
|
'login', rolcanlogin,
|
|
'superuser', rolsuper,
|
|
'createdb', rolcreatedb,
|
|
'createrole', rolcreaterole,
|
|
'replication', rolreplication,
|
|
'bypassrls', rolbypassrls
|
|
) ORDER BY rolname)
|
|
FROM pg_roles
|
|
WHERE rolname IN (${roleNamesSql})
|
|
), '[]'::json)
|
|
)::text`;
|
|
const sourceRoles = await waitFor(
|
|
'thirteen reconciled PostgreSQL database roles',
|
|
10 * 60_000,
|
|
() => {
|
|
try {
|
|
const value = postgresRoleEvidence(
|
|
postgresQueryJson(kubectl, 'ql3-dr', sourcePrimary, roleCatalogSql)
|
|
.roles,
|
|
);
|
|
return { ready: true, value };
|
|
} catch (error) {
|
|
return {
|
|
ready: false,
|
|
fact: error instanceof Error ? error.message : String(error),
|
|
};
|
|
}
|
|
},
|
|
);
|
|
assert.equal(sourceRoles.value.length, 13);
|
|
postgresSql(
|
|
kubectl,
|
|
'ql3-dr',
|
|
sourcePrimary,
|
|
'ALTER DATABASE qinglong OWNER TO ql3_migration',
|
|
);
|
|
const migrationJob = postgresMigrationJobResource({
|
|
controlImage: clusterControlImage.runtimeImage,
|
|
});
|
|
kubectl(['apply', '-f', '-'], {
|
|
input: `${JSON.stringify(migrationJob)}\n`,
|
|
});
|
|
kubectl([
|
|
'-n',
|
|
'ql3-dr',
|
|
'wait',
|
|
'--for=condition=Complete',
|
|
'job/ql3-cluster-migration',
|
|
'--timeout=10m',
|
|
]);
|
|
const migrationLog = kubectl(
|
|
['-n', 'ql3-dr', 'logs', 'job/ql3-cluster-migration'],
|
|
{ capture: true, quiet: true },
|
|
).stdout;
|
|
assert.match(migrationLog, /"event":"migration_completed"/);
|
|
const migrationPod = kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
'pods',
|
|
'-l',
|
|
'job-name=ql3-cluster-migration',
|
|
]).items[0];
|
|
const migrationImage = runtimeImageEvidence(
|
|
migrationPod,
|
|
'migration',
|
|
clusterControlImage.runtimeImage,
|
|
clusterControlImage.platformDigest,
|
|
);
|
|
const databaseContractSql = `SELECT json_build_object(
|
|
'migrationCount', (
|
|
SELECT count(*)::integer FROM ql3.schema_migrations
|
|
),
|
|
'controlCoreCapability', (
|
|
SELECT contract_version::integer
|
|
FROM ql3.schema_capabilities
|
|
WHERE contract_name = 'control-core'
|
|
),
|
|
'databaseOwner', (
|
|
SELECT pg_get_userbyid(datdba)
|
|
FROM pg_database
|
|
WHERE datname = current_database()
|
|
),
|
|
'postgresVersionNumber', current_setting('server_version_num')::integer,
|
|
'roles', COALESCE((
|
|
SELECT json_agg(json_build_object(
|
|
'name', rolname,
|
|
'login', rolcanlogin,
|
|
'superuser', rolsuper,
|
|
'createdb', rolcreatedb,
|
|
'createrole', rolcreaterole,
|
|
'replication', rolreplication,
|
|
'bypassrls', rolbypassrls
|
|
) ORDER BY rolname)
|
|
FROM pg_roles
|
|
WHERE rolname IN (${roleNamesSql})
|
|
), '[]'::json)
|
|
)::text`;
|
|
const sourceDatabaseContract = postgresDatabaseContractEvidence(
|
|
postgresQueryJson(kubectl, 'ql3-dr', sourcePrimary, databaseContractSql),
|
|
);
|
|
postgresSql(
|
|
kubectl,
|
|
'ql3-dr',
|
|
sourcePrimary,
|
|
`CREATE TABLE IF NOT EXISTS public.ql3_dr_marker (
|
|
marker_id text PRIMARY KEY,
|
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
|
)`,
|
|
);
|
|
const recordMarker = (markerId) =>
|
|
postgresMarkerEvidence(
|
|
postgresQueryJson(
|
|
kubectl,
|
|
'ql3-dr',
|
|
sourcePrimary,
|
|
`WITH inserted AS (
|
|
INSERT INTO public.ql3_dr_marker (marker_id)
|
|
VALUES ('${markerId}')
|
|
RETURNING marker_id, created_at
|
|
)
|
|
SELECT json_build_object(
|
|
'id', marker_id,
|
|
'createdAt', to_char(
|
|
created_at AT TIME ZONE 'UTC',
|
|
'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'
|
|
),
|
|
'wal', pg_walfile_name(pg_current_wal_lsn())
|
|
)::text
|
|
FROM inserted`,
|
|
),
|
|
);
|
|
const beforeMarker = recordMarker(crypto.randomUUID());
|
|
assert.ok(
|
|
Date.parse(beforeMarker.createdAt) <= Date.now(),
|
|
'before marker timestamp is in the future',
|
|
);
|
|
postgresSql(kubectl, 'ql3-dr', sourcePrimary, 'SELECT pg_switch_wal()');
|
|
const archiverSql = `SELECT json_build_object(
|
|
'archivedCount', archived_count::text,
|
|
'failedCount', failed_count::text,
|
|
'lastArchivedWal', last_archived_wal,
|
|
'lastArchivedTime', to_char(
|
|
last_archived_time AT TIME ZONE 'UTC',
|
|
'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'
|
|
)
|
|
)::text FROM pg_stat_archiver`;
|
|
const walBeforeBackup = await waitFor(
|
|
'continuous WAL archive before base backup',
|
|
10 * 60_000,
|
|
() => ({
|
|
ready: true,
|
|
value: postgresArchiverEvidence(
|
|
postgresQueryJson(kubectl, 'ql3-dr', sourcePrimary, archiverSql),
|
|
),
|
|
}),
|
|
);
|
|
kubectl(['apply', '-f', '-'], {
|
|
input: `${JSON.stringify(sourceFixture.backup)}\n`,
|
|
});
|
|
const baseBackup = await waitFor(
|
|
'plugin base backup completion',
|
|
15 * 60_000,
|
|
() =>
|
|
backupRuntimeEvidence(
|
|
kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
'backup/ql3-postgres-base-backup',
|
|
]),
|
|
),
|
|
);
|
|
assert.notEqual(
|
|
baseBackup.value.targetPod,
|
|
sourcePrimary,
|
|
'prefer-standby backup unexpectedly ran on the primary',
|
|
);
|
|
const pitrTarget = postgresQueryJson(
|
|
kubectl,
|
|
'ql3-dr',
|
|
sourcePrimary,
|
|
`SELECT json_build_object(
|
|
'targetTime', to_char(
|
|
clock_timestamp() AT TIME ZONE 'UTC',
|
|
'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'
|
|
)
|
|
)::text`,
|
|
).targetTime;
|
|
assert.match(pitrTarget, /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/);
|
|
const afterMarker = recordMarker(crypto.randomUUID());
|
|
assert.notEqual(beforeMarker.id, afterMarker.id);
|
|
assert.ok(
|
|
Date.parse(beforeMarker.createdAt) < Date.parse(afterMarker.createdAt),
|
|
'DR markers must be strictly ordered',
|
|
);
|
|
assert.ok(
|
|
Date.parse(beforeMarker.createdAt) < Date.parse(pitrTarget) &&
|
|
Date.parse(pitrTarget) < Date.parse(afterMarker.createdAt),
|
|
'PITR target must be strictly between DR markers',
|
|
);
|
|
postgresSql(kubectl, 'ql3-dr', sourcePrimary, 'SELECT pg_switch_wal()');
|
|
const walAfterTarget = await waitFor(
|
|
'continuous WAL archive after PITR boundary',
|
|
10 * 60_000,
|
|
() => {
|
|
const evidence = postgresArchiverEvidence(
|
|
postgresQueryJson(kubectl, 'ql3-dr', sourcePrimary, archiverSql),
|
|
);
|
|
return {
|
|
ready:
|
|
evidence.archivedCount > walBeforeBackup.value.archivedCount &&
|
|
evidence.lastArchivedWal !== walBeforeBackup.value.lastArchivedWal,
|
|
value: evidence,
|
|
fact: `archived=${evidence.archivedCount} last=${evidence.lastArchivedWal}`,
|
|
};
|
|
},
|
|
);
|
|
|
|
const clientCertificateBefore = kubectlJson([
|
|
'-n',
|
|
'cnpg-system',
|
|
'get',
|
|
'certificate/barman-cloud-client',
|
|
]);
|
|
const serverCertificateBefore = kubectlJson([
|
|
'-n',
|
|
'cnpg-system',
|
|
'get',
|
|
'certificate/barman-cloud-server',
|
|
]);
|
|
const clientSecretBefore = kubectlJson([
|
|
'-n',
|
|
'cnpg-system',
|
|
'get',
|
|
'secret/barman-cloud-client-tls',
|
|
]);
|
|
const serverSecretBefore = kubectlJson([
|
|
'-n',
|
|
'cnpg-system',
|
|
'get',
|
|
'secret/barman-cloud-server-tls',
|
|
]);
|
|
const clientEvidenceBefore = certificateEvidence(clientSecretBefore);
|
|
const serverEvidenceBefore = certificateEvidence(serverSecretBefore);
|
|
const clientRevisionBefore = Number(
|
|
clientCertificateBefore.status?.revision,
|
|
);
|
|
const serverRevisionBefore = Number(
|
|
serverCertificateBefore.status?.revision,
|
|
);
|
|
assert.ok(clientRevisionBefore >= 1 && serverRevisionBefore >= 1);
|
|
const rotationPatch = JSON.stringify({
|
|
spec: {
|
|
duration: '24h',
|
|
renewBefore: '1h',
|
|
privateKey: { rotationPolicy: 'Always' },
|
|
},
|
|
});
|
|
for (const certificate of ['barman-cloud-client', 'barman-cloud-server']) {
|
|
kubectl([
|
|
'-n',
|
|
'cnpg-system',
|
|
'patch',
|
|
`certificate/${certificate}`,
|
|
'--type=merge',
|
|
'--patch',
|
|
rotationPatch,
|
|
]);
|
|
}
|
|
const rotatedCertificates = await waitFor(
|
|
'Barman mutual TLS certificate rotation',
|
|
10 * 60_000,
|
|
() => {
|
|
const clientCertificate = kubectlJson([
|
|
'-n',
|
|
'cnpg-system',
|
|
'get',
|
|
'certificate/barman-cloud-client',
|
|
]);
|
|
const serverCertificate = kubectlJson([
|
|
'-n',
|
|
'cnpg-system',
|
|
'get',
|
|
'certificate/barman-cloud-server',
|
|
]);
|
|
const clientSecret = kubectlJson([
|
|
'-n',
|
|
'cnpg-system',
|
|
'get',
|
|
'secret/barman-cloud-client-tls',
|
|
]);
|
|
const serverSecret = kubectlJson([
|
|
'-n',
|
|
'cnpg-system',
|
|
'get',
|
|
'secret/barman-cloud-server-tls',
|
|
]);
|
|
const clientEvidence = certificateEvidence(clientSecret);
|
|
const serverEvidence = certificateEvidence(serverSecret);
|
|
const clientRotation = certificateRotationEvidence(
|
|
clientEvidenceBefore,
|
|
clientEvidence,
|
|
clientRevisionBefore,
|
|
Number(clientCertificate.status?.revision),
|
|
);
|
|
const serverRotation = certificateRotationEvidence(
|
|
serverEvidenceBefore,
|
|
serverEvidence,
|
|
serverRevisionBefore,
|
|
Number(serverCertificate.status?.revision),
|
|
);
|
|
return {
|
|
ready: true,
|
|
value: {
|
|
clientSecret,
|
|
serverSecret,
|
|
clientEvidence,
|
|
serverEvidence,
|
|
clientRotation,
|
|
serverRotation,
|
|
},
|
|
};
|
|
},
|
|
);
|
|
const clientSecret = rotatedCertificates.value.clientSecret;
|
|
const serverSecret = rotatedCertificates.value.serverSecret;
|
|
const sourcePrimaryAfterRotation = kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
'cluster/ql3-postgres',
|
|
]).status?.currentPrimary;
|
|
assert.match(
|
|
sourcePrimaryAfterRotation || '',
|
|
/^ql3-postgres-[1-9][0-9]*$/,
|
|
);
|
|
postgresSql(
|
|
kubectl,
|
|
'ql3-dr',
|
|
sourcePrimaryAfterRotation,
|
|
'SELECT pg_switch_wal()',
|
|
);
|
|
const walAfterRotation = await waitFor(
|
|
'continuous WAL archive after Barman certificate rotation',
|
|
10 * 60_000,
|
|
() => {
|
|
const evidence = postgresArchiverEvidence(
|
|
postgresQueryJson(
|
|
kubectl,
|
|
'ql3-dr',
|
|
sourcePrimaryAfterRotation,
|
|
archiverSql,
|
|
),
|
|
);
|
|
return {
|
|
ready:
|
|
evidence.archivedCount > walAfterTarget.value.archivedCount &&
|
|
evidence.lastArchivedWal !== walAfterTarget.value.lastArchivedWal,
|
|
value: evidence,
|
|
fact: `archived=${evidence.archivedCount} last=${evidence.lastArchivedWal}`,
|
|
};
|
|
},
|
|
);
|
|
const postRotationBackupResource = {
|
|
...sourceFixture.backup,
|
|
metadata: {
|
|
...sourceFixture.backup.metadata,
|
|
name: 'ql3-postgres-post-rotation-backup',
|
|
},
|
|
};
|
|
kubectl(['apply', '-f', '-'], {
|
|
input: `${JSON.stringify(postRotationBackupResource)}\n`,
|
|
});
|
|
const postRotationBackup = await waitFor(
|
|
'post-rotation plugin base backup completion',
|
|
15 * 60_000,
|
|
() =>
|
|
backupRuntimeEvidence(
|
|
kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
'backup/ql3-postgres-post-rotation-backup',
|
|
]),
|
|
),
|
|
);
|
|
assert.notEqual(
|
|
postRotationBackup.value.targetPod,
|
|
sourcePrimaryAfterRotation,
|
|
'post-rotation prefer-standby backup unexpectedly ran on the primary',
|
|
);
|
|
postgresSql(
|
|
kubectl,
|
|
'ql3-dr',
|
|
sourcePrimaryAfterRotation,
|
|
'SELECT pg_switch_wal()',
|
|
);
|
|
const walBeforeRestores = await waitFor(
|
|
'final continuous WAL archive before recovery drills',
|
|
10 * 60_000,
|
|
() => {
|
|
const evidence = postgresArchiverEvidence(
|
|
postgresQueryJson(
|
|
kubectl,
|
|
'ql3-dr',
|
|
sourcePrimaryAfterRotation,
|
|
archiverSql,
|
|
),
|
|
);
|
|
return {
|
|
ready:
|
|
evidence.archivedCount > walAfterRotation.value.archivedCount &&
|
|
evidence.lastArchivedWal !== walAfterRotation.value.lastArchivedWal,
|
|
value: evidence,
|
|
fact: `archived=${evidence.archivedCount} last=${evidence.lastArchivedWal}`,
|
|
};
|
|
},
|
|
);
|
|
const observedRpoSeconds = Math.max(
|
|
0,
|
|
(Date.now() - Date.parse(walBeforeRestores.value.lastArchivedTime)) /
|
|
1_000,
|
|
);
|
|
assert.ok(observedRpoSeconds <= 60);
|
|
|
|
const sourceClusterBeforeRestores = kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
'cluster/ql3-postgres',
|
|
]);
|
|
const sourceUid = sourceClusterBeforeRestores.metadata.uid;
|
|
assert.match(sourceUid || '', /^[0-9a-f-]{36}$/);
|
|
const markerSql = `SELECT json_build_object(
|
|
'beforeMarkerPresent', EXISTS (
|
|
SELECT 1 FROM public.ql3_dr_marker
|
|
WHERE marker_id = '${beforeMarker.id}'
|
|
),
|
|
'afterMarkerPresent', EXISTS (
|
|
SELECT 1 FROM public.ql3_dr_marker
|
|
WHERE marker_id = '${afterMarker.id}'
|
|
)
|
|
)::text`;
|
|
const restoreCluster = async ({ clusterName, targetTime, expectAfter }) => {
|
|
const restore = postgresRestoreFixtureResources({
|
|
postgresImage: runtimeImages.postgres,
|
|
clusterName,
|
|
targetTime,
|
|
});
|
|
const startedAt = Date.now();
|
|
kubectl(['apply', '-f', '-'], { input: `${JSON.stringify(restore)}\n` });
|
|
const ready = await waitFor(
|
|
`${clusterName} three-node recovery`,
|
|
20 * 60_000,
|
|
() =>
|
|
postgresClusterRuntimeEvidence(
|
|
kubectlJson(['-n', 'ql3-dr', 'get', `cluster/${clusterName}`]),
|
|
kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
'pods',
|
|
'-l',
|
|
`cnpg.io/cluster=${clusterName}`,
|
|
]).items,
|
|
),
|
|
);
|
|
const databaseRtoMs = Date.now() - startedAt;
|
|
const observedCluster = kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
`cluster/${clusterName}`,
|
|
]);
|
|
assert.equal(observedCluster.spec.plugins, undefined);
|
|
assert.deepEqual(
|
|
observedCluster.spec.externalClusters?.[0]?.plugin?.parameters,
|
|
{
|
|
barmanObjectName: 'ql3-postgres-recovery-source',
|
|
serverName: 'ql3-postgres',
|
|
},
|
|
);
|
|
const markers = restoreMarkerEvidence(
|
|
postgresQueryJson(kubectl, 'ql3-dr', ready.value.primary, markerSql),
|
|
expectAfter,
|
|
);
|
|
const databaseContract = postgresDatabaseContractEvidence(
|
|
postgresQueryJson(
|
|
kubectl,
|
|
'ql3-dr',
|
|
ready.value.primary,
|
|
databaseContractSql,
|
|
),
|
|
);
|
|
const applicationProbe = postgresRestoreApplicationProbeResources({
|
|
clusterName,
|
|
controlImage: clusterControlImage.runtimeImage,
|
|
apiCredentialPepper: crypto.randomBytes(32).toString('base64url'),
|
|
});
|
|
kubectl(['apply', '-f', '-'], {
|
|
input: `${JSON.stringify(applicationProbe)}\n`,
|
|
});
|
|
const applicationName = `ql3-dr-application-${
|
|
clusterName.endsWith('-latest') ? 'latest' : 'pitr'
|
|
}`;
|
|
const applicationReady = await waitFor(
|
|
`${clusterName} production application readiness`,
|
|
10 * 60_000,
|
|
() =>
|
|
postgresRestoreApplicationRuntimeEvidence(
|
|
kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
`deployment/${applicationName}`,
|
|
]),
|
|
kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
'pods',
|
|
'-l',
|
|
`app.kubernetes.io/name=${applicationName}`,
|
|
]).items,
|
|
clusterName,
|
|
),
|
|
);
|
|
const applicationImage = runtimeImageEvidence(
|
|
applicationReady.value.pod,
|
|
'cluster-control',
|
|
clusterControlImage.runtimeImage,
|
|
clusterControlImage.platformDigest,
|
|
);
|
|
const applicationRtoMs = Date.now() - startedAt;
|
|
const images = ready.value.pods.map((pod) =>
|
|
runtimeImageEvidence(
|
|
pod,
|
|
'postgres',
|
|
IMAGES.postgres,
|
|
expected.postgres,
|
|
),
|
|
);
|
|
return Object.freeze({
|
|
cluster: clusterName,
|
|
sourceObjectStore: 'ql3-postgres-recovery-source',
|
|
sourceServerName: 'ql3-postgres',
|
|
targetWalArchiver: false,
|
|
instances: 3,
|
|
ready: true,
|
|
primary: ready.value.primary,
|
|
nodes: ready.value.nodes,
|
|
synchronousCommit: 'remote_apply',
|
|
synchronousStandbys: 1,
|
|
migrationCount: databaseContract.migrationCount,
|
|
controlCoreCapability: databaseContract.controlCoreCapability,
|
|
databaseOwner: databaseContract.databaseOwner,
|
|
roles: databaseContract.roles,
|
|
beforeMarkerPresent: markers.beforeMarkerPresent,
|
|
afterMarkerPresent: markers.afterMarkerPresent,
|
|
databaseRtoMs,
|
|
applicationRtoMs,
|
|
applicationReady: true,
|
|
applicationImage,
|
|
images: Object.freeze(images),
|
|
});
|
|
};
|
|
const latestRestore = await restoreCluster({
|
|
clusterName: 'ql3-postgres-restore-latest',
|
|
expectAfter: true,
|
|
});
|
|
kubectl([
|
|
'-n',
|
|
'ql3-dr',
|
|
'delete',
|
|
'deployment/ql3-dr-application-latest',
|
|
'secret/ql3-dr-application-latest-security',
|
|
'--wait=true',
|
|
'--timeout=5m',
|
|
]);
|
|
kubectl([
|
|
'-n',
|
|
'ql3-dr',
|
|
'delete',
|
|
'cluster/ql3-postgres-restore-latest',
|
|
'--wait=true',
|
|
'--timeout=10m',
|
|
]);
|
|
kubectl([
|
|
'-n',
|
|
'ql3-dr',
|
|
'delete',
|
|
'pvc',
|
|
'-l',
|
|
'cnpg.io/cluster=ql3-postgres-restore-latest',
|
|
'--ignore-not-found=true',
|
|
'--wait=true',
|
|
'--timeout=10m',
|
|
]);
|
|
const pitrRestore = await restoreCluster({
|
|
clusterName: 'ql3-postgres-restore-pitr',
|
|
targetTime: pitrTarget,
|
|
expectAfter: false,
|
|
});
|
|
const sourceClusterAfterRestores = kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
'cluster/ql3-postgres',
|
|
]);
|
|
assert.equal(sourceClusterAfterRestores.metadata.uid, sourceUid);
|
|
assert.deepEqual(
|
|
sourceClusterAfterRestores.spec,
|
|
sourceClusterBeforeRestores.spec,
|
|
);
|
|
const sourceAfterRestores = postgresClusterRuntimeEvidence(
|
|
sourceClusterAfterRestores,
|
|
kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
'pods',
|
|
'-l',
|
|
'cnpg.io/cluster=ql3-postgres',
|
|
]).items,
|
|
);
|
|
assert.equal(sourceAfterRestores.ready, true);
|
|
|
|
const minioPod = kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
'pods',
|
|
'-l',
|
|
'app=ql3-minio',
|
|
]).items[0];
|
|
const minioImage = runtimeImageEvidence(
|
|
minioPod,
|
|
'minio',
|
|
IMAGES.minio,
|
|
expected.minio,
|
|
);
|
|
const minioClientPod = kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
'pods',
|
|
'-l',
|
|
'job-name=ql3-minio-bootstrap',
|
|
]).items[0];
|
|
const minioClientImage = runtimeImageEvidence(
|
|
minioClientPod,
|
|
'mc',
|
|
IMAGES.minioClient,
|
|
expected.minioClient,
|
|
);
|
|
|
|
const certManagerPods = kubectlJson([
|
|
'-n',
|
|
'cert-manager',
|
|
'get',
|
|
'pods',
|
|
]).items;
|
|
const certManagerImageKeys = Object.freeze({
|
|
'cert-manager-controller': 'certManager',
|
|
'cert-manager-cainjector': 'certManagerCainjector',
|
|
'cert-manager-webhook': 'certManagerWebhook',
|
|
});
|
|
const certManagerByContainer = Object.fromEntries(
|
|
certManagerPods.flatMap((pod) =>
|
|
(pod.status.containerStatuses || [])
|
|
.filter((status) => certManagerImageKeys[status.name])
|
|
.map((status) => {
|
|
const imageKey = certManagerImageKeys[status.name];
|
|
return [
|
|
status.name,
|
|
runtimeImageEvidence(
|
|
pod,
|
|
status.name,
|
|
IMAGES[imageKey],
|
|
expected[imageKey],
|
|
),
|
|
];
|
|
}),
|
|
),
|
|
);
|
|
assert.deepEqual(
|
|
Object.keys(certManagerByContainer).sort(),
|
|
Object.keys(certManagerImageKeys).sort(),
|
|
);
|
|
|
|
const cnpgPod = kubectlJson([
|
|
'-n',
|
|
'cnpg-system',
|
|
'get',
|
|
'pods',
|
|
'-l',
|
|
'app.kubernetes.io/name=cloudnative-pg',
|
|
]).items[0];
|
|
const cnpgImage = runtimeImageEvidence(
|
|
cnpgPod,
|
|
'manager',
|
|
IMAGES.cnpg,
|
|
expected.cnpg,
|
|
);
|
|
|
|
const barmanPod = kubectlJson([
|
|
'-n',
|
|
'cnpg-system',
|
|
'get',
|
|
'pods',
|
|
'-l',
|
|
'app=barman-cloud',
|
|
]).items[0];
|
|
const barmanImage = runtimeImageEvidence(
|
|
barmanPod,
|
|
'barman-cloud',
|
|
IMAGES.barman,
|
|
expected.barman,
|
|
);
|
|
|
|
const minioCaSecret = kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
'secret/ql3-minio-ca',
|
|
]);
|
|
const minioServerSecret = kubectlJson([
|
|
'-n',
|
|
'ql3-dr',
|
|
'get',
|
|
'secret/ql3-minio-tls',
|
|
]);
|
|
assert.deepEqual(Object.keys(clientSecret.data).sort(), [
|
|
'ca.crt',
|
|
'tls.crt',
|
|
'tls.key',
|
|
]);
|
|
assert.deepEqual(Object.keys(serverSecret.data).sort(), [
|
|
'ca.crt',
|
|
'tls.crt',
|
|
'tls.key',
|
|
]);
|
|
|
|
assert.equal(migrationImage.imageId, clusterControlImage.platformDigest);
|
|
assert.equal(
|
|
latestRestore.applicationImage.imageId,
|
|
clusterControlImage.platformDigest,
|
|
);
|
|
assert.equal(
|
|
pitrRestore.applicationImage.imageId,
|
|
clusterControlImage.platformDigest,
|
|
);
|
|
assert.equal(minioImage.imageId, expected.minio);
|
|
assert.equal(minioClientImage.imageId, expected.minioClient);
|
|
certificateEvidence(minioCaSecret);
|
|
certificateEvidence(minioServerSecret);
|
|
const restoreEvidence = (restore, targetTime) =>
|
|
Object.freeze({
|
|
cluster: restore.cluster,
|
|
sourceObjectStore: restore.sourceObjectStore,
|
|
sourceServerName: restore.sourceServerName,
|
|
sourceClusterUnmodified: true,
|
|
targetWalArchiver: false,
|
|
instances: restore.instances,
|
|
ready: restore.ready,
|
|
migrationCount: restore.migrationCount,
|
|
controlCoreCapability: restore.controlCoreCapability,
|
|
databaseOwner: restore.databaseOwner,
|
|
synchronousCommit: restore.synchronousCommit,
|
|
synchronousStandbys: restore.synchronousStandbys,
|
|
roles: restore.roles,
|
|
beforeMarkerPresent: restore.beforeMarkerPresent,
|
|
afterMarkerPresent: restore.afterMarkerPresent,
|
|
...(targetTime === undefined ? {} : { targetTime }),
|
|
});
|
|
const toSeconds = (milliseconds) =>
|
|
Number((milliseconds / 1_000).toFixed(3));
|
|
const report = auditedCloudNativePgDrEvidence({
|
|
schemaVersion: 1,
|
|
fixture: 'qinglong/cloudnativepg-disaster-recovery@v1',
|
|
observedAt: new Date().toISOString(),
|
|
sourceRevision,
|
|
platform: Object.freeze({
|
|
kubernetesVersion: K3S_VERSION,
|
|
architecture,
|
|
cloudNativePgVersion: CNPG_VERSION,
|
|
cloudNativePgImageId: cnpgImage.imageId,
|
|
postgresVersionNumber: sourceDatabaseContract.postgresVersionNumber,
|
|
postgresImageId: sourcePostgresImages[0].imageId,
|
|
barmanVersion: BARMAN_VERSION,
|
|
barmanControllerImageId: barmanImage.imageId,
|
|
barmanSidecarImageIds: Object.freeze(
|
|
sourceBarmanSidecarImages.map(({ imageId }) => imageId),
|
|
),
|
|
certManagerVersion: CERT_MANAGER_VERSION,
|
|
certManagerImageIds: Object.freeze([
|
|
certManagerByContainer['cert-manager-controller'].imageId,
|
|
certManagerByContainer['cert-manager-cainjector'].imageId,
|
|
certManagerByContainer['cert-manager-webhook'].imageId,
|
|
]),
|
|
}),
|
|
source: Object.freeze({
|
|
cluster: 'ql3-postgres',
|
|
backup: Object.freeze({
|
|
name: 'ql3-postgres-base-backup',
|
|
phase: baseBackup.value.phase,
|
|
startedAt: baseBackup.value.startedAt,
|
|
completedAt: baseBackup.value.stoppedAt,
|
|
beginWal: baseBackup.value.beginWal,
|
|
endWal: baseBackup.value.endWal,
|
|
}),
|
|
markers: Object.freeze({
|
|
before: beforeMarker,
|
|
after: afterMarker,
|
|
}),
|
|
wal: Object.freeze({
|
|
archiveHealthy: true,
|
|
continuous: true,
|
|
noGaps: true,
|
|
lastArchivedWal: walBeforeRestores.value.lastArchivedWal,
|
|
}),
|
|
}),
|
|
latestRestore: restoreEvidence(latestRestore),
|
|
pitrRestore: restoreEvidence(pitrRestore, pitrTarget),
|
|
certificateRotation: Object.freeze({
|
|
client: rotatedCertificates.value.clientRotation,
|
|
server: rotatedCertificates.value.serverRotation,
|
|
walArchivedDuringRotation: true,
|
|
backupCompletedAfterRotation: true,
|
|
latestRestoreCompletedAfterRotation: true,
|
|
pitrCompletedAfterRotation: true,
|
|
maxObservedInterruptionSeconds: toSeconds(
|
|
rotatedCertificates.elapsedMs,
|
|
),
|
|
}),
|
|
objectStoreAuthority: Object.freeze({
|
|
sourceObjectStore: 'ql3-postgres-backup',
|
|
recoveryObjectStore: 'ql3-postgres-recovery-source',
|
|
sourceWriterIdentitySha256: `sha256:${sha256(
|
|
Buffer.from(minioCredentials.writer.accessKey, 'utf8'),
|
|
)}`,
|
|
recoveryReaderIdentitySha256: `sha256:${sha256(
|
|
Buffer.from(minioCredentials.recovery.accessKey, 'utf8'),
|
|
)}`,
|
|
recoveryReadOnly: true,
|
|
versioning: true,
|
|
immutability: true,
|
|
lifecycleDays: 45,
|
|
}),
|
|
serviceLevels: Object.freeze({
|
|
targetMaxRpoSeconds: 60,
|
|
observedRpoSeconds: Number(observedRpoSeconds.toFixed(3)),
|
|
targetMaxDatabaseRtoSeconds: 1_200,
|
|
latestDatabaseRtoSeconds: toSeconds(latestRestore.databaseRtoMs),
|
|
pitrDatabaseRtoSeconds: toSeconds(pitrRestore.databaseRtoMs),
|
|
targetMaxApplicationRtoSeconds: 1_800,
|
|
latestApplicationRtoSeconds: toSeconds(latestRestore.applicationRtoMs),
|
|
pitrApplicationRtoSeconds: toSeconds(pitrRestore.applicationRtoMs),
|
|
}),
|
|
gates: Object.freeze({
|
|
latestRestore: true,
|
|
pointInTimeRestore: true,
|
|
schemaAndRoles: true,
|
|
sourceIsolation: true,
|
|
certificateRotation: true,
|
|
serviceLevels: true,
|
|
passed: true,
|
|
}),
|
|
});
|
|
if (evidenceReportPath !== undefined) {
|
|
writePrivateEvidenceReport(evidenceReportPath, report);
|
|
}
|
|
process.stdout.write(`${JSON.stringify(report)}\n`);
|
|
} catch (error) {
|
|
try {
|
|
for (const container of nodes) {
|
|
const stateResult = run(
|
|
docker,
|
|
['inspect', '--format', '{{json .State}}', container],
|
|
{ capture: true, quiet: true, allowFailure: true },
|
|
);
|
|
if (stateResult.status !== 0) continue;
|
|
const state = JSON.parse(stateResult.stdout);
|
|
const logs = run(docker, ['logs', '--tail', '80', container], {
|
|
capture: true,
|
|
quiet: true,
|
|
allowFailure: true,
|
|
});
|
|
const diagnostic = {
|
|
container,
|
|
state: {
|
|
status: state.Status,
|
|
running: state.Running,
|
|
oomKilled: state.OOMKilled,
|
|
exitCode: state.ExitCode,
|
|
error: state.Error,
|
|
finishedAt: state.FinishedAt,
|
|
},
|
|
logs: redactRuntimeText(
|
|
[logs.stdout, logs.stderr].filter(Boolean).join('\n'),
|
|
[clusterToken],
|
|
).slice(-16_384),
|
|
};
|
|
process.stderr.write(
|
|
`container-diagnostic ${JSON.stringify(diagnostic)}\n`,
|
|
);
|
|
}
|
|
if (fs.existsSync(kubeconfig)) {
|
|
run(
|
|
kubectlBinary,
|
|
['--kubeconfig', kubeconfig, 'get', 'pods', '-A', '-o', 'wide'],
|
|
{ quiet: true, allowFailure: true },
|
|
);
|
|
}
|
|
} catch {
|
|
// Diagnostics cannot replace the original failure.
|
|
}
|
|
throw error;
|
|
} finally {
|
|
if (keep) {
|
|
process.stderr.write(
|
|
`preserving ${prefix}; kubeconfig=${kubeconfig}; network=${network}\n`,
|
|
);
|
|
} else {
|
|
for (const container of [...createdContainers].reverse()) {
|
|
run(docker, ['rm', '-f', '-v', container], {
|
|
capture: true,
|
|
quiet: true,
|
|
allowFailure: true,
|
|
});
|
|
}
|
|
if (networkCreated) {
|
|
run(docker, ['network', 'rm', network], {
|
|
capture: true,
|
|
quiet: true,
|
|
allowFailure: true,
|
|
});
|
|
}
|
|
for (const image of builtImages) {
|
|
run(docker, ['image', 'rm', '-f', image], {
|
|
capture: true,
|
|
quiet: true,
|
|
allowFailure: true,
|
|
});
|
|
}
|
|
fs.rmSync(temporary, { recursive: true, force: true });
|
|
}
|
|
}
|
|
}
|
|
|
|
if (require.main === module) {
|
|
main().catch((error) => {
|
|
process.stderr.write(
|
|
`${
|
|
error instanceof Error ? error.stack || error.message : String(error)
|
|
}\n`,
|
|
);
|
|
process.exitCode = 1;
|
|
});
|
|
}
|
|
|
|
module.exports = {
|
|
assertCloudNativePgDrRunnerCapacity,
|
|
auditedCloudNativePgDrEvidence,
|
|
backupRuntimeEvidence,
|
|
buildClusterControlImage,
|
|
certificateEvidence,
|
|
certificateRotationEvidence,
|
|
digestOnlyReference,
|
|
imageIdDigest,
|
|
kubernetesSecret,
|
|
minioFixtureResources,
|
|
mirrorPlatformImage,
|
|
parseEvidenceReportPath,
|
|
platformDigestFromImageIndex,
|
|
podReady,
|
|
preflightPrivateEvidenceReportPath,
|
|
privateDockerDataBindArgs,
|
|
postgresArchiverEvidence,
|
|
postgresClusterRuntimeEvidence,
|
|
postgresDatabaseContractEvidence,
|
|
postgresMigrationJobResource,
|
|
postgresMarkerEvidence,
|
|
postgresRoleEvidence,
|
|
postgresRoleSecretResources,
|
|
postgresRestoreFixtureResources,
|
|
postgresRestoreApplicationProbeResources,
|
|
postgresRestoreApplicationRuntimeEvidence,
|
|
postgresSourceFixtureResources,
|
|
postgresQueryJson,
|
|
postgresSql,
|
|
redactRuntimeText,
|
|
replaceExactlyOnce,
|
|
restoreMarkerEvidence,
|
|
reviewedManifest,
|
|
runtimeImageEvidence,
|
|
webhookConfigurationHasCaBundle,
|
|
writePrivateEvidenceReport,
|
|
};
|