open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-20 16:07:11 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
be7a3fea41783f655a1514184ea43912b7af9442
qinglong/scripts/ql3-worker-credential-management-kubernetes-live-contract.cjs
T

2835 lines
91 KiB
JavaScript
Raw Blame History

#!/usr/bin/env node
'use strict';
const assert = require('node:assert/strict');
const {
generateKeyPairSync,
randomBytes,
randomUUID,
sign,
} = require('node:crypto');
const fs = require('node:fs');
const yaml = require('js-yaml');
const os = require('node:os');
const path = require('node:path');
const { createRequire } = require('node:module');
const { spawnSync } = require('node:child_process');
const ROOT = path.resolve(__dirname, '..');
const K3S_IMAGE = 'rancher/k3s:v1.34.3-k3s1';
const K3S_DIGEST =
'sha256:71abd3a56f57884c62732e0e0d87606052cb5f8555b7db7e8e33c04570b8175c';
const POSTGRES_IMAGE = 'postgres:18';
const POSTGRES_DATABASE = 'ql3_contract';
const POSTGRES_SUPERUSER_PASSWORD = 'postgres';
const NAMESPACE = 'qinglong3-system';
const DEPLOYMENT = 'ql3-worker-credential-management';
const SERVICE = 'ql3-worker-credential-management';
const IDENTITY_SECRET = 'ql3-worker-credential-management-identity';
const TLS_SECRET = 'ql3-worker-credential-management-tls';
const DATABASE_SECRET = 'ql3-worker-credential-management-database';
const SERVERNAME = `${SERVICE}.${NAMESPACE}.svc`;
const MANAGEMENT_PATH = '/api/v3/worker-credentials/management';
const ISSUER = 'https://identity.qinglong.test/';
const AUDIENCE = 'qinglong3-worker-credential-management';
const POSTGRES_ROLES = Object.freeze({
migration: Object.freeze({
user: 'ql3_migration',
password: 'ql3_migration_test',
}),
runtime: Object.freeze({ user: 'ql3_runtime', password: 'ql3_runtime_test' }),
admin: Object.freeze({ user: 'ql3_admin', password: 'ql3_admin_test' }),
automationManager: Object.freeze({
user: 'ql3_automation_manager',
password: 'ql3_automation_manager_test',
}),
approvalManager: Object.freeze({
user: 'ql3_approval_manager',
password: 'ql3_approval_manager_test',
}),
packageManager: Object.freeze({
user: 'ql3_package_manager',
password: 'ql3_package_manager_test',
}),
packageExecutor: Object.freeze({
user: 'ql3_package_executor',
password: 'ql3_package_executor_test',
}),
workerCredentialManager: Object.freeze({
user: 'ql3_worker_credential_manager',
password: 'ql3_worker_credential_manager_test',
}),
workerCredentialExecutor: Object.freeze({
user: 'ql3_worker_credential_executor',
password: 'ql3_worker_credential_executor_test',
}),
workerIngress: Object.freeze({
user: 'ql3_worker_ingress',
password: 'ql3_worker_ingress_test',
}),
});
const clusterRequire = createRequire(
path.join(ROOT, 'packages/ql3-cluster-admin/package.json'),
);
const HEALTH_SCRIPT = String.raw`
const fs = require('node:fs');
const https = require('node:https');
const [host, rawPort] = process.argv.slice(1);
const ca = fs.readFileSync('/var/run/qinglong3/ca/ca.crt');
function request(path) {
return new Promise((resolve, reject) => {
const outgoing = https.request({
host,
port: Number(rawPort),
path,
method: 'GET',
servername: host,
ca,
minVersion: 'TLSv1.3',
maxVersion: 'TLSv1.3',
rejectUnauthorized: true,
agent: false,
headers: { connection: 'close' }
}, (incoming) => {
const chunks = [];
incoming.on('data', (chunk) => chunks.push(chunk));
incoming.once('end', () => resolve({
path,
statusCode: incoming.statusCode,
protocol: incoming.socket.getProtocol(),
body: JSON.parse(Buffer.concat(chunks).toString('utf8'))
}));
});
outgoing.setTimeout(10000, () => outgoing.destroy(new Error('timeout')));
outgoing.once('error', reject);
outgoing.end();
});
}
(async () => {
let lastError = 'unavailable';
for (let attempt = 1; attempt <= 30; attempt += 1) {
try {
const responses = await Promise.all([request('/readyz'), request('/livez')]);
const output = JSON.stringify({ schemaVersion: 1, attempt, responses });
fs.writeFileSync('/dev/termination-log', output + '\n');
process.stdout.write(output + '\n');
return;
} catch (error) {
lastError = String(error.code || error.name || 'UNKNOWN').slice(0, 64);
}
await new Promise((resolve) => setTimeout(resolve, 1000));
}
const output = JSON.stringify({ schemaVersion: 1, error: lastError });
fs.writeFileSync('/dev/termination-log', output + '\n');
process.stderr.write(output + '\n');
process.exitCode = 1;
})();
`;
const RAW_REQUEST_DIAGNOSTIC_SCRIPT = String.raw`
const fs = require('node:fs');
const https = require('node:https');
const [configFile, commandFile, assertionFile] = process.argv.slice(1);
const config = JSON.parse(fs.readFileSync(configFile, 'utf8'));
const body = fs.readFileSync(commandFile);
const assertion = fs.readFileSync(assertionFile, 'ascii');
const target = new URL(config.endpoint);
const cert = config.clientCertificateFile
? fs.readFileSync(config.clientCertificateFile)
: undefined;
const key = config.clientPrivateKeyFile
? fs.readFileSync(config.clientPrivateKeyFile)
: undefined;
const outgoing = https.request({
protocol: target.protocol,
hostname: target.hostname,
port: Number(target.port),
path: target.pathname,
method: 'POST',
servername: config.servername,
ca: fs.readFileSync(config.caFile),
cert,
key,
minVersion: 'TLSv1.3',
maxVersion: 'TLSv1.3',
rejectUnauthorized: true,
agent: false,
headers: {
accept: 'application/json',
'accept-encoding': 'identity',
authorization: 'Bearer ' + assertion,
connection: 'close',
'content-type': 'application/json',
'content-length': String(body.length)
}
}, (incoming) => {
const chunks = [];
incoming.on('data', (chunk) => chunks.push(chunk));
incoming.once('end', () => {
const responseBody = Buffer.concat(chunks).toString('utf8');
const output = JSON.stringify({
event: 'raw_request_completed',
statusCode: incoming.statusCode,
protocol: incoming.socket.getProtocol(),
rawHeaders: incoming.rawHeaders,
body: JSON.parse(responseBody)
});
fs.writeFileSync('/dev/termination-log', output + '\n');
process.stdout.write(output + '\n');
});
});
outgoing.setTimeout(10000, () => outgoing.destroy(new Error('timeout')));
outgoing.once('error', (error) => {
const output = JSON.stringify({
event: 'raw_request_failed',
code: String(error.code || error.name || 'UNKNOWN').slice(0, 64),
message: String(error.message || '').slice(0, 256)
});
fs.writeFileSync('/dev/termination-log', output + '\n');
process.stdout.write(output + '\n');
});
outgoing.end(body);
`;
const CONNECTIVITY_SCRIPT = String.raw`
const fs = require('node:fs');
const https = require('node:https');
const [host, rawPort] = process.argv.slice(1);
const ca = fs.readFileSync('/var/run/qinglong3/ca/ca.crt');
function probe() {
return new Promise((resolve, reject) => {
const outgoing = https.request({
host,
port: Number(rawPort),
path: '/readyz',
method: 'GET',
servername: host,
ca,
minVersion: 'TLSv1.3',
maxVersion: 'TLSv1.3',
rejectUnauthorized: true,
agent: false,
headers: { connection: 'close' }
}, (incoming) => {
incoming.resume();
incoming.once('end', () => resolve({
statusCode: incoming.statusCode,
protocol: incoming.socket.getProtocol()
}));
});
outgoing.setTimeout(3000, () => outgoing.destroy(new Error('timeout')));
outgoing.once('error', reject);
outgoing.end();
});
}
(async () => {
let lastError = 'unavailable';
for (let attempt = 1; attempt <= 30; attempt += 1) {
try {
const response = await probe();
if (response.statusCode === 200 && response.protocol === 'TLSv1.3') {
const output = JSON.stringify({
event: 'connectivity_ready',
attempts: attempt,
...response
});
fs.writeFileSync('/dev/termination-log', output + '\n');
process.stdout.write(output + '\n');
return;
}
lastError = 'HTTP_' + String(response.statusCode);
} catch (error) {
lastError = String(error.code || error.name || 'UNKNOWN').slice(0, 64);
}
await new Promise((resolve) => setTimeout(resolve, 1000));
}
const output = JSON.stringify({ event: 'connectivity_failed', error: lastError });
fs.writeFileSync('/dev/termination-log', output + '\n');
process.stderr.write(output + '\n');
process.exitCode = 1;
})();
`;
function run(binary, args, options = {}) {
if (!options.quiet) {
process.stderr.write(`+ ${path.basename(binary)} ${args.join(' ')}\n`);
}
const result = spawnSync(binary, args, {
cwd: ROOT,
env: options.env ?? process.env,
input: options.input,
encoding: 'utf8',
maxBuffer: 64 * 1024 * 1024,
stdio: options.capture
? ['pipe', 'pipe', 'pipe']
: [
options.input === undefined ? 'inherit' : 'pipe',
'inherit',
'inherit',
],
});
if (result.error) throw result.error;
if (result.status !== 0 && !options.allowFailure) {
throw new Error(
`${path.basename(binary)} failed with ${String(result.status)}: ` +
`${result.stderr || result.stdout || ''}`,
);
}
return Object.freeze({
status: result.status,
stdout: options.capture ? result.stdout.trim() : '',
stderr: options.capture ? result.stderr.trim() : '',
});
}
function sleep(milliseconds) {
return new Promise((resolve) => setTimeout(resolve, milliseconds));
}
async function waitFor(description, timeoutMs, inspect) {
const startedAt = Date.now();
let last = 'not observed';
while (Date.now() - startedAt < timeoutMs) {
try {
const value = await inspect();
if (value?.ready) return value.value;
if (value?.fact) last = value.fact;
} catch (error) {
last = error instanceof Error ? error.message : String(error);
}
await sleep(500);
}
throw new Error(`${description} timed out: ${last}`);
}
function reviewedKey(kid) {
const { privateKey, publicKey } = generateKeyPairSync('ed25519');
return Object.freeze({
kid,
privateKey,
publicJwk: Object.freeze({
...publicKey.export({ format: 'jwk' }),
alg: 'EdDSA',
kid,
use: 'sig',
}),
});
}
function keyset(generation, keys, revokedKids = []) {
return Object.freeze({
schemaVersion: 1,
generation,
issuer: ISSUER,
audience: AUDIENCE,
keys: keys.map((key) => key.publicJwk),
revokedKids,
assuranceMappings: [
{
acr: 'urn:ql3:mfa',
assurance: 'multi_factor',
requiredAmr: ['pwd', 'otp'],
},
],
constraints: {
maxAssertionBytes: 8 * 1024,
maxLifetimeMs: 5 * 60 * 1000,
maxAuthenticationAgeMs: 5 * 60 * 1000,
clockSkewMs: 5 * 1000,
},
});
}
function assertion(key, subject = 'operator-a', suffix = randomUUID()) {
const now = Math.floor(Date.now() / 1_000);
const header = Buffer.from(
JSON.stringify({
alg: 'EdDSA',
kid: key.kid,
typ: 'ql3-worker-credential-management+jwt',
}),
).toString('base64url');
const payload = Buffer.from(
JSON.stringify({
acr: 'urn:ql3:mfa',
amr: ['pwd', 'otp'],
aud: AUDIENCE,
auth_time: now - 1,
exp: now + 240,
iat: now,
iss: ISSUER,
jti: `ql3-worker-manager-live-${suffix}`,
ql3_purpose: 'worker-credential-management',
sub: subject,
}),
).toString('base64url');
const signed = `${header}.${payload}`;
return `${signed}.${sign(
null,
Buffer.from(signed, 'ascii'),
key.privateKey,
).toString('base64url')}`;
}
function planCommand(index, nowMs = Date.now()) {
return Object.freeze({
schemaVersion: 1,
operation: 'worker-credential.plan',
request: Object.freeze({
action: 'issue',
actionRef: `worker-credential:manager-live:plan-${index}`,
authorityProjectId: 'cluster-authority',
credentialExpiresAtMs: nowMs + 60 * 60_000,
credentialId: `manager_live_credential_${index}`,
credentialNotBeforeAtMs: nowMs + 5 * 60_000,
deliveryId: randomUUID(),
deploymentGeneration: `generation-manager_live_${index}`,
deploymentTargetDigest: 'a'.repeat(64),
previousCredentialId: null,
workerId: `manager-live-worker-${index}`,
}),
});
}
function inspectCommand(actionRef, suffix = randomUUID()) {
return Object.freeze({
schemaVersion: 1,
operation: 'worker-credential.inspect',
request: Object.freeze({
actionRef,
authorityProjectId: 'cluster-authority',
approvalRequestId: `approval-manager-live-${suffix}`,
inspectionId: `inspection-manager-live-${suffix}`,
}),
});
}
async function main() {
const docker = process.env.QL3_DOCKER_BIN || 'docker';
const kubectlBinary = process.env.QL3_KUBECTL_BIN || 'kubectl';
const suffix = process.pid.toString(36);
const network = `ql3-wcm-live-${suffix}`;
const server = `ql3-wcm-server-${suffix}`;
const agents = [`ql3-wcm-agent-a-${suffix}`, `ql3-wcm-agent-b-${suffix}`];
const postgres = `ql3-wcm-postgres-${suffix}`;
const containers = [server, ...agents, postgres];
const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-wcm-live-'));
const kubeconfig = path.join(temporary, 'kubeconfig');
const imageArchive = path.join(temporary, 'admin-image.tar');
const adminImage = `ql3-worker-credential-manager-live:${suffix}`;
const caKey = path.join(temporary, 'ca.key');
const caCertificate = path.join(temporary, 'ca.crt');
const serverKey = path.join(temporary, 'tls.key');
const serverRequest = path.join(temporary, 'tls.csr');
const serverCertificate = path.join(temporary, 'tls.crt');
const serverExtensions = path.join(temporary, 'server.ext');
const clientOldKey = path.join(temporary, 'client-old.key');
const clientOldRequest = path.join(temporary, 'client-old.csr');
const clientOldCertificate = path.join(temporary, 'client-old.crt');
const clientNewKey = path.join(temporary, 'client-new.key');
const clientNewRequest = path.join(temporary, 'client-new.csr');
const clientNewCertificate = path.join(temporary, 'client-new.crt');
const clientCertificateRevocationList = path.join(temporary, 'client.crl');
const caConfig = path.join(temporary, 'ca.cnf');
const caDatabase = path.join(temporary, 'ca.index');
const caSerial = path.join(temporary, 'ca.serial');
const caCrlNumber = path.join(temporary, 'ca.crlnumber');
const caNewCertificates = path.join(temporary, 'ca-new-certificates');
const clusterToken = randomBytes(32).toString('base64url');
let networkCreated = false;
const createdContainers = new Set();
let adminImageBuilt = false;
let migrationDatabase;
try {
run(docker, ['version'], { capture: true, quiet: true });
const k3sImage = JSON.parse(
run(docker, ['image', 'inspect', K3S_IMAGE], {
capture: true,
quiet: true,
}).stdout,
)[0];
assert.ok(k3sImage.RepoDigests.includes(`rancher/k3s@${K3S_DIGEST}`));
const postgresImage = JSON.parse(
run(docker, ['image', 'inspect', POSTGRES_IMAGE], {
capture: true,
quiet: true,
}).stdout,
)[0];
for (const name of containers) {
assert.equal(
run(docker, ['inspect', name], {
capture: true,
quiet: true,
allowFailure: true,
}).status,
1,
`refusing to reuse Docker container ${name}`,
);
}
assert.equal(
run(docker, ['network', 'inspect', network], {
capture: true,
quiet: true,
allowFailure: true,
}).status,
1,
'refusing to reuse Docker network',
);
run(docker, ['network', 'create', network], { capture: true, quiet: true });
networkCreated = true;
run(
docker,
[
'run',
'-d',
'--privileged',
'--network',
network,
'--name',
server,
'-p',
'127.0.0.1::6443',
K3S_IMAGE,
'server',
'--token',
clusterToken,
'--node-name',
server,
'--disable=traefik',
'--disable=servicelb',
'--write-kubeconfig-mode=600',
'--tls-san=127.0.0.1',
],
{ capture: true, quiet: true },
);
createdContainers.add(server);
await waitFor('K3s server readiness', 120_000, () => {
const result = run(
docker,
['exec', server, 'kubectl', 'get', '--raw=/readyz'],
{ capture: true, quiet: true, allowFailure: true },
);
return result.status === 0 && result.stdout === 'ok'
? { ready: true, value: true }
: { ready: false, fact: result.stderr || result.stdout };
});
for (const agent of agents) {
run(
docker,
[
'run',
'-d',
'--privileged',
'--network',
network,
'--name',
agent,
K3S_IMAGE,
'agent',
'--server',
`https://${server}:6443`,
'--token',
clusterToken,
'--node-name',
agent,
],
{ capture: true, quiet: true },
);
createdContainers.add(agent);
}
const port = run(docker, ['port', server, '6443/tcp'], {
capture: true,
quiet: true,
}).stdout;
assert.match(port, /^127\.0\.0\.1:\d+$/);
const config = run(
docker,
['exec', server, 'cat', '/etc/rancher/k3s/k3s.yaml'],
{ capture: true, quiet: true },
).stdout.replace('https://127.0.0.1:6443', `https://${port}`);
fs.writeFileSync(kubeconfig, `${config}\n`, { mode: 0o600, flag: 'wx' });
const kubectl = (args, options = {}) =>
run(kubectlBinary, ['--kubeconfig', kubeconfig, ...args], options);
const kubectlJson = (args) =>
JSON.parse(
kubectl([...args, '-o', 'json'], {
capture: true,
quiet: true,
}).stdout,
);
const apply = (manifest) =>
kubectl(['apply', '-f', '-'], {
input: `${JSON.stringify(manifest)}\n`,
capture: true,
quiet: true,
});
const create = (manifest) =>
kubectl(['create', '-f', '-'], {
input: `${JSON.stringify(manifest)}\n`,
capture: true,
quiet: true,
});
const loadOperationManifest = (name) =>
yaml.load(
fs.readFileSync(
path.join(
ROOT,
'deploy/kubernetes/ql3-cluster/operations/' +
'worker-credential-management-client/base',
name,
),
'utf8',
),
);
await waitFor('three K3s nodes', 180_000, () => {
const nodes = kubectlJson(['get', 'nodes']).items ?? [];
const ready = nodes.filter((node) =>
node.status.conditions?.some(
(condition) =>
condition.type === 'Ready' && condition.status === 'True',
),
);
return ready.length === 3
? { ready: true, value: ready }
: { ready: false, fact: `${ready.length}/3 Ready nodes` };
});
const sourceRevision = run('git', ['rev-parse', 'HEAD'], {
capture: true,
quiet: true,
}).stdout;
run(docker, [
'build',
'--file',
'deploy/containers/ql3-cluster-admin/Dockerfile',
'--tag',
adminImage,
'--build-arg',
`SOURCE_REVISION=${sourceRevision}`,
'.',
]);
adminImageBuilt = true;
run(docker, ['image', 'save', '--output', imageArchive, adminImage]);
for (const node of [server, ...agents]) {
run(docker, ['cp', imageArchive, `${node}:/tmp/ql3-admin.tar`], {
capture: true,
quiet: true,
});
run(
docker,
[
'exec',
node,
'ctr',
'--address',
'/run/k3s/containerd/containerd.sock',
'--namespace',
'k8s.io',
'images',
'import',
'/tmp/ql3-admin.tar',
],
{ capture: true, quiet: true },
);
run(docker, ['exec', node, 'rm', '-f', '/tmp/ql3-admin.tar'], {
capture: true,
quiet: true,
});
}
fs.unlinkSync(imageArchive);
fs.writeFileSync(
serverExtensions,
[
'basicConstraints=CA:FALSE',
'keyUsage=digitalSignature,keyEncipherment',
'extendedKeyUsage=serverAuth',
`subjectAltName=DNS:${SERVERNAME},DNS:${SERVERNAME}.cluster.local`,
'',
].join('\n'),
{ mode: 0o600, flag: 'wx' },
);
run(
'openssl',
[
'req',
'-x509',
'-newkey',
'rsa:2048',
'-nodes',
'-days',
'1',
'-subj',
'/CN=QL3 Worker Manager Live CA',
'-keyout',
caKey,
'-out',
caCertificate,
],
{ capture: true, quiet: true },
);
fs.mkdirSync(caNewCertificates, { mode: 0o700 });
fs.writeFileSync(caDatabase, '', { mode: 0o600, flag: 'wx' });
fs.writeFileSync(caSerial, '1000\n', { mode: 0o600, flag: 'wx' });
fs.writeFileSync(caCrlNumber, '1000\n', { mode: 0o600, flag: 'wx' });
fs.writeFileSync(
caConfig,
[
'[ca]',
'default_ca=client_ca',
'[client_ca]',
`database=${caDatabase}`,
`new_certs_dir=${caNewCertificates}`,
`certificate=${caCertificate}`,
`private_key=${caKey}`,
`serial=${caSerial}`,
`crlnumber=${caCrlNumber}`,
'default_md=sha256',
'default_days=1',
'default_crl_days=1',
'policy=client_policy',
'unique_subject=no',
'copy_extensions=none',
'[client_policy]',
'commonName=supplied',
'[client_certificate]',
'basicConstraints=critical,CA:FALSE',
'keyUsage=critical,digitalSignature,keyEncipherment',
'extendedKeyUsage=clientAuth',
'',
].join('\n'),
{ mode: 0o600, flag: 'wx' },
);
for (const [commonName, keyFile, requestFile, certificateFile] of [
[
'ql3-worker-credential-management-client-old',
clientOldKey,
clientOldRequest,
clientOldCertificate,
],
[
'ql3-worker-credential-management-client-new',
clientNewKey,
clientNewRequest,
clientNewCertificate,
],
]) {
run(
'openssl',
[
'req',
'-newkey',
'rsa:2048',
'-nodes',
'-subj',
`/CN=${commonName}`,
'-keyout',
keyFile,
'-out',
requestFile,
],
{ capture: true, quiet: true },
);
run(
'openssl',
[
'ca',
'-batch',
'-notext',
'-config',
caConfig,
'-extensions',
'client_certificate',
'-in',
requestFile,
'-out',
certificateFile,
],
{ capture: true, quiet: true },
);
}
run(
'openssl',
[
'ca',
'-gencrl',
'-config',
caConfig,
'-out',
clientCertificateRevocationList,
],
{ capture: true, quiet: true },
);
run(
'openssl',
[
'req',
'-newkey',
'rsa:2048',
'-nodes',
'-subj',
`/CN=${SERVERNAME}`,
'-keyout',
serverKey,
'-out',
serverRequest,
],
{ capture: true, quiet: true },
);
run(
'openssl',
[
'x509',
'-req',
'-days',
'1',
'-in',
serverRequest,
'-CA',
caCertificate,
'-CAkey',
caKey,
'-CAcreateserial',
'-extfile',
serverExtensions,
'-out',
serverCertificate,
],
{ capture: true, quiet: true },
);
run(
docker,
[
'run',
'-d',
'--network',
network,
'--name',
postgres,
'-p',
'127.0.0.1::5432',
'-e',
`POSTGRES_DB=${POSTGRES_DATABASE}`,
'-e',
'POSTGRES_USER=postgres',
'-e',
`POSTGRES_PASSWORD=${POSTGRES_SUPERUSER_PASSWORD}`,
POSTGRES_IMAGE,
],
{ capture: true, quiet: true },
);
createdContainers.add(postgres);
await waitFor('PostgreSQL final TCP readiness', 60_000, () => {
const result = run(
docker,
[
'exec',
postgres,
'pg_isready',
'-h',
'127.0.0.1',
'-U',
'postgres',
'-d',
POSTGRES_DATABASE,
],
{ capture: true, quiet: true, allowFailure: true },
);
return result.status === 0
? { ready: true, value: true }
: { ready: false, fact: result.stderr || result.stdout };
});
const postgresEndpoint = run(docker, ['port', postgres, '5432/tcp'], {
capture: true,
quiet: true,
}).stdout;
assert.match(postgresEndpoint, /^127\.0\.0\.1:\d+$/);
const postgresPort = Number(
postgresEndpoint.slice(postgresEndpoint.lastIndexOf(':') + 1),
);
const postgresAddress = run(
docker,
[
'inspect',
'--format',
'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}',
postgres,
],
{ capture: true, quiet: true },
).stdout;
assert.match(postgresAddress, /^\d{1,3}(?:\.\d{1,3}){3}$/);
const { createPostgresDatabaseOpener } = clusterRequire(
'@qinglong/cluster-postgres/worker-credential-manager',
);
const { runPostgresMigrations } = clusterRequire(
'@qinglong/cluster-postgres/migration',
);
const databaseUrl = ({ user, password }) =>
`postgresql://${encodeURIComponent(user)}:${encodeURIComponent(
password,
)}` + `@127.0.0.1:${postgresPort}/${POSTGRES_DATABASE}`;
const openDatabase = (role, credential, applicationName) =>
createPostgresDatabaseOpener({
role,
connection: {
connectionString: databaseUrl(credential),
tls: { mode: 'disable' },
},
pool: {
applicationName,
maxConnections: 1,
connectionTimeoutMs: 2_000,
},
onPoolError() {},
});
const openMigrationDatabase = openDatabase(
'migration',
POSTGRES_ROLES.migration,
'ql3-worker-manager-live-migration',
);
const bootstrap = await openDatabase(
'migration',
{ user: 'postgres', password: POSTGRES_SUPERUSER_PASSWORD },
'ql3-worker-manager-live-bootstrap',
)();
try {
for (const credential of Object.values(POSTGRES_ROLES)) {
await bootstrap.pool.query(
`CREATE ROLE ${credential.user} LOGIN PASSWORD '${credential.password}'`,
);
}
await bootstrap.pool.query(
`ALTER DATABASE ${POSTGRES_DATABASE} OWNER TO ` +
POSTGRES_ROLES.migration.user,
);
} finally {
await bootstrap.close();
}
migrationDatabase = await openMigrationDatabase();
await runPostgresMigrations({ pool: migrationDatabase.pool });
const seededAtMs = Date.now();
await migrationDatabase.pool.query(
`INSERT INTO "ql3"."projects" (
id, name, slug, status, version, created_at_ms, updated_at_ms
) VALUES (
'cluster-authority', 'Cluster Authority', 'cluster-authority',
'active', 1, $1, $1
)`,
[seededAtMs],
);
await migrationDatabase.pool.query(
`INSERT INTO "ql3"."project_role_bindings" (
project_id, subject_type, subject_id, version, state, role,
mutation_id, changed_by_type, changed_by_id, created_at_ms
) VALUES (
'cluster-authority', 'user', 'operator-a', 1, 'active', 'admin',
'binding-operator-a-v1', 'user', 'owner-a', $1
)`,
[seededAtMs],
);
const oldKey = reviewedKey('worker-manager-key-1');
const newKey = reviewedKey('worker-manager-key-2');
const keyset1 = keyset(1, [oldKey]);
const keyset2 = keyset(2, [oldKey, newKey]);
const keyset3 = keyset(3, [oldKey, newKey], [oldKey.kid]);
const caText = fs.readFileSync(caCertificate, 'utf8');
const tlsCertificateText = fs.readFileSync(serverCertificate, 'utf8');
const tlsKeyText = fs.readFileSync(serverKey, 'utf8');
const clientOldCertificateText = fs.readFileSync(
clientOldCertificate,
'utf8',
);
const clientOldKeyText = fs.readFileSync(clientOldKey, 'utf8');
const clientNewCertificateText = fs.readFileSync(
clientNewCertificate,
'utf8',
);
const clientNewKeyText = fs.readFileSync(clientNewKey, 'utf8');
let activeClientCertificateText = clientOldCertificateText;
let activeClientKeyText = clientOldKeyText;
let clientCrlText = fs.readFileSync(
clientCertificateRevocationList,
'utf8',
);
const managerUrl =
`postgresql://${encodeURIComponent(
POSTGRES_ROLES.workerCredentialManager.user,
)}` +
`:${encodeURIComponent(
POSTGRES_ROLES.workerCredentialManager.password,
)}` +
`@${postgresAddress}:5432/${POSTGRES_DATABASE}`;
apply({
apiVersion: 'v1',
kind: 'Namespace',
metadata: { name: NAMESPACE },
});
apply({
apiVersion: 'v1',
kind: 'Secret',
metadata: { name: TLS_SECRET, namespace: NAMESPACE },
type: 'kubernetes.io/tls',
stringData: {
'tls.crt': tlsCertificateText,
'tls.key': tlsKeyText,
'ca.crt': caText,
'client.crl': clientCrlText,
},
});
const applyIdentity = (document) =>
apply({
apiVersion: 'v1',
kind: 'Secret',
metadata: { name: IDENTITY_SECRET, namespace: NAMESPACE },
type: 'Opaque',
stringData: { 'keyset.json': `${JSON.stringify(document)}\n` },
});
applyIdentity(keyset1);
apply({
apiVersion: 'v1',
kind: 'Secret',
metadata: { name: DATABASE_SECRET, namespace: NAMESPACE },
type: 'Opaque',
stringData: { 'postgres-worker-credential-manager-url': managerUrl },
});
apply({
apiVersion: 'v1',
kind: 'ServiceAccount',
metadata: { name: DEPLOYMENT, namespace: NAMESPACE },
automountServiceAccountToken: false,
});
apply({
apiVersion: 'v1',
kind: 'Service',
metadata: { name: SERVICE, namespace: NAMESPACE },
spec: {
type: 'ClusterIP',
selector: {
'app.kubernetes.io/name': DEPLOYMENT,
'app.kubernetes.io/component': 'worker-credential-management',
},
ports: [
{ name: 'https', port: 8444, targetPort: 'https', protocol: 'TCP' },
],
},
});
apply({
apiVersion: 'policy/v1',
kind: 'PodDisruptionBudget',
metadata: { name: DEPLOYMENT, namespace: NAMESPACE },
spec: {
minAvailable: 1,
selector: {
matchLabels: {
'app.kubernetes.io/name': DEPLOYMENT,
'app.kubernetes.io/component': 'worker-credential-management',
},
},
},
});
apply({
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: { name: DEPLOYMENT, namespace: NAMESPACE },
spec: {
podSelector: {
matchLabels: {
'app.kubernetes.io/name': DEPLOYMENT,
'app.kubernetes.io/component': 'worker-credential-management',
},
},
policyTypes: ['Ingress', 'Egress'],
ingress: [
{
from: [
{
podSelector: {
matchLabels: {
'qinglong.io/worker-credential-management-client': 'true',
},
},
},
],
ports: [{ protocol: 'TCP', port: 8444 }],
},
],
egress: [
{
to: [{ ipBlock: { cidr: `${postgresAddress}/32` } }],
ports: [{ protocol: 'TCP', port: 5432 }],
},
],
},
});
apply({
apiVersion: 'apps/v1',
kind: 'Deployment',
metadata: { name: DEPLOYMENT, namespace: NAMESPACE },
spec: {
replicas: 2,
minReadySeconds: 2,
revisionHistoryLimit: 3,
progressDeadlineSeconds: 300,
strategy: {
type: 'RollingUpdate',
rollingUpdate: { maxUnavailable: 0, maxSurge: 1 },
},
selector: {
matchLabels: {
'app.kubernetes.io/name': DEPLOYMENT,
'app.kubernetes.io/component': 'worker-credential-management',
},
},
template: {
metadata: {
labels: {
'app.kubernetes.io/name': DEPLOYMENT,
'app.kubernetes.io/component': 'worker-credential-management',
'app.kubernetes.io/part-of': 'qinglong3',
},
annotations: { 'qinglong.io/identity-generation': '1' },
},
spec: {
serviceAccountName: DEPLOYMENT,
automountServiceAccountToken: false,
terminationGracePeriodSeconds: 10,
securityContext: {
runAsNonRoot: true,
runAsUser: 10001,
runAsGroup: 10001,
fsGroup: 10001,
seccompProfile: { type: 'RuntimeDefault' },
},
affinity: {
podAntiAffinity: {
requiredDuringSchedulingIgnoredDuringExecution: [
{
topologyKey: 'kubernetes.io/hostname',
labelSelector: {
matchLabels: {
'app.kubernetes.io/name': DEPLOYMENT,
'app.kubernetes.io/component':
'worker-credential-management',
},
},
},
],
},
},
containers: [
{
name: 'management',
image: adminImage,
imagePullPolicy: 'Never',
command: [
'node',
'/opt/qinglong/node_modules/@qinglong/cluster-admin/dist/' +
'worker-credential/management-server/workerCredentialManagementCli.js',
],
securityContext: {
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
capabilities: { drop: ['ALL'] },
},
env: [
{ name: 'QL3_PROFILE', value: 'cluster-admin' },
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_ENABLED',
value: 'true',
},
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_HOST',
value: '0.0.0.0',
},
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_PORT',
value: '8444',
},
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_TLS_CERT_FILE',
value: '/var/run/secrets/qinglong3/management-tls/tls.crt',
},
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_TLS_KEY_FILE',
value: '/var/run/secrets/qinglong3/management-tls/tls.key',
},
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_CLIENT_CA_FILE',
value: '/var/run/secrets/qinglong3/management-tls/ca.crt',
},
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_CLIENT_CRL_FILE',
value:
'/var/run/secrets/qinglong3/management-tls/client.crl',
},
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_IDENTITY_KEYSET_FILE',
value: '/var/run/qinglong3/management-identity/keyset.json',
},
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_QUOTA_WINDOW_MS',
value: '300000',
},
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_PLAN_QUOTA',
value: '8',
},
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_PROPOSE_QUOTA',
value: '32',
},
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_DECIDE_QUOTA',
value: '32',
},
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_INSPECT_QUOTA',
value: '128',
},
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_MAX_CONNECTIONS',
value: '64',
},
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_MAX_CONCURRENT_REQUESTS',
value: '32',
},
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_RATE_WINDOW_MS',
value: '300000',
},
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_PEER_REQUEST_LIMIT',
value: '128',
},
{
name: 'QL3_WORKER_CREDENTIAL_MANAGEMENT_GLOBAL_REQUEST_LIMIT',
value: '10000',
},
{
name: 'QL3_POSTGRES_WORKER_CREDENTIAL_MANAGER_URL',
valueFrom: {
secretKeyRef: {
name: DATABASE_SECRET,
key: 'postgres-worker-credential-manager-url',
},
},
},
{
name: 'QL3_POSTGRES_WORKER_CREDENTIAL_MANAGER_TLS_MODE',
value: 'disable',
},
{
name: 'QL3_POSTGRES_WORKER_CREDENTIAL_MANAGER_ALLOW_INSECURE',
value: 'true',
},
{
name: 'QL3_POSTGRES_WORKER_CREDENTIAL_MANAGER_MAX_CONNECTIONS',
value: '2',
},
{
name: 'QL3_POSTGRES_WORKER_CREDENTIAL_MANAGER_APPLICATION_NAME',
value: 'ql3-worker-manager-live',
},
],
ports: [
{ name: 'https', containerPort: 8444, protocol: 'TCP' },
],
startupProbe: {
httpGet: { path: '/livez', port: 'https', scheme: 'HTTPS' },
periodSeconds: 2,
timeoutSeconds: 1,
failureThreshold: 45,
},
readinessProbe: {
httpGet: { path: '/readyz', port: 'https', scheme: 'HTTPS' },
periodSeconds: 2,
timeoutSeconds: 1,
failureThreshold: 2,
},
livenessProbe: {
httpGet: { path: '/livez', port: 'https', scheme: 'HTTPS' },
periodSeconds: 10,
timeoutSeconds: 2,
failureThreshold: 3,
},
resources: {
requests: { cpu: '50m', memory: '96Mi' },
limits: { cpu: '1', memory: '384Mi' },
},
volumeMounts: [
{ name: 'tmp', mountPath: '/tmp' },
{
name: 'management-tls',
mountPath: '/var/run/secrets/qinglong3/management-tls',
readOnly: true,
},
{
name: 'management-identity',
mountPath: '/var/run/qinglong3/management-identity',
readOnly: true,
},
],
},
],
volumes: [
{
name: 'tmp',
emptyDir: { medium: 'Memory', sizeLimit: '16Mi' },
},
{
name: 'management-tls',
secret: {
secretName: TLS_SECRET,
defaultMode: 288,
items: [
{ key: 'tls.crt', path: 'tls.crt' },
{ key: 'tls.key', path: 'tls.key' },
{ key: 'ca.crt', path: 'ca.crt' },
{ key: 'client.crl', path: 'client.crl' },
],
},
},
{
name: 'management-identity',
secret: {
secretName: IDENTITY_SECRET,
defaultMode: 292,
items: [{ key: 'keyset.json', path: 'keyset.json' }],
},
},
],
},
},
},
});
const readyManagerPods = async (excludedUids = new Set()) =>
waitFor('two Ready manager Pods on distinct nodes', 180_000, () => {
const pods = kubectlJson([
'-n',
NAMESPACE,
'get',
'pods',
'-l',
`app.kubernetes.io/name=${DEPLOYMENT}`,
]).items.filter(
(pod) =>
pod.metadata.deletionTimestamp === undefined &&
!excludedUids.has(pod.metadata.uid) &&
pod.status.conditions?.some(
(condition) =>
condition.type === 'Ready' && condition.status === 'True',
),
);
const nodes = new Set(pods.map((pod) => pod.spec.nodeName));
return pods.length === 2 && nodes.size === 2
? {
ready: true,
value: pods.sort((a, b) =>
a.metadata.name.localeCompare(b.metadata.name),
),
}
: {
ready: false,
fact: `${pods.length} Ready Pods on ${nodes.size} nodes`,
};
});
const patchGeneration = (generation) =>
kubectl(
[
'-n',
NAMESPACE,
'patch',
'deployment',
DEPLOYMENT,
'--type=merge',
'-p',
JSON.stringify({
spec: {
template: {
metadata: {
annotations: {
'qinglong.io/identity-generation': generation,
},
},
},
},
}),
],
{ capture: true, quiet: true },
);
const waitRollout = () =>
kubectl(
[
'-n',
NAMESPACE,
'rollout',
'status',
`deployment/${DEPLOYMENT}`,
'--timeout=180s',
],
{ capture: true, quiet: true },
);
waitRollout();
const generation1Pods = await readyManagerPods();
assert.equal(
new Set(generation1Pods.map((pod) => pod.spec.nodeName)).size,
2,
);
for (const pod of generation1Pods) {
assert.equal(pod.spec.automountServiceAccountToken, false);
assert.equal(pod.spec.serviceAccountName, DEPLOYMENT);
assert.equal(
pod.spec.volumes.some((volume) =>
volume.projected?.sources?.some(
(source) => source.serviceAccountToken !== undefined,
),
),
false,
);
}
const clientShell = [
'umask 077',
'cp /var/run/qinglong3/client/client.json /tmp/client.json',
'cp /var/run/qinglong3/client/command.json /tmp/command.json',
'cp /var/run/qinglong3/client/assertion.jwt /tmp/assertion.jwt',
'cp /var/run/qinglong3/client/ca.crt /tmp/ca.crt',
'cp /var/run/qinglong3/client/client.crt /tmp/client.crt',
'cp /var/run/qinglong3/client/client.key /tmp/client.key',
'chmod 600 /tmp/client.json /tmp/command.json /tmp/assertion.jwt ' +
'/tmp/ca.crt /tmp/client.crt /tmp/client.key',
'set +e',
'attempt=0',
'while true; do',
' attempt=$((attempt + 1))',
' output="$(node /opt/qinglong/node_modules/@qinglong/cluster-admin/dist/' +
'worker-credential/workerCredentialManagementClientCli.js ' +
'--config=/tmp/client.json --command=/tmp/command.json ' +
'--assertion=/tmp/assertion.jwt 2>&1)"',
' status=$?',
' if [ "$status" -eq 0 ] || ! printf \'%s\' "$output" | ' +
'grep -q QL3_PLUGIN_PACKAGE_MANAGEMENT_CLIENT_REQUEST_FAILED || ' +
'[ "$attempt" -ge 12 ]; then',
' break',
' fi',
' sleep 1',
'done',
'if [ "$status" -ne 0 ] && printf \'%s\' "$output" | ' +
'grep -q QL3_PLUGIN_PACKAGE_MANAGEMENT_CLIENT_REQUEST_FAILED; then',
' diagnostic="$(node -e "$QL3_RAW_REQUEST_DIAGNOSTIC_SCRIPT" -- ' +
'/tmp/client.json /tmp/command.json /tmp/assertion.jwt 2>&1)"',
' output="{\\"client\\":$output,\\"diagnostic\\":$diagnostic}"',
'fi',
'printf \'{"transportAttempts":%s}\\n%s\\n\' ' +
'"$attempt" "$output" > /dev/termination-log',
'printf \'%s\\n\' "$output"',
'exit "$status"',
].join('\n');
const clientConfig = Object.freeze({
schemaVersion: 1,
endpoint: `https://${SERVERNAME}:8444${MANAGEMENT_PATH}`,
servername: SERVERNAME,
caFile: '/tmp/ca.crt',
clientCertificateFile: '/tmp/client.crt',
clientPrivateKeyFile: '/tmp/client.key',
requestTimeoutMs: 15_000,
});
const createClientJob = ({
name,
target,
command,
bearer,
clientCertificateText = activeClientCertificateText,
clientKeyText = activeClientKeyText,
}) => {
const inputSecret = `${name}-input`;
create({
apiVersion: 'v1',
kind: 'Secret',
metadata: { name: inputSecret, namespace: NAMESPACE },
type: 'Opaque',
stringData: {
'client.json': `${JSON.stringify(clientConfig)}\n`,
'command.json': `${JSON.stringify(command)}\n`,
'assertion.jwt': bearer,
'ca.crt': caText,
'client.crt': clientCertificateText,
'client.key': clientKeyText,
},
});
create({
apiVersion: 'batch/v1',
kind: 'Job',
metadata: { name, namespace: NAMESPACE },
spec: {
backoffLimit: 0,
activeDeadlineSeconds: 60,
ttlSecondsAfterFinished: 600,
template: {
metadata: {
labels: {
app: 'ql3-worker-manager-live-client',
'qinglong.io/worker-credential-management-client': 'true',
},
},
spec: {
automountServiceAccountToken: false,
restartPolicy: 'Never',
hostAliases: [
{ ip: target.status.podIP, hostnames: [SERVERNAME] },
],
securityContext: {
runAsNonRoot: true,
runAsUser: 10001,
runAsGroup: 10001,
fsGroup: 10001,
seccompProfile: { type: 'RuntimeDefault' },
},
containers: [
{
name: 'client',
image: adminImage,
imagePullPolicy: 'Never',
command: ['/bin/sh', '-c'],
args: [clientShell],
env: [
{
name: 'QL3_RAW_REQUEST_DIAGNOSTIC_SCRIPT',
value: RAW_REQUEST_DIAGNOSTIC_SCRIPT,
},
],
terminationMessagePolicy: 'File',
securityContext: {
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
capabilities: { drop: ['ALL'] },
},
resources: {
requests: { cpu: '10m', memory: '32Mi' },
limits: { cpu: '250m', memory: '128Mi' },
},
volumeMounts: [
{ name: 'tmp', mountPath: '/tmp' },
{
name: 'client-input',
mountPath: '/var/run/qinglong3/client',
readOnly: true,
},
],
},
],
volumes: [
{
name: 'tmp',
emptyDir: { medium: 'Memory', sizeLimit: '4Mi' },
},
{
name: 'client-input',
secret: { secretName: inputSecret, defaultMode: 288 },
},
],
},
},
},
});
return Object.freeze({ name, target, command, bearer });
};
const waitClientJob = async (job, expected) => {
const completion = await waitFor(
`${job.name} completion`,
120_000,
() => {
const current = kubectlJson([
'-n',
NAMESPACE,
'get',
'job',
job.name,
]);
const complete = current.status.conditions?.some(
(condition) =>
condition.type === 'Complete' && condition.status === 'True',
);
const failed = current.status.conditions?.some(
(condition) =>
condition.type === 'Failed' && condition.status === 'True',
);
return complete || failed
? { ready: true, value: { complete, failed } }
: { ready: false, fact: JSON.stringify(current.status ?? {}) };
},
);
const pods = kubectlJson([
'-n',
NAMESPACE,
'get',
'pods',
'-l',
`job-name=${job.name}`,
]).items;
assert.equal(pods.length, 1);
const terminated =
pods[0].status.containerStatuses?.[0]?.state?.terminated;
const outputText = terminated?.message ?? '';
assert.equal(outputText.includes(job.bearer), false);
assert.equal(
outputText.includes(POSTGRES_ROLES.workerCredentialManager.password),
false,
);
const outputLines = outputText.split('\n').filter(Boolean);
const attemptFact = JSON.parse(outputLines[0]);
assert.ok(Number.isSafeInteger(attemptFact.transportAttempts));
assert.ok(attemptFact.transportAttempts >= 1);
assert.ok(attemptFact.transportAttempts <= 12);
const output = JSON.parse(outputLines.at(-1));
const targetFailureEvidence = () => {
const state = kubectl(
[
'-n',
NAMESPACE,
'get',
'pod',
job.target.metadata.name,
'-o',
'json',
],
{ capture: true, quiet: true, allowFailure: true },
);
const currentLogs = kubectl(
[
'-n',
NAMESPACE,
'logs',
job.target.metadata.name,
'--container=management',
'--tail=80',
],
{ capture: true, quiet: true, allowFailure: true },
);
const previousLogs = kubectl(
[
'-n',
NAMESPACE,
'logs',
job.target.metadata.name,
'--container=management',
'--previous',
'--tail=80',
],
{ capture: true, quiet: true, allowFailure: true },
);
return Object.freeze({
targetPod: job.target.metadata.name,
expectedUid: job.target.metadata.uid,
state: state.status === 0 ? JSON.parse(state.stdout) : state.stderr,
currentLogs: currentLogs.stdout || currentLogs.stderr,
previousLogs: previousLogs.stdout || previousLogs.stderr,
});
};
if (expected.statusCode === 200) {
if (!completion.complete || terminated?.exitCode !== 0) {
throw new Error(
`${job.name} unexpectedly failed: ${JSON.stringify({
output,
target: targetFailureEvidence(),
})}`,
);
}
assert.equal(completion.complete, true);
assert.equal(terminated?.exitCode, 0);
assert.equal(output.event, 'command_completed');
assert.equal(output.result.operation, job.command.operation);
if (expected.resultStatus !== undefined) {
assert.equal(
Array.isArray(expected.resultStatus)
? expected.resultStatus.includes(output.result.status)
: output.result.status === expected.resultStatus,
true,
);
}
} else {
if (!completion.failed || terminated?.exitCode !== 1) {
throw new Error(
`${job.name} unexpectedly succeeded: ${JSON.stringify(output)}`,
);
}
assert.equal(completion.failed, true);
assert.equal(terminated?.exitCode, 1);
assert.equal(output.event, 'command_failed');
assert.equal(output.statusCode, expected.statusCode);
assert.equal(output.responseCode, expected.responseCode);
}
return Object.freeze({
name: job.name,
targetPod: job.target.metadata.name,
targetPodUid: job.target.metadata.uid,
targetNode: job.target.spec.nodeName,
statusCode: expected.statusCode,
transportAttempts: attemptFact.transportAttempts,
output,
});
};
const executeClient = async (definition, expected) => {
const job = createClientJob(definition);
return waitClientJob(job, expected);
};
const executeWave = async (definitions, expected) => {
const jobs = definitions.map(createClientJob);
return Promise.all(jobs.map((job) => waitClientJob(job, expected)));
};
const executeWithoutClientCertificate = async ({
target,
command,
bearer,
}) => {
const name = 'ql3-wcm-no-client-certificate';
const inputSecret = `${name}-input`;
const rawConfig = {
schemaVersion: 1,
endpoint: clientConfig.endpoint,
servername: clientConfig.servername,
caFile: '/var/run/qinglong3/client/ca.crt',
requestTimeoutMs: clientConfig.requestTimeoutMs,
};
create({
apiVersion: 'v1',
kind: 'Secret',
metadata: { name: inputSecret, namespace: NAMESPACE },
type: 'Opaque',
stringData: {
'client.json': `${JSON.stringify(rawConfig)}\n`,
'command.json': `${JSON.stringify(command)}\n`,
'assertion.jwt': bearer,
'ca.crt': caText,
},
});
create({
apiVersion: 'batch/v1',
kind: 'Job',
metadata: { name, namespace: NAMESPACE },
spec: {
backoffLimit: 0,
activeDeadlineSeconds: 60,
ttlSecondsAfterFinished: 600,
template: {
metadata: {
labels: {
app: 'ql3-worker-manager-live-no-client-certificate',
'qinglong.io/worker-credential-management-client': 'true',
},
},
spec: {
automountServiceAccountToken: false,
restartPolicy: 'Never',
hostAliases: [
{ ip: target.status.podIP, hostnames: [SERVERNAME] },
],
securityContext: {
runAsNonRoot: true,
runAsUser: 10001,
runAsGroup: 10001,
fsGroup: 10001,
seccompProfile: { type: 'RuntimeDefault' },
},
containers: [
{
name: 'client',
image: adminImage,
imagePullPolicy: 'Never',
command: [
'node',
'-e',
RAW_REQUEST_DIAGNOSTIC_SCRIPT,
'--',
'/var/run/qinglong3/client/client.json',
'/var/run/qinglong3/client/command.json',
'/var/run/qinglong3/client/assertion.jwt',
],
terminationMessagePolicy: 'File',
securityContext: {
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
capabilities: { drop: ['ALL'] },
},
resources: {
requests: { cpu: '5m', memory: '16Mi' },
limits: { cpu: '100m', memory: '64Mi' },
},
volumeMounts: [
{
name: 'client-input',
mountPath: '/var/run/qinglong3/client',
readOnly: true,
},
],
},
],
volumes: [
{
name: 'client-input',
secret: { secretName: inputSecret, defaultMode: 288 },
},
],
},
},
},
});
await waitFor(`${name} completion`, 90_000, () => {
const current = kubectlJson(['-n', NAMESPACE, 'get', 'job', name]);
const complete = current.status.conditions?.some(
(condition) =>
condition.type === 'Complete' && condition.status === 'True',
);
return complete
? { ready: true, value: current }
: { ready: false, fact: JSON.stringify(current.status ?? {}) };
});
const pods = kubectlJson([
'-n',
NAMESPACE,
'get',
'pods',
'-l',
`job-name=${name}`,
]).items;
assert.equal(pods.length, 1);
const terminated =
pods[0].status.containerStatuses?.[0]?.state?.terminated;
assert.equal(terminated?.exitCode, 0);
const output = JSON.parse(terminated?.message ?? '{}');
assert.equal(output.event, 'raw_request_completed');
assert.equal(output.statusCode, 401);
assert.equal(output.protocol, 'TLSv1.3');
assert.equal(output.body.error.code, 'client_certificate_required');
assert.equal(String(terminated?.message ?? '').includes(bearer), false);
return Object.freeze({
targetPod: target.metadata.name,
statusCode: output.statusCode,
responseCode: output.body.error.code,
});
};
const executeConnectivity = async (target, index) => {
const name = `ql3-wcm-initial-connectivity-${index + 1}`;
create({
apiVersion: 'batch/v1',
kind: 'Job',
metadata: { name, namespace: NAMESPACE },
spec: {
backoffLimit: 0,
activeDeadlineSeconds: 60,
ttlSecondsAfterFinished: 600,
template: {
metadata: {
labels: {
app: 'ql3-worker-manager-live-connectivity',
'qinglong.io/worker-credential-management-client': 'true',
},
},
spec: {
automountServiceAccountToken: false,
restartPolicy: 'Never',
hostAliases: [
{ ip: target.status.podIP, hostnames: [SERVERNAME] },
],
securityContext: {
runAsNonRoot: true,
runAsUser: 10001,
runAsGroup: 10001,
fsGroup: 10001,
seccompProfile: { type: 'RuntimeDefault' },
},
containers: [
{
name: 'connectivity',
image: adminImage,
imagePullPolicy: 'Never',
command: [
'node',
'-e',
CONNECTIVITY_SCRIPT,
'--',
SERVERNAME,
'8444',
],
terminationMessagePolicy: 'File',
securityContext: {
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
capabilities: { drop: ['ALL'] },
},
resources: {
requests: { cpu: '5m', memory: '16Mi' },
limits: { cpu: '100m', memory: '64Mi' },
},
volumeMounts: [
{
name: 'ca',
mountPath: '/var/run/qinglong3/ca',
readOnly: true,
},
],
},
],
volumes: [
{
name: 'ca',
secret: {
secretName: TLS_SECRET,
defaultMode: 292,
items: [{ key: 'ca.crt', path: 'ca.crt' }],
},
},
],
},
},
},
});
await waitFor(`${name} completion`, 90_000, () => {
const current = kubectlJson(['-n', NAMESPACE, 'get', 'job', name]);
const complete = current.status.conditions?.some(
(condition) =>
condition.type === 'Complete' && condition.status === 'True',
);
const failed = current.status.conditions?.some(
(condition) =>
condition.type === 'Failed' && condition.status === 'True',
);
return complete || failed
? { ready: true, value: { complete, failed } }
: { ready: false, fact: JSON.stringify(current.status ?? {}) };
});
const pods = kubectlJson([
'-n',
NAMESPACE,
'get',
'pods',
'-l',
`job-name=${name}`,
]).items;
assert.equal(pods.length, 1);
const terminated =
pods[0].status.containerStatuses?.[0]?.state?.terminated;
const output = JSON.parse(terminated?.message ?? '{}');
assert.equal(terminated?.exitCode, 0);
assert.equal(output.event, 'connectivity_ready');
assert.equal(output.statusCode, 200);
assert.equal(output.protocol, 'TLSv1.3');
return Object.freeze({
name,
targetPod: target.metadata.name,
targetNode: target.spec.nodeName,
sourceNode: pods[0].spec.nodeName,
attempts: output.attempts,
});
};
const initialConnectivity = await Promise.all(
generation1Pods.map(executeConnectivity),
);
const firstCommands = Array.from({ length: 8 }, (_, index) =>
planCommand(index + 1),
);
const inspectExistingPlan = (suffix) =>
inspectCommand(firstCommands[0].request.actionRef, suffix);
const noClientCertificate = await executeWithoutClientCertificate({
target: generation1Pods[0],
command: inspectExistingPlan('no-client-certificate'),
bearer: assertion(oldKey, 'operator-a', 'no-client-certificate'),
});
const firstWave = await executeWave(
firstCommands.map((command, index) => ({
name: `ql3-wcm-plan-admit-${String(index + 1).padStart(2, '0')}`,
target: generation1Pods[index % 2],
command,
bearer: assertion(oldKey, 'operator-a', `admit-${index + 1}`),
})),
{ statusCode: 200, resultStatus: ['created', 'existing'] },
);
const committedOperationName = 'ql3-worker-credential-management-client';
const committedOperationCommand = inspectExistingPlan(
'committed-operation',
);
const committedOperationAssertion = assertion(
oldKey,
'operator-a',
'committed-operation',
);
create(loadOperationManifest('service-account.yaml'));
create(loadOperationManifest('network-policy.yaml'));
create({
apiVersion: 'v1',
kind: 'ConfigMap',
metadata: {
name: 'ql3-worker-credential-management-request',
namespace: NAMESPACE,
},
immutable: true,
data: {
'client.json': `${JSON.stringify(clientConfig)}\n`,
'command.json': `${JSON.stringify(committedOperationCommand)}\n`,
},
});
create({
apiVersion: 'v1',
kind: 'ConfigMap',
metadata: {
name: 'ql3-worker-credential-management-client-trust',
namespace: NAMESPACE,
},
immutable: true,
data: { 'ca.crt': caText },
});
create({
apiVersion: 'v1',
kind: 'Secret',
metadata: {
name: 'ql3-worker-credential-management-assertion',
namespace: NAMESPACE,
},
immutable: true,
type: 'Opaque',
stringData: { 'assertion.jwt': committedOperationAssertion },
});
create({
apiVersion: 'v1',
kind: 'Secret',
metadata: {
name: 'ql3-worker-credential-management-client-identity',
namespace: NAMESPACE,
},
immutable: true,
type: 'kubernetes.io/tls',
stringData: {
'tls.crt': activeClientCertificateText,
'tls.key': activeClientKeyText,
},
});
const committedOperationJob = loadOperationManifest('job.yaml');
for (const candidate of [
...committedOperationJob.spec.template.spec.initContainers,
...committedOperationJob.spec.template.spec.containers,
]) {
candidate.image = adminImage;
candidate.imagePullPolicy = 'Never';
}
create(committedOperationJob);
const committedOperationCompletion = await waitFor(
'committed management client operation completion',
150_000,
() => {
const current = kubectlJson([
'-n',
NAMESPACE,
'get',
'job',
committedOperationName,
]);
const complete = current.status.conditions?.some(
(condition) =>
condition.type === 'Complete' && condition.status === 'True',
);
const failed = current.status.conditions?.some(
(condition) =>
condition.type === 'Failed' && condition.status === 'True',
);
return complete || failed
? { ready: true, value: { complete, failed, current } }
: { ready: false, fact: JSON.stringify(current.status ?? {}) };
},
);
const committedOperationPods = kubectlJson([
'-n',
NAMESPACE,
'get',
'pods',
'-l',
`job-name=${committedOperationName}`,
]).items;
assert.equal(committedOperationPods.length, 1);
const committedOperationPod = committedOperationPods[0];
const committedOperationInit =
committedOperationPod.status.initContainerStatuses?.[0];
const committedOperationClient =
committedOperationPod.status.containerStatuses?.[0];
assert.equal(committedOperationCompletion.complete, true);
assert.equal(committedOperationCompletion.failed, false);
assert.equal(committedOperationCompletion.current.status.succeeded, 1);
assert.equal(committedOperationInit?.state?.terminated?.exitCode, 0);
assert.equal(committedOperationInit?.restartCount, 0);
assert.equal(committedOperationClient?.state?.terminated?.exitCode, 0);
assert.equal(committedOperationClient?.restartCount, 0);
assert.equal(
committedOperationPod.spec.automountServiceAccountToken,
false,
);
assert.equal(committedOperationPod.spec.enableServiceLinks, false);
assert.equal(
committedOperationPod.spec.serviceAccountName,
committedOperationName,
);
assert.equal(
committedOperationPod.spec.volumes.some((volume) =>
volume.projected?.sources?.some(
(source) => source.serviceAccountToken !== undefined,
),
),
false,
);
assert.equal(
committedOperationPod.metadata.labels?.[
'qinglong.io/worker-credential-management-client'
],
'true',
);
assert.equal(
String(
committedOperationClient?.state?.terminated?.message ?? '',
).includes(committedOperationAssertion),
false,
);
assert.equal(
String(
committedOperationClient?.state?.terminated?.message ?? '',
).includes(POSTGRES_ROLES.workerCredentialManager.password),
false,
);
for (const [kind, resourceName] of [
['configmap', 'ql3-worker-credential-management-request'],
['configmap', 'ql3-worker-credential-management-client-trust'],
['secret', 'ql3-worker-credential-management-assertion'],
['secret', 'ql3-worker-credential-management-client-identity'],
]) {
assert.equal(
kubectlJson(['-n', NAMESPACE, 'get', kind, resourceName]).immutable,
true,
);
}
const committedClientOperation = Object.freeze({
job: committedOperationName,
source: 'repository base manifests with test-only image substitution',
initExitCode: committedOperationInit.state.terminated.exitCode,
clientExitCode: committedOperationClient.state.terminated.exitCode,
clientRestarts: committedOperationClient.restartCount,
operation: committedOperationCommand.operation,
completed: committedOperationCompletion.complete,
assertionInTerminationMessage: false,
serviceAccountTokenMounted: false,
immutableInputs: true,
});
const limitedWave = await executeWave(
Array.from({ length: 8 }, (_, index) => ({
name: `ql3-wcm-plan-limit-${String(index + 1).padStart(2, '0')}`,
target: generation1Pods[index % 2],
command: planCommand(index + 9),
bearer: assertion(oldKey, 'operator-a', `limit-${index + 1}`),
})),
{ statusCode: 429, responseCode: 'quota_exceeded' },
);
const replay = await executeClient(
{
name: 'ql3-wcm-plan-replay-cross-pod',
target: generation1Pods[1],
command: firstCommands[0],
bearer: assertion(oldKey, 'operator-a', 'replay-cross-pod'),
},
{ statusCode: 200, resultStatus: 'existing' },
);
run(
'openssl',
['ca', '-batch', '-config', caConfig, '-revoke', clientOldCertificate],
{ capture: true, quiet: true },
);
run(
'openssl',
[
'ca',
'-gencrl',
'-config',
caConfig,
'-out',
clientCertificateRevocationList,
],
{ capture: true, quiet: true },
);
clientCrlText = fs.readFileSync(clientCertificateRevocationList, 'utf8');
apply({
apiVersion: 'v1',
kind: 'Secret',
metadata: { name: TLS_SECRET, namespace: NAMESPACE },
type: 'kubernetes.io/tls',
stringData: {
'tls.crt': tlsCertificateText,
'tls.key': tlsKeyText,
'ca.crt': caText,
'client.crl': clientCrlText,
},
});
const preRevocationUids = new Set(
generation1Pods.map((pod) => pod.metadata.uid),
);
patchGeneration('client-crl-2');
waitRollout();
const clientRevocationPods = await readyManagerPods(preRevocationUids);
const revokedClientCertificate = await executeClient(
{
name: 'ql3-wcm-revoked-client-certificate',
target: clientRevocationPods[0],
command: inspectExistingPlan('revoked-client-certificate'),
bearer: assertion(oldKey, 'operator-a', 'revoked-client-certificate'),
clientCertificateText: clientOldCertificateText,
clientKeyText: clientOldKeyText,
},
{ statusCode: 401, responseCode: 'client_certificate_required' },
);
activeClientCertificateText = clientNewCertificateText;
activeClientKeyText = clientNewKeyText;
const activeClientCertificate = await executeClient(
{
name: 'ql3-wcm-active-client-certificate',
target: clientRevocationPods[1],
command: inspectExistingPlan('active-client-certificate'),
bearer: assertion(oldKey, 'operator-a', 'active-client-certificate'),
},
{ statusCode: 200 },
);
const generation1Uids = new Set(
clientRevocationPods.map((pod) => pod.metadata.uid),
);
applyIdentity(keyset2);
patchGeneration('2');
waitRollout();
const generation2Pods = await readyManagerPods(generation1Uids);
const overlapOld = await executeClient(
{
name: 'ql3-wcm-overlap-old-key',
target: generation2Pods[0],
command: inspectExistingPlan('overlap-old'),
bearer: assertion(oldKey, 'operator-a', 'overlap-old'),
},
{ statusCode: 200 },
);
const overlapNew = await executeClient(
{
name: 'ql3-wcm-overlap-new-key',
target: generation2Pods[1],
command: inspectExistingPlan('overlap-new'),
bearer: assertion(newKey, 'operator-a', 'overlap-new'),
},
{ statusCode: 200 },
);
const generation2Uids = new Set(
generation2Pods.map((pod) => pod.metadata.uid),
);
applyIdentity(keyset3);
patchGeneration('3');
waitRollout();
const generation3Pods = await readyManagerPods(generation2Uids);
const revokedOld = await executeClient(
{
name: 'ql3-wcm-revoked-old-key',
target: generation3Pods[0],
command: inspectExistingPlan('revoked-old'),
bearer: assertion(oldKey, 'operator-a', 'revoked-old'),
},
{ statusCode: 401, responseCode: 'authentication_required' },
);
const activeNew = await executeClient(
{
name: 'ql3-wcm-active-new-key',
target: generation3Pods[1],
command: inspectExistingPlan('active-new'),
bearer: assertion(newKey, 'operator-a', 'active-new'),
},
{ statusCode: 200 },
);
applyIdentity(keyset2);
patchGeneration('rollback-2');
const rollbackPod = await waitFor(
'rollback surge Pod fail-closed',
120_000,
() => {
const deployment = kubectlJson([
'-n',
NAMESPACE,
'get',
'deployment',
DEPLOYMENT,
]);
const pods = kubectlJson([
'-n',
NAMESPACE,
'get',
'pods',
'-l',
`app.kubernetes.io/name=${DEPLOYMENT}`,
]).items.filter((pod) => pod.metadata.deletionTimestamp === undefined);
const ready = pods.filter((pod) =>
pod.status.conditions?.some(
(condition) =>
condition.type === 'Ready' && condition.status === 'True',
),
);
const candidate = pods.find(
(pod) =>
!generation3Pods.some(
(current) => current.metadata.uid === pod.metadata.uid,
) &&
pod.status.containerStatuses?.[0] &&
!pod.status.containerStatuses[0].ready &&
(pod.status.containerStatuses[0].restartCount > 0 ||
pod.status.containerStatuses[0].state?.waiting?.reason ===
'CrashLoopBackOff'),
);
return ready.length === 2 &&
deployment.status.readyReplicas === 2 &&
candidate
? { ready: true, value: candidate }
: {
ready: false,
fact:
`ready=${ready.length} replicas=${
deployment.status.readyReplicas ?? 0
} ` + `pods=${pods.length}`,
};
},
);
assert.equal(
rollbackPod.status.containerStatuses[0].lastState?.terminated?.exitCode ??
rollbackPod.status.containerStatuses[0].state?.terminated?.exitCode,
1,
);
applyIdentity(keyset3);
patchGeneration('3-recovered');
kubectl(
[
'-n',
NAMESPACE,
'delete',
'pod',
rollbackPod.metadata.name,
'--grace-period=0',
'--force',
'--wait=true',
],
{ capture: true, quiet: true },
);
waitRollout();
const recoveredIdentityPods = await readyManagerPods();
const recoveredIdentity = await Promise.all(
recoveredIdentityPods.map((pod, index) =>
executeClient(
{
name: `ql3-wcm-identity-recovered-${index + 1}`,
target: pod,
command: inspectExistingPlan(`identity-recovered-${index + 1}`),
bearer: assertion(
newKey,
'operator-a',
`identity-recovered-${index + 1}`,
),
},
{ statusCode: 200 },
),
),
);
const beforeFailureFacts = (
await migrationDatabase.pool.query(
`SELECT
(SELECT count(*)::integer
FROM "ql3"."worker_credential_management_plans") AS "plans",
(SELECT consumed_count::integer
FROM "ql3"."worker_credential_management_quota_buckets"
WHERE project_id = 'cluster-authority'
AND subject_type = 'user'
AND subject_id = 'operator-a'
AND operation = 'worker-credential.plan') AS "consumedCount",
(SELECT jsonb_array_length(receipt_ids)::integer
FROM "ql3"."worker_credential_management_quota_buckets"
WHERE project_id = 'cluster-authority'
AND subject_type = 'user'
AND subject_id = 'operator-a'
AND operation = 'worker-credential.plan') AS "receiptCount",
(SELECT generation::integer
FROM "ql3"."plugin_package_identity_keyset_ledger"
WHERE authority = 'worker-credential-management') AS "identityGeneration"`,
)
).rows[0];
assert.deepEqual(beforeFailureFacts, {
plans: 8,
consumedCount: 8,
receiptCount: 8,
identityGeneration: 3,
});
await migrationDatabase.close();
migrationDatabase = undefined;
const createHealthJob = ({ name, target }) => {
create({
apiVersion: 'batch/v1',
kind: 'Job',
metadata: { name, namespace: NAMESPACE },
spec: {
backoffLimit: 0,
activeDeadlineSeconds: 60,
ttlSecondsAfterFinished: 600,
template: {
metadata: {
labels: {
app: 'ql3-worker-manager-live-health',
'qinglong.io/worker-credential-management-client': 'true',
},
},
spec: {
automountServiceAccountToken: false,
restartPolicy: 'Never',
hostAliases: [
{ ip: target.status.podIP, hostnames: [SERVERNAME] },
],
securityContext: {
runAsNonRoot: true,
runAsUser: 10001,
runAsGroup: 10001,
fsGroup: 10001,
seccompProfile: { type: 'RuntimeDefault' },
},
containers: [
{
name: 'health',
image: adminImage,
imagePullPolicy: 'Never',
command: [
'node',
'-e',
HEALTH_SCRIPT,
'--',
SERVERNAME,
'8444',
],
terminationMessagePolicy: 'File',
securityContext: {
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
capabilities: { drop: ['ALL'] },
},
resources: {
requests: { cpu: '5m', memory: '16Mi' },
limits: { cpu: '100m', memory: '64Mi' },
},
volumeMounts: [
{
name: 'ca',
mountPath: '/var/run/qinglong3/ca',
readOnly: true,
},
],
},
],
volumes: [
{
name: 'ca',
secret: {
secretName: TLS_SECRET,
defaultMode: 292,
items: [{ key: 'ca.crt', path: 'ca.crt' }],
},
},
],
},
},
},
});
return Object.freeze({ name, target });
};
const waitHealthJob = async (job) => {
await waitFor(`${job.name} completion`, 90_000, () => {
const current = kubectlJson(['-n', NAMESPACE, 'get', 'job', job.name]);
const complete = current.status.conditions?.some(
(condition) =>
condition.type === 'Complete' && condition.status === 'True',
);
const failed = current.status.conditions?.some(
(condition) =>
condition.type === 'Failed' && condition.status === 'True',
);
return complete || failed
? { ready: true, value: current }
: { ready: false, fact: JSON.stringify(current.status ?? {}) };
});
const pods = kubectlJson([
'-n',
NAMESPACE,
'get',
'pods',
'-l',
`job-name=${job.name}`,
]).items;
assert.equal(pods.length, 1);
const terminated =
pods[0].status.containerStatuses?.[0]?.state?.terminated;
assert.equal(terminated?.exitCode, 0);
const output = JSON.parse(
(terminated.message ?? '').split('\n').filter(Boolean).at(-1),
);
assert.deepEqual(
output.responses.map((entry) => entry.protocol),
['TLSv1.3', 'TLSv1.3'],
);
return Object.freeze({
targetPod: job.target.metadata.name,
readyStatus: output.responses.find((entry) => entry.path === '/readyz')
.statusCode,
liveStatus: output.responses.find((entry) => entry.path === '/livez')
.statusCode,
});
};
const healthWave = async (label, pods) => {
const jobs = pods.map((pod, index) =>
createHealthJob({
name: `ql3-wcm-health-${label}-${index + 1}`,
target: pod,
}),
);
return Promise.all(jobs.map(waitHealthJob));
};
run(docker, ['stop', '--time', '1', postgres], {
capture: true,
quiet: true,
});
const unavailableRequests = await Promise.all(
recoveredIdentityPods.map((pod, index) =>
executeClient(
{
name: `ql3-wcm-postgres-down-${index + 1}`,
target: pod,
command: inspectExistingPlan(`postgres-down-${index + 1}`),
bearer: assertion(
newKey,
'operator-a',
`postgres-down-${index + 1}`,
),
},
{ statusCode: 503, responseCode: 'unavailable' },
),
),
);
const withdrawnHealth = await healthWave(
'withdrawn',
recoveredIdentityPods,
);
assert.deepEqual(
withdrawnHealth.map((entry) => entry.readyStatus),
[503, 503],
);
assert.deepEqual(
withdrawnHealth.map((entry) => entry.liveStatus),
[200, 200],
);
await waitFor(
'manager readiness withdrawn from both replicas',
30_000,
() => {
const current = kubectlJson([
'-n',
NAMESPACE,
'get',
'deployment',
DEPLOYMENT,
]);
return (current.status.readyReplicas ?? 0) === 0
? { ready: true, value: current }
: {
ready: false,
fact: `${
current.status.readyReplicas ?? 0
} replicas remain Ready`,
};
},
);
run(docker, ['start', postgres], { capture: true, quiet: true });
await waitFor('restarted PostgreSQL readiness', 60_000, () => {
const result = run(
docker,
[
'exec',
postgres,
'pg_isready',
'-h',
'127.0.0.1',
'-U',
'postgres',
'-d',
POSTGRES_DATABASE,
],
{ capture: true, quiet: true, allowFailure: true },
);
return result.status === 0
? { ready: true, value: true }
: { ready: false, fact: result.stderr || result.stdout };
});
const stillWithdrawnHealth = await healthWave(
'still-withdrawn',
recoveredIdentityPods,
);
assert.deepEqual(
stillWithdrawnHealth.map((entry) => entry.readyStatus),
[503, 503],
);
const withdrawnUids = new Set(
recoveredIdentityPods.map((pod) => pod.metadata.uid),
);
patchGeneration('3-fresh-postgres');
waitRollout();
const freshPods = await readyManagerPods(withdrawnUids);
const freshRequests = await Promise.all(
freshPods.map((pod, index) =>
executeClient(
{
name: `ql3-wcm-postgres-fresh-${index + 1}`,
target: pod,
command: inspectExistingPlan(`postgres-fresh-${index + 1}`),
bearer: assertion(
newKey,
'operator-a',
`postgres-fresh-${index + 1}`,
),
},
{ statusCode: 200 },
),
),
);
const finalFacts = JSON.parse(
run(
docker,
[
'exec',
'-e',
`PGPASSWORD=${POSTGRES_ROLES.workerCredentialManager.password}`,
postgres,
'psql',
'-U',
POSTGRES_ROLES.workerCredentialManager.user,
'-d',
POSTGRES_DATABASE,
'--tuples-only',
'--no-align',
'--command',
`SELECT json_build_object(
'plans', (SELECT count(*)::integer
FROM "ql3"."worker_credential_management_plans"),
'consumedCount', (SELECT consumed_count::integer
FROM "ql3"."worker_credential_management_quota_buckets"
WHERE project_id = 'cluster-authority'
AND subject_type = 'user'
AND subject_id = 'operator-a'
AND operation = 'worker-credential.plan'),
'receiptCount', (SELECT jsonb_array_length(receipt_ids)::integer
FROM "ql3"."worker_credential_management_quota_buckets"
WHERE project_id = 'cluster-authority'
AND subject_type = 'user'
AND subject_id = 'operator-a'
AND operation = 'worker-credential.plan'),
'identityGeneration', (SELECT generation::integer
FROM "ql3"."plugin_package_identity_keyset_ledger"
WHERE authority = 'worker-credential-management'),
'activeKeyIds', (SELECT active_key_ids
FROM "ql3"."plugin_package_identity_keyset_ledger"
WHERE authority = 'worker-credential-management'),
'revokedKeyIds', (SELECT revoked_key_ids
FROM "ql3"."plugin_package_identity_keyset_ledger"
WHERE authority = 'worker-credential-management'))`,
],
{ capture: true, quiet: true },
).stdout,
);
assert.deepEqual(finalFacts, {
plans: 8,
consumedCount: 8,
receiptCount: 8,
identityGeneration: 3,
activeKeyIds: [newKey.kid],
revokedKeyIds: [oldKey.kid],
});
const report = {
schemaVersion: 1,
fixture:
'qinglong/worker-credential-management-kubernetes-live-contract@v1',
kubernetes: {
distribution: 'k3s',
image: K3S_IMAGE,
imageDigest: K3S_DIGEST,
architecture: k3sImage.Architecture,
serverVersion: kubectlJson(['version']).serverVersion.gitVersion,
nodes: kubectlJson(['get', 'nodes']).items.map((node) => ({
name: node.metadata.name,
uid: node.metadata.uid,
})),
},
postgresql: {
image: POSTGRES_IMAGE,
imageId: postgresImage.Id,
architecture: postgresImage.Architecture,
database: POSTGRES_DATABASE,
managerRole: POSTGRES_ROLES.workerCredentialManager.user,
maxConnectionsPerPod: 2,
},
deployment: {
replicas: 2,
requiredPodAntiAffinity: true,
initialConnectivityChecks: initialConnectivity.length,
generation1Pods: generation1Pods.map((pod) => ({
name: pod.metadata.name,
uid: pod.metadata.uid,
node: pod.spec.nodeName,
})),
generation2Pods: generation2Pods.map((pod) => ({
name: pod.metadata.name,
uid: pod.metadata.uid,
node: pod.spec.nodeName,
})),
generation3Pods: generation3Pods.map((pod) => ({
name: pod.metadata.name,
uid: pod.metadata.uid,
node: pod.spec.nodeName,
})),
freshPostgresPods: freshPods.map((pod) => ({
name: pod.metadata.name,
uid: pod.metadata.uid,
node: pod.spec.nodeName,
})),
serviceAccountAutomount: false,
projectedServiceAccountToken: false,
exactPostgresEgressCidr: `${postgresAddress}/32`,
},
managementClientOperation: committedClientOperation,
clientCertificateAuthentication: {
healthWithoutClientCertificate: initialConnectivity.length === 2,
businessRouteWithoutClientCertificateStatus:
noClientCertificate.statusCode,
businessRouteWithoutClientCertificateCode:
noClientCertificate.responseCode,
revokedCertificateStatus: revokedClientCertificate.statusCode,
activeCertificateStatus: activeClientCertificate.statusCode,
rotationRequiresServerRollout: true,
},
quota: {
competingPods: 2,
admitted: firstWave.length,
limited: limitedWave.length,
requestsPerPodPerWave: 4,
crossPodReplayStatus: replay.output.result.status,
durablePlans: finalFacts.plans,
durableConsumedCount: finalFacts.consumedCount,
durableReceiptCount: finalFacts.receiptCount,
},
identity: {
generations: [1, 2, 3],
overlapOldAccepted: overlapOld.statusCode === 200,
overlapNewAccepted: overlapNew.statusCode === 200,
revokedOldRejected: revokedOld.statusCode === 401,
activeNewAccepted: activeNew.statusCode === 200,
rollbackPod: {
name: rollbackPod.metadata.name,
uid: rollbackPod.metadata.uid,
exitCode: 1,
},
readyReplicasDuringRollback: 2,
recoveredReplicas: recoveredIdentity.length,
durableGeneration: finalFacts.identityGeneration,
activeKeyIds: finalFacts.activeKeyIds,
revokedKeyIds: finalFacts.revokedKeyIds,
},
availability: {
unavailableResponses: unavailableRequests.map(
(entry) => entry.statusCode,
),
withdrawnReadyStatuses: withdrawnHealth.map(
(entry) => entry.readyStatus,
),
withdrawnLiveStatuses: withdrawnHealth.map((entry) => entry.liveStatus),
stillWithdrawnAfterDatabaseRestart: stillWithdrawnHealth.every(
(entry) => entry.readyStatus === 503,
),
freshActivationResponses: freshRequests.map(
(entry) => entry.statusCode,
),
},
gates: {
realThreeNodeKubernetes: true,
twoManagerPodsOnDistinctNodes: true,
tls13ProductionClientAcrossBothPods: true,
committedOneShotClientOperation: true,
clientCertificateAndOidcRequiredForBusinessRoutes: true,
revokedClientCertificateRejectedAfterRollout: true,
durableQuotaConvergedAcrossPods: true,
crossPodReplayConsumedNoAdditionalQuota: true,
identityProjectionOverlapAndRevocation: true,
rollbackSurgeFailedClosedWithoutAvailabilityLoss: true,
databaseFailureWithdrewReadinessButNotLiveness: true,
availabilityFenceDidNotRecoverInPlace: true,
freshActivationRecoveredBothReplicas: true,
passed: true,
},
limitations: [
'three privileged K3s Docker nodes are not production infrastructure or control-plane HA evidence',
'PostgreSQL transport is deliberately plaintext only inside the isolated disposable live fixture',
'database failure uses container stop rather than a raw-wire or CNI partition',
'identity assertions use deterministic local strong-User ceremony rather than an external IdP',
],
};
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
} finally {
if (migrationDatabase) {
await migrationDatabase.close().catch(() => undefined);
}
for (const name of [...createdContainers].reverse()) {
run(docker, ['rm', '-f', '-v', name], {
capture: true,
quiet: true,
allowFailure: true,
});
}
if (networkCreated) {
run(docker, ['network', 'rm', network], {
capture: true,
quiet: true,
allowFailure: true,
});
}
if (adminImageBuilt) {
run(docker, ['image', 'rm', '-f', adminImage], {
capture: true,
quiet: true,
allowFailure: true,
});
}
fs.rmSync(temporary, { recursive: true, force: true });
}
}
main().catch((error) => {
process.stderr.write(
`ql3 Worker credential manager Kubernetes live contract failed: ` +
`${error.stack || error}\n`,
);
process.exitCode = 1;
});
Reference in New Issue View Git Blame Copy Permalink