Files
qinglong/scripts/ql3-plugin-package-kubernetes-live-contract.cjs
T

665 lines
20 KiB
JavaScript

#!/usr/bin/env node
'use strict';
const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { spawnSync } = require('node:child_process');
const ROOT = path.resolve(__dirname, '..');
const ACTOR_FILE = path.join(
__dirname,
'ql3-plugin-package-kubernetes-live-actor.cjs',
);
const RESULT_SCHEMA = 'qinglong/plugin-package-kubernetes-live-actor-result@v1';
const REPORT_FIXTURE = 'qinglong/plugin-package-kubernetes-live-contract@v1';
const NAMESPACE = 'ql3-plugin-package-live';
const SERVICE_ACCOUNT = 'ql3-plugin-package-recovery-live';
const KIND_NODE_IMAGE =
'kindest/node:v1.32.8@sha256:abd489f042d2b644e2d033f5c2d900bc707798d075e8186cb65e3f1367a9d5a1';
const DEFAULT_IMAGE = 'qinglong3-cluster-admin:ql3-kubernetes-live';
const SAFE_CLUSTER =
/^ql3-plugin-activation(?:-[a-z0-9](?:[-a-z0-9]{0,30}[a-z0-9])?)?$/;
function fail(message) {
throw new Error(message);
}
function executable(environmentName, fallback) {
return process.env[environmentName] || fallback;
}
const KIND = executable('QL3_KIND_BIN', 'kind');
const KUBECTL = executable('QL3_KUBECTL_BIN', 'kubectl');
const DOCKER = executable('QL3_DOCKER_BIN', 'docker');
function commandLabel(binary, args) {
return [path.basename(binary), ...args].join(' ');
}
function run(binary, args, options = {}) {
if (!options.quiet) {
process.stderr.write(`+ ${options.label || commandLabel(binary, args)}\n`);
}
const capture = options.capture === true;
const result = spawnSync(binary, args, {
cwd: options.cwd || ROOT,
env: options.env || process.env,
input: options.input,
encoding: 'utf8',
maxBuffer: 64 * 1024 * 1024,
stdio: capture
? ['pipe', 'pipe', 'pipe']
: [
options.input === undefined ? 'inherit' : 'pipe',
'inherit',
'inherit',
],
});
if (result.error) throw result.error;
if (result.status !== 0 && !options.allowFailure) {
const detail = capture ? `\n${result.stderr || result.stdout || ''}` : '';
fail(
`${path.basename(binary)} exited with status ${String(
result.status,
)}${detail}`,
);
}
return Object.freeze({
status: result.status,
stdout: capture ? result.stdout.trim() : '',
stderr: capture ? result.stderr.trim() : '',
});
}
function kind(args, options) {
return run(KIND, args, options);
}
let kubeconfig = '';
function kubectl(args, options = {}) {
return run(KUBECTL, ['--kubeconfig', kubeconfig, ...args], options);
}
function kubectlJson(args) {
const output = kubectl([...args, '-o', 'json'], {
capture: true,
quiet: true,
}).stdout;
return JSON.parse(output);
}
function apply(body, label) {
kubectl(['apply', '-f', '-'], {
input: `${JSON.stringify(body)}\n`,
label,
});
}
function sleep(milliseconds) {
return new Promise((resolve) => setTimeout(resolve, milliseconds));
}
async function waitFor(description, timeoutMs, inspect) {
const startedAt = Date.now();
let lastFact = 'not observed';
while (Date.now() - startedAt < timeoutMs) {
let result;
try {
result = inspect();
if (result?.ready) {
return {
elapsedMs: Date.now() - startedAt,
value: result.value,
};
}
if (result?.fact) lastFact = result.fact;
} catch (error) {
lastFact = error instanceof Error ? error.message : String(error);
}
if (result?.fatal) {
fail(`${description} failed: ${result.fatal}`);
}
await sleep(1_000);
}
fail(`${description} timed out after ${timeoutMs}ms: ${lastFact}`);
}
function exactClusterName() {
const configured =
process.env.QL3_KIND_CLUSTER ??
`ql3-plugin-activation-${process.pid.toString(36)}`;
if (!SAFE_CLUSTER.test(configured)) {
fail(
'QL3_KIND_CLUSTER must be an exact ql3-plugin-activation[-suffix] name',
);
}
return configured;
}
function kindCreateEnvironment(clusterName) {
const required = [
'127.0.0.1',
'localhost',
`${clusterName}-control-plane`,
'.svc',
'.cluster.local',
'10.96.0.0/12',
'10.244.0.0/16',
'172.16.0.0/12',
];
const configured = [process.env.NO_PROXY, process.env.no_proxy, ...required]
.flatMap((value) => (value ?? '').split(','))
.map((value) => value.trim())
.filter(Boolean);
const noProxy = [...new Set(configured)].join(',');
return {
...process.env,
NO_PROXY: noProxy,
no_proxy: noProxy,
};
}
function roleDocuments() {
return [
{
apiVersion: 'v1',
kind: 'Namespace',
metadata: { name: NAMESPACE },
},
{
apiVersion: 'v1',
kind: 'ServiceAccount',
metadata: { name: SERVICE_ACCOUNT, namespace: NAMESPACE },
automountServiceAccountToken: false,
},
{
apiVersion: 'rbac.authorization.k8s.io/v1',
kind: 'Role',
metadata: { name: SERVICE_ACCOUNT, namespace: NAMESPACE },
rules: [
{
apiGroups: [''],
resources: ['configmaps'],
verbs: ['get', 'create', 'update'],
},
],
},
{
apiVersion: 'rbac.authorization.k8s.io/v1',
kind: 'RoleBinding',
metadata: { name: SERVICE_ACCOUNT, namespace: NAMESPACE },
roleRef: {
apiGroup: 'rbac.authorization.k8s.io',
kind: 'Role',
name: SERVICE_ACCOUNT,
},
subjects: [
{
kind: 'ServiceAccount',
name: SERVICE_ACCOUNT,
namespace: NAMESPACE,
},
],
},
];
}
function actorPod(actor, image) {
return {
apiVersion: 'v1',
kind: 'Pod',
metadata: {
name: `ql3-plugin-package-live-${actor}`,
namespace: NAMESPACE,
labels: {
'app.kubernetes.io/name': 'ql3-plugin-package-live',
'app.kubernetes.io/component': 'plugin-package-recovery',
'app.kubernetes.io/part-of': 'qinglong3',
'qinglong.io/live-actor': actor,
},
},
spec: {
serviceAccountName: SERVICE_ACCOUNT,
automountServiceAccountToken: true,
restartPolicy: 'Never',
securityContext: {
runAsNonRoot: true,
runAsUser: 10001,
runAsGroup: 10001,
fsGroup: 10001,
seccompProfile: { type: 'RuntimeDefault' },
},
containers: [
{
name: 'recovery',
image,
imagePullPolicy: 'Never',
command: ['node', '/opt/ql3-live/actor.cjs'],
env: [
{
name: 'NODE_PATH',
value: '/opt/qinglong/node_modules',
},
{ name: 'QL3_LIVE_NAMESPACE', value: NAMESPACE },
{ name: 'QL3_LIVE_ACTOR', value: actor },
],
securityContext: {
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
capabilities: { drop: ['ALL'] },
},
resources: {
requests: { cpu: '25m', memory: '64Mi' },
limits: { cpu: '500m', memory: '256Mi' },
},
volumeMounts: [
{
name: 'actor',
mountPath: '/opt/ql3-live',
readOnly: true,
},
],
},
],
volumes: [
{
name: 'actor',
configMap: {
name: 'ql3-plugin-package-live-actor',
defaultMode: 292,
items: [{ key: 'actor.cjs', path: 'actor.cjs' }],
},
},
],
},
};
}
function canI(verb, resource, options = {}) {
const namespace = options.namespace ?? NAMESPACE;
const result = kubectl(
[
'auth',
'can-i',
verb,
resource,
'--namespace',
namespace,
'--as',
`system:serviceaccount:${NAMESPACE}:${SERVICE_ACCOUNT}`,
],
{ capture: true, quiet: true, allowFailure: true },
);
assert.ok(result.status === 0 || result.status === 1);
assert.ok(result.stdout === 'yes' || result.stdout === 'no');
return result.stdout === 'yes';
}
function actorResult(actor) {
const podName = `ql3-plugin-package-live-${actor}`;
const output = kubectl(['-n', NAMESPACE, 'logs', podName, '-c', 'recovery'], {
capture: true,
quiet: true,
}).stdout;
const line = output
.split('\n')
.map((candidate) => candidate.trim())
.filter(Boolean)
.at(-1);
assert.ok(line, `${podName} emitted no result`);
const result = JSON.parse(line);
assert.equal(result.schema, RESULT_SCHEMA);
assert.equal(result.actor, actor);
assert.equal(result.error, undefined);
return result;
}
async function main() {
if (process.env.QL3_PLUGIN_PACKAGE_KUBERNETES_LIVE !== '1') {
fail('refusing to run without QL3_PLUGIN_PACKAGE_KUBERNETES_LIVE=1');
}
run(DOCKER, ['version'], { capture: true, quiet: true });
run(KIND, ['version'], { capture: true, quiet: true });
run(KUBECTL, ['version', '--client=true'], {
capture: true,
quiet: true,
});
const clusterName = exactClusterName();
const image =
process.env.QL3_PLUGIN_PACKAGE_KUBERNETES_IMAGE ?? DEFAULT_IMAGE;
if (
!/^[a-z0-9][a-z0-9._/-]{0,255}:[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(
image,
)
) {
fail('QL3_PLUGIN_PACKAGE_KUBERNETES_IMAGE must be an exact local tag');
}
const existingClusters = kind(['get', 'clusters'], {
capture: true,
quiet: true,
allowFailure: true,
})
.stdout.split('\n')
.filter(Boolean);
if (existingClusters.includes(clusterName)) {
fail(`refusing to reuse or delete existing Kind cluster ${clusterName}`);
}
const temporaryDirectory = fs.mkdtempSync(
path.join(os.tmpdir(), 'ql3-plugin-kubernetes-live-'),
);
kubeconfig = path.join(temporaryDirectory, 'kubeconfig');
let clusterCreated = false;
const startedAt = Date.now();
try {
if (process.env.QL3_PLUGIN_PACKAGE_KUBERNETES_BUILD_IMAGE !== '0') {
run(
DOCKER,
[
'build',
'--file',
'deploy/containers/ql3-cluster-admin/Dockerfile',
'--tag',
image,
'--build-arg',
`SOURCE_REVISION=${process.env.GITHUB_SHA ?? 'local-live-contract'}`,
'.',
],
{ label: `docker build ${image}` },
);
} else {
run(DOCKER, ['image', 'inspect', image], {
capture: true,
quiet: true,
});
}
kind(
[
'create',
'cluster',
'--name',
clusterName,
'--image',
KIND_NODE_IMAGE,
'--kubeconfig',
kubeconfig,
'--wait',
'120s',
],
{
label: `kind create isolated cluster ${clusterName}`,
env: kindCreateEnvironment(clusterName),
},
);
clusterCreated = true;
kind(['load', 'docker-image', '--name', clusterName, image]);
for (const document of roleDocuments()) {
apply(
document,
`kubectl apply ${document.kind}/${document.metadata.name}`,
);
}
apply(
{
apiVersion: 'v1',
kind: 'ConfigMap',
metadata: {
name: 'ql3-plugin-package-live-actor',
namespace: NAMESPACE,
},
data: { 'actor.cjs': fs.readFileSync(ACTOR_FILE, 'utf8') },
},
'kubectl apply ConfigMap/ql3-plugin-package-live-actor',
);
apply(actorPod('a', image), 'kubectl apply Pod/ql3-plugin-package-live-a');
apply(actorPod('b', image), 'kubectl apply Pod/ql3-plugin-package-live-b');
const completion = await waitFor(
'both restricted recovery Pods',
120_000,
() => {
const pods = kubectlJson([
'-n',
NAMESPACE,
'get',
'pods',
'-l',
'app.kubernetes.io/name=ql3-plugin-package-live',
]).items;
const facts = Object.fromEntries(
pods.map((pod) => [pod.metadata.name, pod.status.phase]),
);
const failed = pods.find((pod) => pod.status.phase === 'Failed');
if (failed) {
const logs = kubectl(
['-n', NAMESPACE, 'logs', failed.metadata.name],
{ capture: true, quiet: true, allowFailure: true },
);
return {
ready: false,
fatal: `${failed.metadata.name}: ${logs.stderr || logs.stdout}`,
};
}
return {
ready:
pods.length === 2 &&
pods.every((pod) => pod.status.phase === 'Succeeded'),
value: facts,
fact: JSON.stringify(facts),
};
},
);
const actors = [actorResult('a'), actorResult('b')];
assert.equal(
actors.filter((actor) => actor.cas.status === 'fulfilled').length,
1,
);
assert.equal(
actors.filter((actor) => actor.cas.status === 'conflict').length,
1,
);
assert.ok(
actors.every(
(actor) =>
actor.serviceAccountTokenMounted === true &&
actor.cas.replaceCalls === 1 &&
/^[1-9][0-9]*$/.test(actor.cas.attemptedResourceVersion),
),
);
assert.equal(
new Set(actors.map((actor) => actor.cas.attemptedResourceVersion)).size,
1,
'both recovery Pods must attempt the same resourceVersion',
);
assert.deepEqual(actors[0].rbac, {
listConfigMaps: 403,
deleteConfigMap: 403,
readSecret: 403,
crossNamespaceRead: 403,
});
assert.deepEqual(actors[1].rbac, actors[0].rbac);
assert.deepEqual(actors[0].responseLoss, {
injectedAfterApiConfirmedCreate: true,
firstCallFailedClosed: true,
durableInspectPublished: true,
replayReturnedExactReceipt: true,
createCalls: 1,
nowCalls: 1,
});
assert.equal(actors[1].responseLoss, null);
const winner = actors.find((actor) => actor.cas.status === 'fulfilled');
const loser = actors.find((actor) => actor.cas.status === 'conflict');
assert.equal(winner.final.lockDigest, loser.final.lockDigest);
assert.equal(
winner.final.lockDigest,
winner.actor === 'a' ? '1'.repeat(64) : '6'.repeat(64),
);
assert.equal(winner.final.generation, 2);
assert.equal(winner.final.resourceVersion, loser.final.resourceVersion);
assert.equal(winner.final.pointerSchema, loser.final.pointerSchema);
assert.equal(winner.final.projectionDigest, loser.final.projectionDigest);
assert.equal(
winner.final.transitionReceiptDigest,
loser.final.transitionReceiptDigest,
);
assert.match(winner.final.projectionDigest, /^[0-9a-f]{64}$/);
assert.match(winner.final.transitionReceiptDigest, /^[0-9a-f]{64}$/);
assert.equal(winner.final.pointerSchema.endsWith('@v3'), true);
assert.equal(winner.final.projectionItemCount, 1);
assert.equal(winner.final.projectedWorkloadVolume, true);
const activePointers = kubectlJson([
'-n',
NAMESPACE,
'get',
'configmaps',
'-l',
'qinglong.io/plugin-package-active=v3',
]).items;
assert.equal(activePointers.length, 1);
const barriers = kubectlJson([
'-n',
NAMESPACE,
'get',
'configmaps',
'-l',
'qinglong.io/live-gate-role=cas-barrier',
]).items;
assert.equal(barriers.length, 2);
const secrets = kubectlJson(['-n', NAMESPACE, 'get', 'secrets']).items;
assert.equal(secrets.length, 0);
const rbac = Object.freeze({
getConfigMaps: canI('get', 'configmaps'),
createConfigMaps: canI('create', 'configmaps'),
updateConfigMaps: canI('update', 'configmaps'),
listConfigMaps: canI('list', 'configmaps'),
deleteConfigMaps: canI('delete', 'configmaps'),
getSecrets: canI('get', 'secrets'),
createSecrets: canI('create', 'secrets'),
crossNamespaceGetConfigMaps: canI('get', 'configmaps', {
namespace: 'default',
}),
});
assert.deepEqual(rbac, {
getConfigMaps: true,
createConfigMaps: true,
updateConfigMaps: true,
listConfigMaps: false,
deleteConfigMaps: false,
getSecrets: false,
createSecrets: false,
crossNamespaceGetConfigMaps: false,
});
process.stdout.write(
`${JSON.stringify(
{
schemaVersion: 1,
fixture: REPORT_FIXTURE,
cluster: {
kindName: clusterName,
nodeImage: KIND_NODE_IMAGE,
namespace: NAMESPACE,
image,
},
recoveryPods: {
count: 2,
completionMs: completion.elapsedMs,
serviceAccountTokenMounted: true,
distinctProcesses: true,
},
responseLoss: {
scope:
'client boundary after Kubernetes API-confirmed create; not raw-wire packet loss',
createCalls: actors[0].responseLoss.createCalls,
durableInspectPublished: true,
exactReplayWithoutRepublish: true,
},
resourceVersionCas: {
attemptedResourceVersion: winner.cas.attemptedResourceVersion,
fulfilled: 1,
conflicts: 1,
finalResourceVersion: winner.final.resourceVersion,
winner: winner.actor,
activePointers: activePointers.length,
},
secretProjection: {
schema: winner.final.pointerSchema,
projectionDigest: winner.final.projectionDigest,
transitionReceiptDigest: winner.final.transitionReceiptDigest,
itemCount: winner.final.projectionItemCount,
defaultMode: 0o440,
workloadVolumeRendered: winner.final.projectedWorkloadVolume,
secretApiReadRequired: false,
},
rbac,
sideEffects: {
activePointers: activePointers.length,
casBarriers: barriers.length,
secrets: secrets.length,
},
gates: {
realKubernetesApi: true,
realProjectedServiceAccountTokens: true,
twoRestrictedRecoveryPods: true,
apiConfirmedCreateResponseLossFailedClosed: true,
durableInspectRecoveredPublication: true,
exactReplayDidNotRepublish: true,
sameResourceVersionAttemptedByBothPods: true,
concurrentReplacementSingleWinner: true,
loserObservedConflict: true,
finalPointerExactlyOne: true,
transitionReceiptBoundToV3Pointer: true,
exactSecretProjectionItemPublished: true,
projectedWorkloadVolumeUses0440: true,
configMapGetCreateUpdateAllowed: true,
configMapListDeleteDenied: true,
secretReadCreateDenied: true,
crossNamespaceReadDenied: true,
passed: true,
},
elapsedMs: Date.now() - startedAt,
limitations: [
'response loss is injected after an API-confirmed create at the Kubernetes client boundary, not by dropping raw network packets',
'single-control-plane Kind proves API-server resourceVersion and RBAC semantics, not Kubernetes control-plane HA',
'the gate uses in-memory content-blind binding/transition sources; durable PostgreSQL transition authority is proven independently',
'the gate renders the exact Secret volume source but deliberately does not create or read Secret material',
'the gate isolates ConfigMap publication authority and does not exercise OCI registry recovery',
],
},
null,
2,
)}\n`,
);
} finally {
if (
clusterCreated &&
process.env.QL3_KEEP_PLUGIN_PACKAGE_KUBERNETES_LIVE !== '1'
) {
kind(['delete', 'cluster', '--name', clusterName], {
label: `kind delete exact cluster ${clusterName}`,
});
}
fs.rmSync(temporaryDirectory, { recursive: true, force: true });
}
}
main().catch((error) => {
process.stderr.write(
`ql3 Plugin Package Kubernetes live contract failed: ${
error instanceof Error ? error.stack || error.message : String(error)
}\n`,
);
process.exitCode = 1;
});