feat(ql3): establish 3.0 incubation baseline

scripts/lib/ql3-k3s-docker-live.cjs

@@ -0,0 +1,417 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const assert = require('node:assert/strict');
+const { randomBytes } = require('node:crypto');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+
+const ROOT = path.resolve(__dirname, '../..');
+const DEFAULT_K3S_IMAGE = 'rancher/k3s:v1.34.3-k3s1';
+const DEFAULT_K3S_DIGEST =
+  'sha256:71abd3a56f57884c62732e0e0d87606052cb5f8555b7db7e8e33c04570b8175c';
+
+function run(binary, args, options = {}) {
+  if (!options.quiet) {
+    process.stderr.write(`+ ${path.basename(binary)} ${args.join(' ')}\n`);
+  }
+  const result = spawnSync(binary, args, {
+    cwd: options.cwd ?? ROOT,
+    env: options.env ?? process.env,
+    input: options.input,
+    encoding: 'utf8',
+    maxBuffer: 64 * 1024 * 1024,
+    stdio: options.capture
+      ? ['pipe', 'pipe', 'pipe']
+      : [
+          options.input === undefined ? 'inherit' : 'pipe',
+          'inherit',
+          'inherit',
+        ],
+  });
+  if (result.error) throw result.error;
+  if (result.status !== 0 && !options.allowFailure) {
+    throw new Error(
+      `${path.basename(binary)} failed with ${String(result.status)}: ` +
+        `${result.stderr || result.stdout || ''}`,
+    );
+  }
+  return Object.freeze({
+    status: result.status,
+    stdout: options.capture ? result.stdout.trim() : '',
+    stderr: options.capture ? result.stderr.trim() : '',
+  });
+}
+
+function sleep(milliseconds) {
+  return new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+
+async function waitFor(description, timeoutMs, inspect, intervalMs = 500) {
+  const startedAt = Date.now();
+  let last = 'not observed';
+  while (Date.now() - startedAt < timeoutMs) {
+    try {
+      const value = await inspect();
+      if (value?.ready) {
+        return Object.freeze({
+          value: value.value,
+          elapsedMs: Date.now() - startedAt,
+        });
+      }
+      if (value?.fact) last = value.fact;
+    } catch (error) {
+      last = error instanceof Error ? error.message : String(error);
+    }
+    await sleep(intervalMs);
+  }
+  throw new Error(`${description} timed out: ${last}`);
+}
+
+function safePrefix(value) {
+  assert.match(value, /^ql3-[a-z0-9-]{1,32}$/);
+  return value;
+}
+
+class K3sDockerLiveFixture {
+  constructor(options = {}) {
+    this.docker = options.docker ?? process.env.QL3_DOCKER_BIN ?? 'docker';
+    this.kubectlBinary =
+      options.kubectl ?? process.env.QL3_KUBECTL_BIN ?? 'kubectl';
+    this.k3sImage = options.k3sImage ?? DEFAULT_K3S_IMAGE;
+    this.k3sDigest = options.k3sDigest ?? DEFAULT_K3S_DIGEST;
+    this.prefix = safePrefix(options.prefix ?? 'ql3-k3s-live');
+    this.suffix = `${process.pid.toString(36)}-${randomBytes(3).toString(
+      'hex',
+    )}`;
+    this.network = `${this.prefix}-network-${this.suffix}`;
+    this.server = `${this.prefix}-server-${this.suffix}`;
+    this.agents = [
+      `${this.prefix}-agent-a-${this.suffix}`,
+      `${this.prefix}-agent-b-${this.suffix}`,
+    ];
+    this.nodes = Object.freeze([this.server, ...this.agents]);
+    this.temporary = fs.mkdtempSync(
+      path.join(os.tmpdir(), `${this.prefix}-${this.suffix}-`),
+    );
+    this.kubeconfig = path.join(this.temporary, 'kubeconfig');
+    this.createdContainers = new Set();
+    this.networkCreated = false;
+    this.started = false;
+  }
+
+  dockerRun(args, options = {}) {
+    return run(this.docker, args, options);
+  }
+
+  kubectl(args, options = {}) {
+    assert.equal(this.started, true, 'K3s fixture is not started');
+    return run(
+      this.kubectlBinary,
+      ['--kubeconfig', this.kubeconfig, ...args],
+      options,
+    );
+  }
+
+  kubectlJson(args) {
+    return JSON.parse(
+      this.kubectl([...args, '-o', 'json'], {
+        capture: true,
+        quiet: true,
+      }).stdout,
+    );
+  }
+
+  apply(manifest) {
+    return this.kubectl(['apply', '-f', '-'], {
+      input: `${JSON.stringify(manifest)}\n`,
+      capture: true,
+      quiet: true,
+    });
+  }
+
+  create(manifest) {
+    return this.kubectl(['create', '-f', '-'], {
+      input: `${JSON.stringify(manifest)}\n`,
+      capture: true,
+      quiet: true,
+    });
+  }
+
+  async start() {
+    assert.equal(this.started, false, 'K3s fixture already started');
+    this.dockerRun(['version'], { capture: true, quiet: true });
+    const image = JSON.parse(
+      this.dockerRun(['image', 'inspect', this.k3sImage], {
+        capture: true,
+        quiet: true,
+      }).stdout,
+    )[0];
+    assert.ok(
+      image.RepoDigests?.includes(`rancher/k3s@${this.k3sDigest}`),
+      `K3s image does not retain reviewed digest ${this.k3sDigest}`,
+    );
+    for (const name of this.nodes) {
+      assert.equal(
+        this.dockerRun(['inspect', name], {
+          capture: true,
+          quiet: true,
+          allowFailure: true,
+        }).status,
+        1,
+        `refusing to reuse Docker container ${name}`,
+      );
+    }
+    assert.equal(
+      this.dockerRun(['network', 'inspect', this.network], {
+        capture: true,
+        quiet: true,
+        allowFailure: true,
+      }).status,
+      1,
+      `refusing to reuse Docker network ${this.network}`,
+    );
+    this.dockerRun(['network', 'create', this.network], {
+      capture: true,
+      quiet: true,
+    });
+    this.networkCreated = true;
+    const token = randomBytes(32).toString('base64url');
+    this.dockerRun(
+      [
+        'run',
+        '-d',
+        '--privileged',
+        '--network',
+        this.network,
+        '--name',
+        this.server,
+        '-p',
+        '127.0.0.1::6443',
+        this.k3sImage,
+        'server',
+        '--token',
+        token,
+        '--node-name',
+        this.server,
+        '--disable=traefik',
+        '--disable=servicelb',
+        '--write-kubeconfig-mode=600',
+        '--tls-san=127.0.0.1',
+      ],
+      { capture: true, quiet: true },
+    );
+    this.createdContainers.add(this.server);
+    try {
+      await waitFor('K3s control-plane readiness', 120_000, () => {
+        const result = this.dockerRun(
+          ['exec', this.server, 'kubectl', 'get', '--raw=/readyz'],
+          { capture: true, quiet: true, allowFailure: true },
+        );
+        return result.status === 0 && result.stdout === 'ok'
+          ? { ready: true, value: true }
+          : { ready: false, fact: result.stderr || result.stdout };
+      });
+    } catch (error) {
+      const state = this.dockerRun(
+        ['inspect', '--format', '{{json .State}}', this.server],
+        { capture: true, quiet: true, allowFailure: true },
+      );
+      const logs = this.dockerRun(['logs', '--tail', '120', this.server], {
+        capture: true,
+        quiet: true,
+        allowFailure: true,
+      });
+      throw new Error(
+        `${error instanceof Error ? error.message : String(error)}; ` +
+          `state=${state.stdout || state.stderr}; ` +
+          `logs=${logs.stderr || logs.stdout}`,
+      );
+    }
+    for (const agent of this.agents) {
+      this.dockerRun(
+        [
+          'run',
+          '-d',
+          '--privileged',
+          '--network',
+          this.network,
+          '--name',
+          agent,
+          this.k3sImage,
+          'agent',
+          '--server',
+          `https://${this.server}:6443`,
+          '--token',
+          token,
+          '--node-name',
+          agent,
+        ],
+        { capture: true, quiet: true },
+      );
+      this.createdContainers.add(agent);
+    }
+    const port = this.dockerRun(['port', this.server, '6443/tcp'], {
+      capture: true,
+      quiet: true,
+    }).stdout;
+    assert.match(port, /^127\.0\.0\.1:\d+$/);
+    const config = this.dockerRun(
+      ['exec', this.server, 'cat', '/etc/rancher/k3s/k3s.yaml'],
+      { capture: true, quiet: true },
+    ).stdout.replace('https://127.0.0.1:6443', `https://${port}`);
+    fs.writeFileSync(this.kubeconfig, `${config}\n`, {
+      mode: 0o600,
+      flag: 'wx',
+    });
+    this.started = true;
+    let ready;
+    try {
+      ready = await waitFor('three ready K3s nodes', 300_000, () => {
+        const nodes = this.kubectlJson(['get', 'nodes']).items ?? [];
+        const readyNodes = nodes.filter((node) =>
+          node.status.conditions?.some(
+            (condition) =>
+              condition.type === 'Ready' && condition.status === 'True',
+          ),
+        );
+        return readyNodes.length === 3
+          ? { ready: true, value: readyNodes }
+          : { ready: false, fact: `${readyNodes.length}/3 Ready nodes` };
+      });
+    } catch (error) {
+      const diagnostics = this.nodes.map((node) => {
+        const state = this.dockerRun(
+          ['inspect', '--format', '{{json .State}}', node],
+          { capture: true, quiet: true, allowFailure: true },
+        );
+        const logs = this.dockerRun(['logs', '--tail', '80', node], {
+          capture: true,
+          quiet: true,
+          allowFailure: true,
+        });
+        return {
+          node,
+          state: state.stdout || state.stderr,
+          logs: logs.stderr || logs.stdout,
+        };
+      });
+      throw new Error(
+        `${error instanceof Error ? error.message : String(error)}; ` +
+          `nodes=${JSON.stringify(diagnostics)}`,
+      );
+    }
+    return ready.value;
+  }
+
+  inspectImage(reference) {
+    const images = JSON.parse(
+      this.dockerRun(['image', 'inspect', reference], {
+        capture: true,
+        quiet: true,
+      }).stdout,
+    );
+    assert.equal(images.length, 1);
+    return images[0];
+  }
+
+  loadImage(reference, archiveName = 'image.tar') {
+    const archive = path.join(this.temporary, archiveName);
+    this.dockerRun(['image', 'save', '--output', archive, reference]);
+    try {
+      for (const node of this.nodes) {
+        const remote = `/tmp/${path.basename(archive)}`;
+        this.dockerRun(['cp', archive, `${node}:${remote}`], {
+          capture: true,
+          quiet: true,
+        });
+        try {
+          this.dockerRun(
+            [
+              'exec',
+              node,
+              'ctr',
+              '--address',
+              '/run/k3s/containerd/containerd.sock',
+              '--namespace',
+              'k8s.io',
+              'images',
+              'import',
+              remote,
+            ],
+            { capture: true, quiet: true },
+          );
+        } finally {
+          this.dockerRun(['exec', node, 'rm', '-f', remote], {
+            capture: true,
+            quiet: true,
+            allowFailure: true,
+          });
+        }
+      }
+    } finally {
+      fs.rmSync(archive, { force: true });
+    }
+  }
+
+  containerAddress(name) {
+    assert.ok(this.nodes.includes(name), `unknown fixture node ${name}`);
+    const address = this.dockerRun(
+      [
+        'inspect',
+        '--format',
+        '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}',
+        name,
+      ],
+      { capture: true, quiet: true },
+    ).stdout;
+    assert.match(address, /^\d{1,3}(?:\.\d{1,3}){3}$/);
+    return address;
+  }
+
+  stopNode(name) {
+    assert.ok(this.nodes.includes(name), `unknown fixture node ${name}`);
+    this.dockerRun(['stop', '--time', '1', name], {
+      capture: true,
+      quiet: true,
+    });
+  }
+
+  startNode(name) {
+    assert.ok(this.nodes.includes(name), `unknown fixture node ${name}`);
+    this.dockerRun(['start', name], { capture: true, quiet: true });
+  }
+
+  async cleanup() {
+    for (const name of [...this.createdContainers].reverse()) {
+      this.dockerRun(['rm', '-f', '-v', name], {
+        capture: true,
+        quiet: true,
+        allowFailure: true,
+      });
+    }
+    this.createdContainers.clear();
+    if (this.networkCreated) {
+      this.dockerRun(['network', 'rm', this.network], {
+        capture: true,
+        quiet: true,
+        allowFailure: true,
+      });
+      this.networkCreated = false;
+    }
+    fs.rmSync(this.temporary, { recursive: true, force: true });
+    this.started = false;
+  }
+}
+
+module.exports = {
+  DEFAULT_K3S_DIGEST,
+  DEFAULT_K3S_IMAGE,
+  K3sDockerLiveFixture,
+  run,
+  sleep,
+  waitFor,
+};

scripts/lib/ql3-live-pki.cjs

@@ -0,0 +1,267 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+
+function sha256File(crypto, filePath) {
+  return `sha256:${crypto
+    .createHash('sha256')
+    .update(fs.readFileSync(filePath))
+    .digest('hex')}`;
+}
+
+function createMutualTlsPki({ directory, servername, label, run, crypto }) {
+  const file = (name) => path.join(directory, name);
+  const paths = Object.freeze({
+    caKey: file('ca.key'),
+    caCertificate: file('ca.crt'),
+    caConfig: file('ca.cnf'),
+    caDatabase: file('ca.index'),
+    caSerial: file('ca.serial'),
+    caCrlNumber: file('ca.crlnumber'),
+    caNewCertificates: file('ca-new-certificates'),
+    serverKey: file('server.key'),
+    serverRequest: file('server.csr'),
+    serverCertificate: file('server.crt'),
+    serverExtensions: file('server.ext'),
+    oldClientKey: file('client-old.key'),
+    oldClientRequest: file('client-old.csr'),
+    oldClientCertificate: file('client-old.crt'),
+    newClientKey: file('client-new.key'),
+    newClientRequest: file('client-new.csr'),
+    newClientCertificate: file('client-new.crt'),
+    clientCertificateRevocationList: file('client.crl'),
+  });
+
+  run(
+    'openssl',
+    [
+      'req',
+      '-x509',
+      '-newkey',
+      'rsa:2048',
+      '-nodes',
+      '-days',
+      '1',
+      '-subj',
+      `/CN=${label} CA`,
+      '-keyout',
+      paths.caKey,
+      '-out',
+      paths.caCertificate,
+    ],
+    { capture: true, quiet: true },
+  );
+  fs.mkdirSync(paths.caNewCertificates, { mode: 0o700 });
+  fs.writeFileSync(paths.caDatabase, '', { mode: 0o600, flag: 'wx' });
+  fs.writeFileSync(paths.caSerial, '1000\n', {
+    mode: 0o600,
+    flag: 'wx',
+  });
+  fs.writeFileSync(paths.caCrlNumber, '1000\n', {
+    mode: 0o600,
+    flag: 'wx',
+  });
+  fs.writeFileSync(
+    paths.caConfig,
+    [
+      '[ca]',
+      'default_ca=client_ca',
+      '[client_ca]',
+      `database=${paths.caDatabase}`,
+      `new_certs_dir=${paths.caNewCertificates}`,
+      `certificate=${paths.caCertificate}`,
+      `private_key=${paths.caKey}`,
+      `serial=${paths.caSerial}`,
+      `crlnumber=${paths.caCrlNumber}`,
+      'default_md=sha256',
+      'default_days=1',
+      'default_crl_days=1',
+      'policy=client_policy',
+      'unique_subject=no',
+      'copy_extensions=none',
+      '[client_policy]',
+      'commonName=supplied',
+      '[client_certificate]',
+      'basicConstraints=critical,CA:FALSE',
+      'keyUsage=critical,digitalSignature,keyEncipherment',
+      'extendedKeyUsage=clientAuth',
+      '',
+    ].join('\n'),
+    { mode: 0o600, flag: 'wx' },
+  );
+  fs.writeFileSync(
+    paths.serverExtensions,
+    [
+      'basicConstraints=CA:FALSE',
+      'keyUsage=digitalSignature,keyEncipherment',
+      'extendedKeyUsage=serverAuth',
+      `subjectAltName=DNS:${servername},DNS:${servername}.cluster.local`,
+      '',
+    ].join('\n'),
+    { mode: 0o600, flag: 'wx' },
+  );
+  for (const [commonName, key, request, certificate] of [
+    [
+      `${label} old client`,
+      paths.oldClientKey,
+      paths.oldClientRequest,
+      paths.oldClientCertificate,
+    ],
+    [
+      `${label} replacement client`,
+      paths.newClientKey,
+      paths.newClientRequest,
+      paths.newClientCertificate,
+    ],
+  ]) {
+    run(
+      'openssl',
+      [
+        'req',
+        '-newkey',
+        'rsa:2048',
+        '-nodes',
+        '-subj',
+        `/CN=${commonName}`,
+        '-keyout',
+        key,
+        '-out',
+        request,
+      ],
+      { capture: true, quiet: true },
+    );
+    run(
+      'openssl',
+      [
+        'ca',
+        '-batch',
+        '-notext',
+        '-config',
+        paths.caConfig,
+        '-extensions',
+        'client_certificate',
+        '-in',
+        request,
+        '-out',
+        certificate,
+      ],
+      { capture: true, quiet: true },
+    );
+  }
+  run(
+    'openssl',
+    [
+      'req',
+      '-newkey',
+      'rsa:2048',
+      '-nodes',
+      '-subj',
+      `/CN=${servername}`,
+      '-keyout',
+      paths.serverKey,
+      '-out',
+      paths.serverRequest,
+    ],
+    { capture: true, quiet: true },
+  );
+  run(
+    'openssl',
+    [
+      'x509',
+      '-req',
+      '-days',
+      '1',
+      '-in',
+      paths.serverRequest,
+      '-CA',
+      paths.caCertificate,
+      '-CAkey',
+      paths.caKey,
+      '-CAcreateserial',
+      '-extfile',
+      paths.serverExtensions,
+      '-out',
+      paths.serverCertificate,
+    ],
+    { capture: true, quiet: true },
+  );
+  const generateCrl = () =>
+    run(
+      'openssl',
+      [
+        'ca',
+        '-gencrl',
+        '-config',
+        paths.caConfig,
+        '-out',
+        paths.clientCertificateRevocationList,
+      ],
+      { capture: true, quiet: true },
+    );
+  generateCrl();
+
+  const read = () =>
+    Object.freeze({
+      ca: fs.readFileSync(paths.caCertificate, 'utf8'),
+      serverCertificate: fs.readFileSync(paths.serverCertificate, 'utf8'),
+      serverKey: fs.readFileSync(paths.serverKey, 'utf8'),
+      oldClientCertificate: fs.readFileSync(paths.oldClientCertificate, 'utf8'),
+      oldClientKey: fs.readFileSync(paths.oldClientKey, 'utf8'),
+      newClientCertificate: fs.readFileSync(paths.newClientCertificate, 'utf8'),
+      newClientKey: fs.readFileSync(paths.newClientKey, 'utf8'),
+      clientCrl: fs.readFileSync(paths.clientCertificateRevocationList, 'utf8'),
+    });
+
+  return Object.freeze({
+    paths,
+    read,
+    bundleSha256: () =>
+      `sha256:${crypto
+        .createHash('sha256')
+        .update(fs.readFileSync(paths.caCertificate))
+        .update(fs.readFileSync(paths.clientCertificateRevocationList))
+        .digest('hex')}`,
+    oldSerialSha256: () => {
+      const serial = run(
+        'openssl',
+        ['x509', '-in', paths.oldClientCertificate, '-noout', '-serial'],
+        { capture: true, quiet: true },
+      ).stdout;
+      return `sha256:${crypto
+        .createHash('sha256')
+        .update(serial)
+        .digest('hex')}`;
+    },
+    newSerialSha256: () => {
+      const serial = run(
+        'openssl',
+        ['x509', '-in', paths.newClientCertificate, '-noout', '-serial'],
+        { capture: true, quiet: true },
+      ).stdout;
+      return `sha256:${crypto
+        .createHash('sha256')
+        .update(serial)
+        .digest('hex')}`;
+    },
+    revokeOldClient() {
+      run(
+        'openssl',
+        [
+          'ca',
+          '-batch',
+          '-config',
+          paths.caConfig,
+          '-revoke',
+          paths.oldClientCertificate,
+        ],
+        { capture: true, quiet: true },
+      );
+      generateCrl();
+    },
+  });
+}
+
+module.exports = { createMutualTlsPki, sha256File };

scripts/lib/ql3-management-kubernetes-live.cjs

@@ -0,0 +1,589 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const assert = require('node:assert/strict');
+const { waitFor } = require('./ql3-k3s-docker-live.cjs');
+
+function podReady(pod) {
+  return Boolean(
+    pod?.metadata?.deletionTimestamp === undefined &&
+      pod?.status?.conditions?.some(
+        (condition) =>
+          condition.type === 'Ready' && condition.status === 'True',
+      ),
+  );
+}
+
+async function readyManagementPods(options) {
+  const observed = await waitFor(options.description, 300_000, () => {
+    const pods = options.fixture
+      .kubectlJson([
+        '-n',
+        options.namespace,
+        'get',
+        'pods',
+        '-l',
+        'app.kubernetes.io/name=' + options.deployment,
+      ])
+      .items.filter(
+        (pod) =>
+          podReady(pod) &&
+          !(options.excludedUids ?? new Set()).has(pod.metadata.uid) &&
+          (options.expectedGeneration === undefined ||
+            pod.metadata.annotations?.['qinglong.io/identity-generation'] ===
+              String(options.expectedGeneration)),
+      );
+    const nodes = new Set(pods.map((pod) => pod.spec.nodeName));
+    return pods.length === 2 && nodes.size === 2
+      ? {
+          ready: true,
+          value: pods.sort((left, right) =>
+            left.metadata.name.localeCompare(right.metadata.name),
+          ),
+        }
+      : {
+          ready: false,
+          fact: pods.length + ' Ready Pods on ' + nodes.size + ' nodes',
+        };
+  });
+  return observed.value;
+}
+
+function patchManagementGeneration(options) {
+  options.fixture.kubectl(
+    [
+      '-n',
+      options.namespace,
+      'patch',
+      'deployment',
+      options.deployment,
+      '--type=merge',
+      '-p',
+      JSON.stringify({
+        spec: {
+          template: {
+            metadata: {
+              annotations: {
+                'qinglong.io/identity-generation': String(options.generation),
+                ...(options.annotations ?? {}),
+              },
+            },
+          },
+        },
+      }),
+    ],
+    { capture: true, quiet: true },
+  );
+}
+
+function waitManagementRollout(options) {
+  options.fixture.kubectl([
+    '-n',
+    options.namespace,
+    'rollout',
+    'status',
+    'deployment/' + options.deployment,
+    '--timeout=5m',
+  ]);
+}
+
+async function waitForTwoPreserved(options) {
+  let minimumReady = Number.POSITIVE_INFINITY;
+  const observed = await waitFor(options.description, 300_000, () => {
+    const pods = options.fixture
+      .kubectlJson([
+        '-n',
+        options.namespace,
+        'get',
+        'pods',
+        '-l',
+        'app.kubernetes.io/name=' + options.deployment,
+      ])
+      .items.filter(
+        (pod) => pod.metadata.deletionTimestamp === undefined,
+      );
+    const ready = pods.filter(podReady);
+    minimumReady = Math.min(minimumReady, ready.length);
+    const replacements = ready.filter(
+      (pod) =>
+        !options.excludedUids.has(pod.metadata.uid) &&
+        pod.metadata.annotations?.['qinglong.io/identity-generation'] ===
+          String(options.expectedGeneration),
+    );
+    const nodes = new Set(replacements.map((pod) => pod.spec.nodeName));
+    return replacements.length === 2 && nodes.size === 2
+      ? { ready: true, value: replacements }
+      : {
+          ready: false,
+          fact:
+            ready.length +
+            ' ready, ' +
+            replacements.length +
+            ' replacements',
+        };
+  });
+  assert.ok(
+    minimumReady >= 2,
+    'rollout availability dropped to ' + minimumReady,
+  );
+  return Object.freeze({ pods: observed.value, minimumReady });
+}
+
+function createManagementClientExecutor(options) {
+  options.fixture.apply({
+    apiVersion: 'v1',
+    kind: 'ServiceAccount',
+    metadata: {
+      name: options.serviceAccount,
+      namespace: options.namespace,
+    },
+    automountServiceAccountToken: false,
+  });
+  return async function executeClient(definition, expected) {
+    const input = definition.name + '-input';
+    const clientConfig = {
+      schemaVersion: 1,
+      endpoint:
+        'https://' +
+        options.servername +
+        ':' +
+        String(options.port) +
+        options.managementPath,
+      servername: options.servername,
+      caFile: '/tmp/ca.crt',
+      clientCertificateFile: '/tmp/client.crt',
+      clientPrivateKeyFile: '/tmp/client.key',
+      requestTimeoutMs: 5_000,
+    };
+    options.fixture.create({
+      apiVersion: 'v1',
+      kind: 'Secret',
+      metadata: { name: input, namespace: options.namespace },
+      immutable: true,
+      type: 'Opaque',
+      stringData: {
+        'client.json': JSON.stringify(clientConfig) + '\n',
+        'command.json': JSON.stringify(definition.command) + '\n',
+        'assertion.jwt': definition.bearer,
+        'ca.crt': options.ca,
+        'client.crt': definition.clientCertificate,
+        'client.key': definition.clientKey,
+      },
+    });
+    options.fixture.create({
+      apiVersion: 'batch/v1',
+      kind: 'Job',
+      metadata: {
+        name: definition.name,
+        namespace: options.namespace,
+        labels: {
+          'app.kubernetes.io/name': options.appName,
+          'app.kubernetes.io/component': options.component,
+          'qinglong.io/execution-model': 'caller-driven',
+        },
+      },
+      spec: {
+        backoffLimit: 0,
+        activeDeadlineSeconds: 240,
+        ttlSecondsAfterFinished: 600,
+        template: {
+          metadata: {
+            labels: {
+              'app.kubernetes.io/name': options.appName,
+              'app.kubernetes.io/component': options.component,
+              [options.networkPolicyLabel]: 'true',
+              'qinglong.io/execution-model': 'caller-driven',
+            },
+          },
+          spec: {
+            serviceAccountName: options.serviceAccount,
+            automountServiceAccountToken: false,
+            enableServiceLinks: false,
+            restartPolicy: 'Never',
+            hostAliases: [
+              {
+                ip: definition.target.status.podIP,
+                hostnames: [options.servername],
+              },
+            ],
+            securityContext: {
+              runAsNonRoot: true,
+              runAsUser: 10001,
+              runAsGroup: 10001,
+              fsGroup: 10001,
+              seccompProfile: { type: 'RuntimeDefault' },
+            },
+            containers: [
+              {
+                name: 'client',
+                image: options.adminImage,
+                imagePullPolicy: 'Never',
+                command: ['/bin/sh', '-c'],
+                args: [
+                  [
+                    'set -eu',
+                    'umask 077',
+                    'cp /var/run/qinglong3/client/client.json /tmp/client.json',
+                    'cp /var/run/qinglong3/client/command.json /tmp/command.json',
+                    'cp /var/run/qinglong3/client/assertion.jwt /tmp/assertion.jwt',
+                    'cp /var/run/qinglong3/client/ca.crt /tmp/ca.crt',
+                    'cp /var/run/qinglong3/client/client.crt /tmp/client.crt',
+                    'cp /var/run/qinglong3/client/client.key /tmp/client.key',
+                    'chmod 600 /tmp/client.json /tmp/command.json ' +
+                      '/tmp/assertion.jwt /tmp/ca.crt /tmp/client.crt ' +
+                      '/tmp/client.key',
+                    'set +e',
+                    'attempt=0',
+                    'while true; do',
+                    '  attempt=$((attempt + 1))',
+                    '  output="$(node ' +
+                      options.clientCliPath +
+                      ' --config=/tmp/client.json ' +
+                      '--command=/tmp/command.json ' +
+                      '--assertion=/tmp/assertion.jwt 2>&1)"',
+                    '  status=$?',
+                    '  if [ "$status" -eq 0 ] || { ! printf \'%s\' "$output" | ' +
+                      'grep -q QL3_PLUGIN_PACKAGE_MANAGEMENT_CLIENT_REQUEST_FAILED && ' +
+                      '! printf \'%s\' "$output" | grep -q \'"statusCode":503\'; } || ' +
+                      '[ "$attempt" -ge 60 ]; then',
+                    '    break',
+                    '  fi',
+                    '  sleep 1',
+                    'done',
+                    'printf \'%s\\n\' "$output" > /dev/termination-log',
+                    'printf \'%s\\n\' "$output"',
+                    'exit "$status"',
+                  ].join('\n'),
+                ],
+                terminationMessagePolicy: 'File',
+                securityContext: {
+                  allowPrivilegeEscalation: false,
+                  readOnlyRootFilesystem: true,
+                  capabilities: { drop: ['ALL'] },
+                },
+                resources: {
+                  requests: { cpu: '10m', memory: '32Mi' },
+                  limits: { cpu: '250m', memory: '128Mi' },
+                },
+                volumeMounts: [
+                  { name: 'tmp', mountPath: '/tmp' },
+                  {
+                    name: 'input',
+                    mountPath: '/var/run/qinglong3/client',
+                    readOnly: true,
+                  },
+                ],
+              },
+            ],
+            volumes: [
+              {
+                name: 'tmp',
+                emptyDir: { medium: 'Memory', sizeLimit: '4Mi' },
+              },
+              {
+                name: 'input',
+                secret: { secretName: input, defaultMode: 288 },
+              },
+            ],
+          },
+        },
+      },
+    });
+
+    const completion = await waitFor(
+      definition.name + ' completion',
+      300_000,
+      () => {
+        const job = options.fixture.kubectlJson([
+          '-n',
+          options.namespace,
+          'get',
+          'job',
+          definition.name,
+        ]);
+        const complete = job.status.conditions?.some(
+          (condition) =>
+            condition.type === 'Complete' && condition.status === 'True',
+        );
+        const failed = job.status.conditions?.some(
+          (condition) =>
+            condition.type === 'Failed' && condition.status === 'True',
+        );
+        return complete || failed
+          ? { ready: true, value: { complete, failed } }
+          : { ready: false, fact: JSON.stringify(job.status ?? {}) };
+      },
+    );
+    const clientPod = (
+      await waitFor(definition.name + ' terminal pod', 60_000, () => {
+        const pods = options.fixture.kubectlJson([
+          '-n',
+          options.namespace,
+          'get',
+          'pods',
+          '-l',
+          'batch.kubernetes.io/job-name=' + definition.name,
+        ]).items;
+        return pods.length === 1 && pods[0].status.containerStatuses?.[0]
+          ? { ready: true, value: pods[0] }
+          : {
+              ready: false,
+              fact: 'observed ' + pods.length + ' client pods',
+            };
+      })
+    ).value;
+    assert.equal(clientPod.spec.automountServiceAccountToken, false);
+    assert.equal(
+      clientPod.spec.volumes.some((volume) =>
+        volume.projected?.sources?.some(
+          (source) => source.serviceAccountToken !== undefined,
+        ),
+      ),
+      false,
+    );
+    assert.equal(
+      options.fixture.kubectlJson([
+        '-n',
+        options.namespace,
+        'get',
+        'secret',
+        input,
+      ]).immutable,
+      true,
+    );
+    const terminated =
+      clientPod.status.containerStatuses?.[0]?.state?.terminated;
+    assert.ok(
+      terminated,
+      options.description + ' client termination state is missing',
+    );
+    const message = terminated.message ?? '';
+    assert.equal(message.includes(definition.bearer), false);
+    assert.equal(message.includes(definition.clientKey), false);
+    const output = JSON.parse(message.split('\n').filter(Boolean).at(-1));
+    if (expected.statusCode === 200) {
+      assert.equal(
+        completion.value.complete,
+        true,
+        options.description +
+          ' client unexpectedly failed: ' +
+          JSON.stringify(output),
+      );
+      assert.equal(output.event, 'command_completed');
+      assert.equal(output.result.operation, definition.command.operation);
+      if (expected.resultStatus) {
+        assert.ok(expected.resultStatus.includes(output.result.status));
+      }
+    } else {
+      assert.equal(
+        completion.value.failed,
+        true,
+        options.description +
+          ' client unexpectedly succeeded: ' +
+          JSON.stringify(output),
+      );
+      assert.equal(output.event, 'command_failed');
+      assert.equal(output.statusCode, expected.statusCode);
+      assert.equal(output.responseCode, expected.responseCode);
+    }
+    const result = Object.freeze({
+      targetPod: definition.target.metadata.name,
+      targetPodUid: definition.target.metadata.uid,
+      targetNode: definition.target.spec.nodeName,
+      statusCode: expected.statusCode,
+      output,
+    });
+    options.fixture.kubectl(
+      [
+        '-n',
+        options.namespace,
+        'delete',
+        'job/' + definition.name,
+        'secret/' + input,
+        '--wait=false',
+      ],
+      { capture: true, quiet: true },
+    );
+    return result;
+  };
+}
+
+function podTcpProbe(options) {
+  const script = [
+    "const net=require('node:net');let finished=false;",
+    'const socket=net.createConnection({host:process.argv[1],port:Number(process.argv[2])});',
+    'const finish=(status)=>{if(finished)return;finished=true;socket.destroy();process.exitCode=status};',
+    "socket.setTimeout(3000);socket.once('connect',()=>finish(0));",
+    "socket.once('timeout',()=>finish(1));socket.once('error',()=>finish(1));",
+  ].join('\n');
+  return options.fixture.kubectl(
+    [
+      '-n',
+      options.namespace,
+      'exec',
+      options.podName,
+      '--',
+      'node',
+      '-e',
+      script,
+      options.host,
+      String(options.port),
+    ],
+    { capture: true, quiet: true, allowFailure: true },
+  );
+}
+
+async function clientTcpProbe(options) {
+  const labels = {
+    'app.kubernetes.io/name': options.appName,
+    ...(options.labelled
+      ? { [options.networkPolicyLabel]: 'true' }
+      : {}),
+  };
+  const script = [
+    "const fs=require('node:fs');const net=require('node:net');let finished=false;let attempt=0;let socket;",
+    'const maximum=Number(process.argv[3]);',
+    "const finish=(message,status)=>{if(finished)return;finished=true;fs.writeFileSync('/dev/termination-log',message);socket?.destroy();process.exitCode=status};",
+    "const retry=(reason)=>{socket.destroy();attempt+=1;if(attempt>=maximum){finish('denied:'+reason,1);return;}setTimeout(connect,500);};",
+    "const connect=()=>{let settled=false;const failed=(reason)=>{if(settled)return;settled=true;retry(reason)};socket=net.createConnection({host:process.argv[1],port:Number(process.argv[2])});socket.setTimeout(3000);socket.once('connect',()=>{if(settled)return;settled=true;finish('connected',0)});socket.once('timeout',()=>failed('timeout'));socket.once('error',(error)=>failed(error.code||'error'));};",
+    'connect();',
+  ].join('\n');
+  options.fixture.create({
+    apiVersion: 'batch/v1',
+    kind: 'Job',
+    metadata: { name: options.name, namespace: options.namespace },
+    spec: {
+      backoffLimit: 0,
+      activeDeadlineSeconds: 120,
+      ttlSecondsAfterFinished: 600,
+      template: {
+        metadata: { labels },
+        spec: {
+          automountServiceAccountToken: false,
+          restartPolicy: 'Never',
+          securityContext: {
+            runAsNonRoot: true,
+            runAsUser: 10001,
+            runAsGroup: 10001,
+            seccompProfile: { type: 'RuntimeDefault' },
+          },
+          containers: [
+            {
+              name: 'probe',
+              image: options.adminImage,
+              imagePullPolicy: 'Never',
+              command: [
+                'node',
+                '-e',
+                script,
+                options.targetHost,
+                String(options.port),
+                String(options.expectedConnected ? 12 : 1),
+              ],
+              terminationMessagePolicy: 'File',
+              securityContext: {
+                allowPrivilegeEscalation: false,
+                readOnlyRootFilesystem: true,
+                capabilities: { drop: ['ALL'] },
+              },
+              resources: {
+                requests: { cpu: '5m', memory: '16Mi' },
+                limits: { cpu: '100m', memory: '64Mi' },
+              },
+            },
+          ],
+        },
+      },
+    },
+  });
+  const observed = await waitFor(
+    options.name + ' completion',
+    180_000,
+    () => {
+      const job = options.fixture.kubectlJson([
+        '-n',
+        options.namespace,
+        'get',
+        'job',
+        options.name,
+      ]);
+      const complete = job.status.conditions?.some(
+        (condition) =>
+          condition.type === 'Complete' && condition.status === 'True',
+      );
+      const failed = job.status.conditions?.some(
+        (condition) =>
+          condition.type === 'Failed' && condition.status === 'True',
+      );
+      return complete || failed
+        ? { ready: true, value: { complete, failed } }
+        : { ready: false, fact: JSON.stringify(job.status ?? {}) };
+    },
+  );
+  const probePod = (
+    await waitFor(options.name + ' terminal pod', 30_000, () => {
+      const pods = options.fixture.kubectlJson([
+        '-n',
+        options.namespace,
+        'get',
+        'pods',
+        '-l',
+        'batch.kubernetes.io/job-name=' + options.name,
+      ]).items;
+      const terminated =
+        pods[0]?.status.containerStatuses?.[0]?.state?.terminated;
+      return pods.length === 1 && terminated
+        ? { ready: true, value: pods[0] }
+        : { ready: false, fact: 'observed ' + pods.length + ' probe pods' };
+    })
+  ).value;
+  const terminated = probePod.status.containerStatuses[0].state.terminated;
+  const observation =
+    options.name + ': ' + (terminated.message ?? 'no-message');
+  assert.equal(
+    observed.value.complete,
+    options.expectedConnected,
+    observation,
+  );
+  assert.equal(
+    observed.value.failed,
+    !options.expectedConnected,
+    observation,
+  );
+  assert.equal(
+    terminated.exitCode === 0,
+    options.expectedConnected,
+    observation,
+  );
+  if (options.expectedConnected) {
+    assert.equal(terminated.message, 'connected');
+  } else {
+    assert.match(terminated.message ?? '', /^denied:/);
+  }
+  options.fixture.kubectl(
+    [
+      '-n',
+      options.namespace,
+      'delete',
+      'job',
+      options.name,
+      '--wait=false',
+    ],
+    { capture: true, quiet: true },
+  );
+  return options.expectedConnected
+    ? observed.value.complete
+    : observed.value.failed;
+}
+
+module.exports = {
+  clientTcpProbe,
+  createManagementClientExecutor,
+  patchManagementGeneration,
+  podReady,
+  podTcpProbe,
+  readyManagementPods,
+  waitForTwoPreserved,
+  waitManagementRollout,
+};

scripts/lib/ql3-management-live-identity.cjs

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const assert = require('node:assert/strict');
+const crypto = require('node:crypto');
+
+function createManagementIdentityCeremony(options) {
+  assert.match(options.issuer, /^https:\/\/[A-Za-z0-9.-]+\/$/);
+  for (const value of [
+    options.audience,
+    options.purpose,
+    options.tokenType,
+    options.subject,
+    options.jtiPrefix,
+  ]) {
+    assert.match(value, /^[A-Za-z0-9][A-Za-z0-9._:+-]{0,127}$/);
+  }
+
+  function reviewedKey(kid) {
+    assert.match(kid, /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/);
+    const { privateKey, publicKey } = crypto.generateKeyPairSync('ed25519');
+    return Object.freeze({
+      kid,
+      privateKey,
+      publicJwk: Object.freeze({
+        ...publicKey.export({ format: 'jwk' }),
+        alg: 'EdDSA',
+        kid,
+        use: 'sig',
+      }),
+    });
+  }
+
+  function keyset(generation, keys, revokedKids = []) {
+    assert.ok(Number.isSafeInteger(generation) && generation >= 1);
+    return Object.freeze({
+      schemaVersion: 1,
+      generation,
+      issuer: options.issuer,
+      audience: options.audience,
+      keys: keys.map((key) => key.publicJwk),
+      revokedKids: [...revokedKids],
+      assuranceMappings: [
+        {
+          acr: 'urn:ql3:mfa',
+          assurance: 'multi_factor',
+          requiredAmr: ['pwd', 'otp'],
+        },
+      ],
+      constraints: {
+        maxAssertionBytes: 8 * 1024,
+        maxLifetimeMs: 5 * 60 * 1000,
+        maxAuthenticationAgeMs: 5 * 60 * 1000,
+        clockSkewMs: 5 * 1000,
+      },
+    });
+  }
+
+  function assertion(key, suffix = crypto.randomUUID()) {
+    const now = Math.floor(Date.now() / 1_000);
+    const header = Buffer.from(
+      JSON.stringify({
+        alg: 'EdDSA',
+        kid: key.kid,
+        typ: options.tokenType,
+      }),
+    ).toString('base64url');
+    const payload = Buffer.from(
+      JSON.stringify({
+        acr: 'urn:ql3:mfa',
+        amr: ['pwd', 'otp'],
+        aud: options.audience,
+        auth_time: now - 1,
+        exp: now + 290,
+        iat: now,
+        iss: options.issuer,
+        jti: options.jtiPrefix + '-' + suffix,
+        ql3_purpose: options.purpose,
+        sub: options.subject,
+      }),
+    ).toString('base64url');
+    const signed = header + '.' + payload;
+    return (
+      signed +
+      '.' +
+      crypto
+        .sign(null, Buffer.from(signed, 'ascii'), key.privateKey)
+        .toString('base64url')
+    );
+  }
+
+  return Object.freeze({ assertion, keyset, reviewedKey });
+}
+
+module.exports = { createManagementIdentityCeremony };