qinglong/scripts/lib/ql3-k3s-docker-live.cjs

#!/usr/bin/env node

'use strict';

const assert = require('node:assert/strict');
const { randomBytes } = require('node:crypto');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { spawnSync } = require('node:child_process');

const ROOT = path.resolve(__dirname, '../..');
const DEFAULT_K3S_IMAGE = 'rancher/k3s:v1.34.3-k3s1';
const DEFAULT_K3S_DIGEST =
  'sha256:71abd3a56f57884c62732e0e0d87606052cb5f8555b7db7e8e33c04570b8175c';

function run(binary, args, options = {}) {
  if (!options.quiet) {
    process.stderr.write(`+ ${path.basename(binary)} ${args.join(' ')}\n`);
  }
  const result = spawnSync(binary, args, {
    cwd: options.cwd ?? ROOT,
    env: options.env ?? process.env,
    input: options.input,
    encoding: 'utf8',
    maxBuffer: 64 * 1024 * 1024,
    stdio: options.capture
      ? ['pipe', 'pipe', 'pipe']
      : [
          options.input === undefined ? 'inherit' : 'pipe',
          'inherit',
          'inherit',
        ],
  });
  if (result.error) throw result.error;
  if (result.status !== 0 && !options.allowFailure) {
    throw new Error(
      `${path.basename(binary)} failed with ${String(result.status)}: ` +
        `${result.stderr || result.stdout || ''}`,
    );
  }
  return Object.freeze({
    status: result.status,
    stdout: options.capture ? result.stdout.trim() : '',
    stderr: options.capture ? result.stderr.trim() : '',
  });
}

function sleep(milliseconds) {
  return new Promise((resolve) => setTimeout(resolve, milliseconds));
}

async function waitFor(description, timeoutMs, inspect, intervalMs = 500) {
  const startedAt = Date.now();
  let last = 'not observed';
  while (Date.now() - startedAt < timeoutMs) {
    try {
      const value = await inspect();
      if (value?.ready) {
        return Object.freeze({
          value: value.value,
          elapsedMs: Date.now() - startedAt,
        });
      }
      if (value?.fact) last = value.fact;
    } catch (error) {
      last = error instanceof Error ? error.message : String(error);
    }
    await sleep(intervalMs);
  }
  throw new Error(`${description} timed out: ${last}`);
}

function safePrefix(value) {
  assert.match(value, /^ql3-[a-z0-9-]{1,32}$/);
  return value;
}

class K3sDockerLiveFixture {
  constructor(options = {}) {
    this.docker = options.docker ?? process.env.QL3_DOCKER_BIN ?? 'docker';
    this.kubectlBinary =
      options.kubectl ?? process.env.QL3_KUBECTL_BIN ?? 'kubectl';
    this.k3sImage = options.k3sImage ?? DEFAULT_K3S_IMAGE;
    this.k3sDigest = options.k3sDigest ?? DEFAULT_K3S_DIGEST;
    this.prefix = safePrefix(options.prefix ?? 'ql3-k3s-live');
    this.suffix = `${process.pid.toString(36)}-${randomBytes(3).toString(
      'hex',
    )}`;
    this.network = `${this.prefix}-network-${this.suffix}`;
    this.server = `${this.prefix}-server-${this.suffix}`;
    this.agents = [
      `${this.prefix}-agent-a-${this.suffix}`,
      `${this.prefix}-agent-b-${this.suffix}`,
    ];
    this.nodes = Object.freeze([this.server, ...this.agents]);
    this.temporary = fs.mkdtempSync(
      path.join(os.tmpdir(), `${this.prefix}-${this.suffix}-`),
    );
    this.kubeconfig = path.join(this.temporary, 'kubeconfig');
    this.createdContainers = new Set();
    this.networkCreated = false;
    this.started = false;
  }

  dockerRun(args, options = {}) {
    return run(this.docker, args, options);
  }

  kubectl(args, options = {}) {
    assert.equal(this.started, true, 'K3s fixture is not started');
    return run(
      this.kubectlBinary,
      ['--kubeconfig', this.kubeconfig, ...args],
      options,
    );
  }

  kubectlJson(args) {
    return JSON.parse(
      this.kubectl([...args, '-o', 'json'], {
        capture: true,
        quiet: true,
      }).stdout,
    );
  }

  apply(manifest) {
    return this.kubectl(['apply', '-f', '-'], {
      input: `${JSON.stringify(manifest)}\n`,
      capture: true,
      quiet: true,
    });
  }

  create(manifest) {
    return this.kubectl(['create', '-f', '-'], {
      input: `${JSON.stringify(manifest)}\n`,
      capture: true,
      quiet: true,
    });
  }

  async start() {
    assert.equal(this.started, false, 'K3s fixture already started');
    this.dockerRun(['version'], { capture: true, quiet: true });
    const image = JSON.parse(
      this.dockerRun(['image', 'inspect', this.k3sImage], {
        capture: true,
        quiet: true,
      }).stdout,
    )[0];
    assert.ok(
      image.RepoDigests?.includes(`rancher/k3s@${this.k3sDigest}`),
      `K3s image does not retain reviewed digest ${this.k3sDigest}`,
    );
    for (const name of this.nodes) {
      assert.equal(
        this.dockerRun(['inspect', name], {
          capture: true,
          quiet: true,
          allowFailure: true,
        }).status,
        1,
        `refusing to reuse Docker container ${name}`,
      );
    }
    assert.equal(
      this.dockerRun(['network', 'inspect', this.network], {
        capture: true,
        quiet: true,
        allowFailure: true,
      }).status,
      1,
      `refusing to reuse Docker network ${this.network}`,
    );
    this.dockerRun(['network', 'create', this.network], {
      capture: true,
      quiet: true,
    });
    this.networkCreated = true;
    const token = randomBytes(32).toString('base64url');
    this.dockerRun(
      [
        'run',
        '-d',
        '--privileged',
        '--network',
        this.network,
        '--name',
        this.server,
        '-p',
        '127.0.0.1::6443',
        this.k3sImage,
        'server',
        '--token',
        token,
        '--node-name',
        this.server,
        '--disable=traefik',
        '--disable=servicelb',
        '--write-kubeconfig-mode=600',
        '--tls-san=127.0.0.1',
      ],
      { capture: true, quiet: true },
    );
    this.createdContainers.add(this.server);
    try {
      await waitFor('K3s control-plane readiness', 120_000, () => {
        const result = this.dockerRun(
          ['exec', this.server, 'kubectl', 'get', '--raw=/readyz'],
          { capture: true, quiet: true, allowFailure: true },
        );
        return result.status === 0 && result.stdout === 'ok'
          ? { ready: true, value: true }
          : { ready: false, fact: result.stderr || result.stdout };
      });
    } catch (error) {
      const state = this.dockerRun(
        ['inspect', '--format', '{{json .State}}', this.server],
        { capture: true, quiet: true, allowFailure: true },
      );
      const logs = this.dockerRun(['logs', '--tail', '120', this.server], {
        capture: true,
        quiet: true,
        allowFailure: true,
      });
      throw new Error(
        `${error instanceof Error ? error.message : String(error)}; ` +
          `state=${state.stdout || state.stderr}; ` +
          `logs=${logs.stderr || logs.stdout}`,
      );
    }
    for (const agent of this.agents) {
      this.dockerRun(
        [
          'run',
          '-d',
          '--privileged',
          '--network',
          this.network,
          '--name',
          agent,
          this.k3sImage,
          'agent',
          '--server',
          `https://${this.server}:6443`,
          '--token',
          token,
          '--node-name',
          agent,
        ],
        { capture: true, quiet: true },
      );
      this.createdContainers.add(agent);
    }
    const port = this.dockerRun(['port', this.server, '6443/tcp'], {
      capture: true,
      quiet: true,
    }).stdout;
    assert.match(port, /^127\.0\.0\.1:\d+$/);
    const config = this.dockerRun(
      ['exec', this.server, 'cat', '/etc/rancher/k3s/k3s.yaml'],
      { capture: true, quiet: true },
    ).stdout.replace('https://127.0.0.1:6443', `https://${port}`);
    fs.writeFileSync(this.kubeconfig, `${config}\n`, {
      mode: 0o600,
      flag: 'wx',
    });
    this.started = true;
    let ready;
    try {
      ready = await waitFor('three ready K3s nodes', 300_000, () => {
        const nodes = this.kubectlJson(['get', 'nodes']).items ?? [];
        const readyNodes = nodes.filter((node) =>
          node.status.conditions?.some(
            (condition) =>
              condition.type === 'Ready' && condition.status === 'True',
          ),
        );
        return readyNodes.length === 3
          ? { ready: true, value: readyNodes }
          : { ready: false, fact: `${readyNodes.length}/3 Ready nodes` };
      });
    } catch (error) {
      const diagnostics = this.nodes.map((node) => {
        const state = this.dockerRun(
          ['inspect', '--format', '{{json .State}}', node],
          { capture: true, quiet: true, allowFailure: true },
        );
        const logs = this.dockerRun(['logs', '--tail', '80', node], {
          capture: true,
          quiet: true,
          allowFailure: true,
        });
        return {
          node,
          state: state.stdout || state.stderr,
          logs: logs.stderr || logs.stdout,
        };
      });
      throw new Error(
        `${error instanceof Error ? error.message : String(error)}; ` +
          `nodes=${JSON.stringify(diagnostics)}`,
      );
    }
    return ready.value;
  }

  inspectImage(reference) {
    const images = JSON.parse(
      this.dockerRun(['image', 'inspect', reference], {
        capture: true,
        quiet: true,
      }).stdout,
    );
    assert.equal(images.length, 1);
    return images[0];
  }

  loadImage(reference, archiveName = 'image.tar') {
    const archive = path.join(this.temporary, archiveName);
    this.dockerRun(['image', 'save', '--output', archive, reference]);
    try {
      for (const node of this.nodes) {
        const remote = `/tmp/${path.basename(archive)}`;
        this.dockerRun(['cp', archive, `${node}:${remote}`], {
          capture: true,
          quiet: true,
        });
        try {
          this.dockerRun(
            [
              'exec',
              node,
              'ctr',
              '--address',
              '/run/k3s/containerd/containerd.sock',
              '--namespace',
              'k8s.io',
              'images',
              'import',
              remote,
            ],
            { capture: true, quiet: true },
          );
        } finally {
          this.dockerRun(['exec', node, 'rm', '-f', remote], {
            capture: true,
            quiet: true,
            allowFailure: true,
          });
        }
      }
    } finally {
      fs.rmSync(archive, { force: true });
    }
  }

  containerAddress(name) {
    assert.ok(this.nodes.includes(name), `unknown fixture node ${name}`);
    const address = this.dockerRun(
      [
        'inspect',
        '--format',
        '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}',
        name,
      ],
      { capture: true, quiet: true },
    ).stdout;
    assert.match(address, /^\d{1,3}(?:\.\d{1,3}){3}$/);
    return address;
  }

  stopNode(name) {
    assert.ok(this.nodes.includes(name), `unknown fixture node ${name}`);
    this.dockerRun(['stop', '--time', '1', name], {
      capture: true,
      quiet: true,
    });
  }

  startNode(name) {
    assert.ok(this.nodes.includes(name), `unknown fixture node ${name}`);
    this.dockerRun(['start', name], { capture: true, quiet: true });
  }

  async cleanup() {
    for (const name of [...this.createdContainers].reverse()) {
      this.dockerRun(['rm', '-f', '-v', name], {
        capture: true,
        quiet: true,
        allowFailure: true,
      });
    }
    this.createdContainers.clear();
    if (this.networkCreated) {
      this.dockerRun(['network', 'rm', this.network], {
        capture: true,
        quiet: true,
        allowFailure: true,
      });
      this.networkCreated = false;
    }
    fs.rmSync(this.temporary, { recursive: true, force: true });
    this.started = false;
  }
}

module.exports = {
  DEFAULT_K3S_DIGEST,
  DEFAULT_K3S_IMAGE,
  K3sDockerLiveFixture,
  run,
  sleep,
  waitFor,
};