mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,417 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { randomBytes } = require('node:crypto');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { spawnSync } = require('node:child_process');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '../..');
|
||||
const DEFAULT_K3S_IMAGE = 'rancher/k3s:v1.34.3-k3s1';
|
||||
const DEFAULT_K3S_DIGEST =
|
||||
'sha256:71abd3a56f57884c62732e0e0d87606052cb5f8555b7db7e8e33c04570b8175c';
|
||||
|
||||
function run(binary, args, options = {}) {
|
||||
if (!options.quiet) {
|
||||
process.stderr.write(`+ ${path.basename(binary)} ${args.join(' ')}\n`);
|
||||
}
|
||||
const result = spawnSync(binary, args, {
|
||||
cwd: options.cwd ?? ROOT,
|
||||
env: options.env ?? process.env,
|
||||
input: options.input,
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 64 * 1024 * 1024,
|
||||
stdio: options.capture
|
||||
? ['pipe', 'pipe', 'pipe']
|
||||
: [
|
||||
options.input === undefined ? 'inherit' : 'pipe',
|
||||
'inherit',
|
||||
'inherit',
|
||||
],
|
||||
});
|
||||
if (result.error) throw result.error;
|
||||
if (result.status !== 0 && !options.allowFailure) {
|
||||
throw new Error(
|
||||
`${path.basename(binary)} failed with ${String(result.status)}: ` +
|
||||
`${result.stderr || result.stdout || ''}`,
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
status: result.status,
|
||||
stdout: options.capture ? result.stdout.trim() : '',
|
||||
stderr: options.capture ? result.stderr.trim() : '',
|
||||
});
|
||||
}
|
||||
|
||||
function sleep(milliseconds) {
|
||||
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||
}
|
||||
|
||||
async function waitFor(description, timeoutMs, inspect, intervalMs = 500) {
|
||||
const startedAt = Date.now();
|
||||
let last = 'not observed';
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
try {
|
||||
const value = await inspect();
|
||||
if (value?.ready) {
|
||||
return Object.freeze({
|
||||
value: value.value,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
});
|
||||
}
|
||||
if (value?.fact) last = value.fact;
|
||||
} catch (error) {
|
||||
last = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
await sleep(intervalMs);
|
||||
}
|
||||
throw new Error(`${description} timed out: ${last}`);
|
||||
}
|
||||
|
||||
function safePrefix(value) {
|
||||
assert.match(value, /^ql3-[a-z0-9-]{1,32}$/);
|
||||
return value;
|
||||
}
|
||||
|
||||
class K3sDockerLiveFixture {
|
||||
constructor(options = {}) {
|
||||
this.docker = options.docker ?? process.env.QL3_DOCKER_BIN ?? 'docker';
|
||||
this.kubectlBinary =
|
||||
options.kubectl ?? process.env.QL3_KUBECTL_BIN ?? 'kubectl';
|
||||
this.k3sImage = options.k3sImage ?? DEFAULT_K3S_IMAGE;
|
||||
this.k3sDigest = options.k3sDigest ?? DEFAULT_K3S_DIGEST;
|
||||
this.prefix = safePrefix(options.prefix ?? 'ql3-k3s-live');
|
||||
this.suffix = `${process.pid.toString(36)}-${randomBytes(3).toString(
|
||||
'hex',
|
||||
)}`;
|
||||
this.network = `${this.prefix}-network-${this.suffix}`;
|
||||
this.server = `${this.prefix}-server-${this.suffix}`;
|
||||
this.agents = [
|
||||
`${this.prefix}-agent-a-${this.suffix}`,
|
||||
`${this.prefix}-agent-b-${this.suffix}`,
|
||||
];
|
||||
this.nodes = Object.freeze([this.server, ...this.agents]);
|
||||
this.temporary = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), `${this.prefix}-${this.suffix}-`),
|
||||
);
|
||||
this.kubeconfig = path.join(this.temporary, 'kubeconfig');
|
||||
this.createdContainers = new Set();
|
||||
this.networkCreated = false;
|
||||
this.started = false;
|
||||
}
|
||||
|
||||
dockerRun(args, options = {}) {
|
||||
return run(this.docker, args, options);
|
||||
}
|
||||
|
||||
kubectl(args, options = {}) {
|
||||
assert.equal(this.started, true, 'K3s fixture is not started');
|
||||
return run(
|
||||
this.kubectlBinary,
|
||||
['--kubeconfig', this.kubeconfig, ...args],
|
||||
options,
|
||||
);
|
||||
}
|
||||
|
||||
kubectlJson(args) {
|
||||
return JSON.parse(
|
||||
this.kubectl([...args, '-o', 'json'], {
|
||||
capture: true,
|
||||
quiet: true,
|
||||
}).stdout,
|
||||
);
|
||||
}
|
||||
|
||||
apply(manifest) {
|
||||
return this.kubectl(['apply', '-f', '-'], {
|
||||
input: `${JSON.stringify(manifest)}\n`,
|
||||
capture: true,
|
||||
quiet: true,
|
||||
});
|
||||
}
|
||||
|
||||
create(manifest) {
|
||||
return this.kubectl(['create', '-f', '-'], {
|
||||
input: `${JSON.stringify(manifest)}\n`,
|
||||
capture: true,
|
||||
quiet: true,
|
||||
});
|
||||
}
|
||||
|
||||
async start() {
|
||||
assert.equal(this.started, false, 'K3s fixture already started');
|
||||
this.dockerRun(['version'], { capture: true, quiet: true });
|
||||
const image = JSON.parse(
|
||||
this.dockerRun(['image', 'inspect', this.k3sImage], {
|
||||
capture: true,
|
||||
quiet: true,
|
||||
}).stdout,
|
||||
)[0];
|
||||
assert.ok(
|
||||
image.RepoDigests?.includes(`rancher/k3s@${this.k3sDigest}`),
|
||||
`K3s image does not retain reviewed digest ${this.k3sDigest}`,
|
||||
);
|
||||
for (const name of this.nodes) {
|
||||
assert.equal(
|
||||
this.dockerRun(['inspect', name], {
|
||||
capture: true,
|
||||
quiet: true,
|
||||
allowFailure: true,
|
||||
}).status,
|
||||
1,
|
||||
`refusing to reuse Docker container ${name}`,
|
||||
);
|
||||
}
|
||||
assert.equal(
|
||||
this.dockerRun(['network', 'inspect', this.network], {
|
||||
capture: true,
|
||||
quiet: true,
|
||||
allowFailure: true,
|
||||
}).status,
|
||||
1,
|
||||
`refusing to reuse Docker network ${this.network}`,
|
||||
);
|
||||
this.dockerRun(['network', 'create', this.network], {
|
||||
capture: true,
|
||||
quiet: true,
|
||||
});
|
||||
this.networkCreated = true;
|
||||
const token = randomBytes(32).toString('base64url');
|
||||
this.dockerRun(
|
||||
[
|
||||
'run',
|
||||
'-d',
|
||||
'--privileged',
|
||||
'--network',
|
||||
this.network,
|
||||
'--name',
|
||||
this.server,
|
||||
'-p',
|
||||
'127.0.0.1::6443',
|
||||
this.k3sImage,
|
||||
'server',
|
||||
'--token',
|
||||
token,
|
||||
'--node-name',
|
||||
this.server,
|
||||
'--disable=traefik',
|
||||
'--disable=servicelb',
|
||||
'--write-kubeconfig-mode=600',
|
||||
'--tls-san=127.0.0.1',
|
||||
],
|
||||
{ capture: true, quiet: true },
|
||||
);
|
||||
this.createdContainers.add(this.server);
|
||||
try {
|
||||
await waitFor('K3s control-plane readiness', 120_000, () => {
|
||||
const result = this.dockerRun(
|
||||
['exec', this.server, 'kubectl', 'get', '--raw=/readyz'],
|
||||
{ capture: true, quiet: true, allowFailure: true },
|
||||
);
|
||||
return result.status === 0 && result.stdout === 'ok'
|
||||
? { ready: true, value: true }
|
||||
: { ready: false, fact: result.stderr || result.stdout };
|
||||
});
|
||||
} catch (error) {
|
||||
const state = this.dockerRun(
|
||||
['inspect', '--format', '{{json .State}}', this.server],
|
||||
{ capture: true, quiet: true, allowFailure: true },
|
||||
);
|
||||
const logs = this.dockerRun(['logs', '--tail', '120', this.server], {
|
||||
capture: true,
|
||||
quiet: true,
|
||||
allowFailure: true,
|
||||
});
|
||||
throw new Error(
|
||||
`${error instanceof Error ? error.message : String(error)}; ` +
|
||||
`state=${state.stdout || state.stderr}; ` +
|
||||
`logs=${logs.stderr || logs.stdout}`,
|
||||
);
|
||||
}
|
||||
for (const agent of this.agents) {
|
||||
this.dockerRun(
|
||||
[
|
||||
'run',
|
||||
'-d',
|
||||
'--privileged',
|
||||
'--network',
|
||||
this.network,
|
||||
'--name',
|
||||
agent,
|
||||
this.k3sImage,
|
||||
'agent',
|
||||
'--server',
|
||||
`https://${this.server}:6443`,
|
||||
'--token',
|
||||
token,
|
||||
'--node-name',
|
||||
agent,
|
||||
],
|
||||
{ capture: true, quiet: true },
|
||||
);
|
||||
this.createdContainers.add(agent);
|
||||
}
|
||||
const port = this.dockerRun(['port', this.server, '6443/tcp'], {
|
||||
capture: true,
|
||||
quiet: true,
|
||||
}).stdout;
|
||||
assert.match(port, /^127\.0\.0\.1:\d+$/);
|
||||
const config = this.dockerRun(
|
||||
['exec', this.server, 'cat', '/etc/rancher/k3s/k3s.yaml'],
|
||||
{ capture: true, quiet: true },
|
||||
).stdout.replace('https://127.0.0.1:6443', `https://${port}`);
|
||||
fs.writeFileSync(this.kubeconfig, `${config}\n`, {
|
||||
mode: 0o600,
|
||||
flag: 'wx',
|
||||
});
|
||||
this.started = true;
|
||||
let ready;
|
||||
try {
|
||||
ready = await waitFor('three ready K3s nodes', 300_000, () => {
|
||||
const nodes = this.kubectlJson(['get', 'nodes']).items ?? [];
|
||||
const readyNodes = nodes.filter((node) =>
|
||||
node.status.conditions?.some(
|
||||
(condition) =>
|
||||
condition.type === 'Ready' && condition.status === 'True',
|
||||
),
|
||||
);
|
||||
return readyNodes.length === 3
|
||||
? { ready: true, value: readyNodes }
|
||||
: { ready: false, fact: `${readyNodes.length}/3 Ready nodes` };
|
||||
});
|
||||
} catch (error) {
|
||||
const diagnostics = this.nodes.map((node) => {
|
||||
const state = this.dockerRun(
|
||||
['inspect', '--format', '{{json .State}}', node],
|
||||
{ capture: true, quiet: true, allowFailure: true },
|
||||
);
|
||||
const logs = this.dockerRun(['logs', '--tail', '80', node], {
|
||||
capture: true,
|
||||
quiet: true,
|
||||
allowFailure: true,
|
||||
});
|
||||
return {
|
||||
node,
|
||||
state: state.stdout || state.stderr,
|
||||
logs: logs.stderr || logs.stdout,
|
||||
};
|
||||
});
|
||||
throw new Error(
|
||||
`${error instanceof Error ? error.message : String(error)}; ` +
|
||||
`nodes=${JSON.stringify(diagnostics)}`,
|
||||
);
|
||||
}
|
||||
return ready.value;
|
||||
}
|
||||
|
||||
inspectImage(reference) {
|
||||
const images = JSON.parse(
|
||||
this.dockerRun(['image', 'inspect', reference], {
|
||||
capture: true,
|
||||
quiet: true,
|
||||
}).stdout,
|
||||
);
|
||||
assert.equal(images.length, 1);
|
||||
return images[0];
|
||||
}
|
||||
|
||||
loadImage(reference, archiveName = 'image.tar') {
|
||||
const archive = path.join(this.temporary, archiveName);
|
||||
this.dockerRun(['image', 'save', '--output', archive, reference]);
|
||||
try {
|
||||
for (const node of this.nodes) {
|
||||
const remote = `/tmp/${path.basename(archive)}`;
|
||||
this.dockerRun(['cp', archive, `${node}:${remote}`], {
|
||||
capture: true,
|
||||
quiet: true,
|
||||
});
|
||||
try {
|
||||
this.dockerRun(
|
||||
[
|
||||
'exec',
|
||||
node,
|
||||
'ctr',
|
||||
'--address',
|
||||
'/run/k3s/containerd/containerd.sock',
|
||||
'--namespace',
|
||||
'k8s.io',
|
||||
'images',
|
||||
'import',
|
||||
remote,
|
||||
],
|
||||
{ capture: true, quiet: true },
|
||||
);
|
||||
} finally {
|
||||
this.dockerRun(['exec', node, 'rm', '-f', remote], {
|
||||
capture: true,
|
||||
quiet: true,
|
||||
allowFailure: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
fs.rmSync(archive, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
containerAddress(name) {
|
||||
assert.ok(this.nodes.includes(name), `unknown fixture node ${name}`);
|
||||
const address = this.dockerRun(
|
||||
[
|
||||
'inspect',
|
||||
'--format',
|
||||
'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}',
|
||||
name,
|
||||
],
|
||||
{ capture: true, quiet: true },
|
||||
).stdout;
|
||||
assert.match(address, /^\d{1,3}(?:\.\d{1,3}){3}$/);
|
||||
return address;
|
||||
}
|
||||
|
||||
stopNode(name) {
|
||||
assert.ok(this.nodes.includes(name), `unknown fixture node ${name}`);
|
||||
this.dockerRun(['stop', '--time', '1', name], {
|
||||
capture: true,
|
||||
quiet: true,
|
||||
});
|
||||
}
|
||||
|
||||
startNode(name) {
|
||||
assert.ok(this.nodes.includes(name), `unknown fixture node ${name}`);
|
||||
this.dockerRun(['start', name], { capture: true, quiet: true });
|
||||
}
|
||||
|
||||
async cleanup() {
|
||||
for (const name of [...this.createdContainers].reverse()) {
|
||||
this.dockerRun(['rm', '-f', '-v', name], {
|
||||
capture: true,
|
||||
quiet: true,
|
||||
allowFailure: true,
|
||||
});
|
||||
}
|
||||
this.createdContainers.clear();
|
||||
if (this.networkCreated) {
|
||||
this.dockerRun(['network', 'rm', this.network], {
|
||||
capture: true,
|
||||
quiet: true,
|
||||
allowFailure: true,
|
||||
});
|
||||
this.networkCreated = false;
|
||||
}
|
||||
fs.rmSync(this.temporary, { recursive: true, force: true });
|
||||
this.started = false;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
DEFAULT_K3S_DIGEST,
|
||||
DEFAULT_K3S_IMAGE,
|
||||
K3sDockerLiveFixture,
|
||||
run,
|
||||
sleep,
|
||||
waitFor,
|
||||
};
|
||||
Reference in New Issue
Block a user