feat(ql3): add generic worker management entry

This commit is contained in:
whyour
2026-08-20 13:30:06 +08:00
parent 4a4fa85f51
commit af5d5bfc0b
20 changed files with 1191 additions and 42 deletions
+11
View File
@@ -180,6 +180,16 @@
"require": "./dist/worker-credential/workerCredentialManagementClient.js",
"default": "./dist/worker-credential/workerCredentialManagementClient.js"
},
"./worker-management-client": {
"types": "./dist/worker-management/workerManagementClient.d.ts",
"require": "./dist/worker-management/workerManagementClient.js",
"default": "./dist/worker-management/workerManagementClient.js"
},
"./worker-management-product": {
"types": "./dist/worker-management/workerManagementProduct.d.ts",
"require": "./dist/worker-management/workerManagementProduct.js",
"default": "./dist/worker-management/workerManagementProduct.js"
},
"./worker-credential-management-executor": {
"types": "./dist/worker-credential/workerCredentialManagementExecutor.d.ts",
"require": "./dist/worker-credential/workerCredentialManagementExecutor.js",
@@ -420,6 +430,7 @@
"ql3-worker-credential-manage": "dist/worker-credential/management-server/workerCredentialManagementCli.js",
"ql3-worker-credential-execute": "dist/worker-credential/workerCredentialExecutorCli.js",
"ql3-worker-credential-client": "dist/worker-credential/workerCredentialManagementClientCli.js",
"ql3-worker-client": "dist/worker-management/workerManagementClientCli.js",
"ql3-approval-manage": "dist/approval-management/approvalManagementCli.js",
"ql3-approval-client": "dist/approval-management/approvalManagementClientCli.js",
"ql3-run-manage": "dist/run-management/runManagementCli.js",
@@ -15,6 +15,7 @@ import { TextDecoder } from 'node:util';
export type ClusterAuthenticatedManagementClientKind =
| 'package'
| 'worker'
| 'worker-credential'
| 'automation'
| 'approval'
@@ -34,6 +35,10 @@ const MANAGEMENT_CLIENT_POLICIES: Readonly<
managementPath: '/api/v3/plugin-packages/management',
clientCertificate: 'forbidden',
}),
worker: Object.freeze({
managementPath: '/api/v3/workers/management',
clientCertificate: 'required',
}),
'worker-credential': Object.freeze({
managementPath: '/api/v3/worker-credentials/management',
clientCertificate: 'required',
@@ -147,7 +152,9 @@ export function readCanonicalFile(
throw configurationFailure();
}
} catch (error) {
if (error instanceof ClusterPluginPackageManagementClientConfigurationError) {
if (
error instanceof ClusterPluginPackageManagementClientConfigurationError
) {
throw error;
}
throw configurationFailure();
@@ -209,7 +216,9 @@ export function readCanonicalFile(
return bytes;
} catch (error) {
bytes?.fill(0);
if (error instanceof ClusterPluginPackageManagementClientConfigurationError) {
if (
error instanceof ClusterPluginPackageManagementClientConfigurationError
) {
throw error;
}
throw configurationFailure();
@@ -349,7 +358,10 @@ export function prepareClusterAuthenticatedManagementClientConfiguration(
throw configurationFailure();
}
} catch (error) {
if (error instanceof ClusterPluginPackageManagementClientConfigurationError) {
if (
error instanceof
ClusterPluginPackageManagementClientConfigurationError
) {
throw error;
}
throw configurationFailure();
@@ -380,7 +392,9 @@ export function prepareClusterAuthenticatedManagementClientConfiguration(
caBytes?.fill(0);
clientCertificateBytes?.fill(0);
clientPrivateKeyBytes?.fill(0);
if (error instanceof ClusterPluginPackageManagementClientConfigurationError) {
if (
error instanceof ClusterPluginPackageManagementClientConfigurationError
) {
throw error;
}
throw configurationFailure();
@@ -408,11 +422,10 @@ export function validateClusterAuthenticatedManagementClientConfiguration(
): Readonly<ClusterAuthenticatedManagementClientConfigurationSummary> {
const policy = MANAGEMENT_CLIENT_POLICIES[kind];
if (policy === undefined) throw configurationFailure();
const prepared =
prepareClusterAuthenticatedManagementClientKindConfiguration(
configFile,
kind,
);
const prepared = prepareClusterAuthenticatedManagementClientKindConfiguration(
configFile,
kind,
);
try {
return Object.freeze({
schemaVersion: 1,
@@ -85,6 +85,7 @@ export const CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH =
'/api/v3/plugin-packages/management';
export const CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH =
'/api/v3/worker-credentials/management';
export const CLUSTER_WORKER_MANAGEMENT_PATH = '/api/v3/workers/management';
export const CLUSTER_AUTOMATION_MANAGEMENT_PATH =
'/api/v3/automations/management';
export const CLUSTER_APPROVAL_MANAGEMENT_PATH = '/api/v3/approvals/management';
@@ -94,6 +95,7 @@ export const CLUSTER_RUN_MANAGEMENT_PATH = '/api/v3/runs/management';
export type ClusterAuthenticatedManagementPath =
| typeof CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH
| typeof CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH
| typeof CLUSTER_WORKER_MANAGEMENT_PATH
| typeof CLUSTER_AUTOMATION_MANAGEMENT_PATH
| typeof CLUSTER_APPROVAL_MANAGEMENT_PATH
| typeof CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH
@@ -101,6 +103,7 @@ export type ClusterAuthenticatedManagementPath =
const MANAGEMENT_PATHS = new Set<ClusterAuthenticatedManagementPath>([
CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH,
CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH,
CLUSTER_WORKER_MANAGEMENT_PATH,
CLUSTER_AUTOMATION_MANAGEMENT_PATH,
CLUSTER_APPROVAL_MANAGEMENT_PATH,
CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH,
@@ -143,6 +146,7 @@ export interface StartClusterPluginPackageManagementHttpOptions {
readonly transport: ClusterAuthenticatedManagementTransport;
readonly identities: ClusterPluginPackageIdentityKeysetFile;
readonly managementPath?: ClusterAuthenticatedManagementPath;
readonly compatibleManagementPaths?: readonly ClusterAuthenticatedManagementPath[];
readonly limits?: ClusterPluginPackageManagementHttpLimits;
readonly now?: () => number;
readonly createRequestId?: () => string;
@@ -547,7 +551,8 @@ function responseError(error: unknown): HttpRequestError {
error instanceof ClusterApprovalManagementTransportAuthenticationError ||
error instanceof
ClusterModelProviderCredentialManagementTransportAuthenticationError ||
error instanceof ClusterModelProviderCredentialManagementAuthenticationError ||
error instanceof
ClusterModelProviderCredentialManagementAuthenticationError ||
error instanceof ClusterRunManagementTransportAuthenticationError
) {
return new HttpRequestError(401, 'authentication_required');
@@ -573,7 +578,8 @@ function responseError(error: unknown): HttpRequestError {
error instanceof WorkerCredentialManagementAuthorizationError ||
error instanceof ClusterAutomationManagementAuthorizationError ||
error instanceof ClusterApprovalManagementTransportAuthorizationError ||
error instanceof ClusterModelProviderCredentialManagementAuthorizationError ||
error instanceof
ClusterModelProviderCredentialManagementAuthorizationError ||
error instanceof ClusterRunManagementAuthorizationError
) {
return new HttpRequestError(403, 'forbidden');
@@ -665,6 +671,7 @@ export async function startClusterPluginPackageManagementHttp(
'transport',
'identities',
'managementPath',
'compatibleManagementPaths',
'limits',
'now',
'createRequestId',
@@ -711,6 +718,12 @@ export async function startClusterPluginPackageManagementHttp(
typeof options.identities.reload !== 'function' ||
(options.managementPath !== undefined &&
!MANAGEMENT_PATHS.has(options.managementPath)) ||
(options.compatibleManagementPaths !== undefined &&
(!Array.isArray(options.compatibleManagementPaths) ||
options.managementPath !== CLUSTER_WORKER_MANAGEMENT_PATH ||
options.compatibleManagementPaths.length !== 1 ||
options.compatibleManagementPaths[0] !==
CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH)) ||
(options.now !== undefined && typeof options.now !== 'function') ||
(options.createRequestId !== undefined &&
typeof options.createRequestId !== 'function') ||
@@ -721,6 +734,10 @@ export async function startClusterPluginPackageManagementHttp(
const limits = reviewedLimits(options.limits);
const managementPath =
options.managementPath ?? CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH;
const managementPaths = new Set<ClusterAuthenticatedManagementPath>([
managementPath,
...(options.compatibleManagementPaths ?? []),
]);
const now = options.now ?? Date.now;
const createRequestId = options.createRequestId ?? randomUUID;
const clientCertificateRequired =
@@ -824,7 +841,10 @@ export async function startClusterPluginPackageManagementHttp(
) {
throw new HttpRequestError(401, 'client_certificate_required');
}
if (request.method !== 'POST' || url !== managementPath) {
if (
request.method !== 'POST' ||
!managementPaths.has(url as ClusterAuthenticatedManagementPath)
) {
throw new HttpRequestError(404, 'not_found');
}
if (availability !== 'ready') {
@@ -71,6 +71,12 @@ export const QINGLONG3_CLUSTER_PRODUCT_COMMANDS: readonly QingLong3ClusterProduc
'plugin-package/management/pluginPackageManagementKubernetesClientCli.js',
description: 'manage Plugin Packages through a bounded Kubernetes tunnel',
}),
Object.freeze({
name: 'worker',
binary: 'ql3-worker-client',
target: 'worker-management/workerManagementClientCli.js',
description: 'inspect bounded Worker session state',
}),
Object.freeze({
name: 'worker-credential',
binary: 'ql3-worker-credential-client',
@@ -14,9 +14,7 @@ import {
probeClusterCopilotClientReadiness,
validateClusterCopilotClientConfiguration,
} from '../copilot-client/client';
import {
validateClusterAuthenticatedManagementClientConfiguration,
} from '../management-support/pluginPackageManagementClient';
import { validateClusterAuthenticatedManagementClientConfiguration } from '../management-support/pluginPackageManagementClient';
import type { ClusterAuthenticatedManagementClientKind } from '../management-support/pluginPackageManagementClient';
import { probeClusterAuthenticatedManagementClientReadiness } from '../management-support/managementReadinessProbe';
import {
@@ -31,6 +29,7 @@ const CONTEXT_COMMANDS = Object.freeze([
'copilot',
'package',
'package-kubernetes',
'worker',
'worker-credential',
'approval',
'run',
@@ -92,6 +91,7 @@ const CONTEXT_COMMAND_CLIENT_KINDS: Readonly<
> = Object.freeze({
package: 'package',
'package-kubernetes': 'package',
worker: 'worker',
'worker-credential': 'worker-credential',
approval: 'approval',
run: 'run',
@@ -337,11 +337,10 @@ export async function validateQingLong3ClusterProductContext(
}),
);
} else if (name === 'package-kubernetes') {
const https =
validateClusterAuthenticatedManagementClientConfiguration(
command.configFile,
CONTEXT_COMMAND_CLIENT_KINDS[name],
);
const https = validateClusterAuthenticatedManagementClientConfiguration(
command.configFile,
CONTEXT_COMMAND_CLIENT_KINDS[name],
);
const kubernetes =
await validateClusterPluginPackageManagementKubernetesConfiguration(
command.kubernetesFile!,
@@ -355,11 +354,10 @@ export async function validateQingLong3ClusterProductContext(
}),
);
} else {
const https =
validateClusterAuthenticatedManagementClientConfiguration(
command.configFile,
CONTEXT_COMMAND_CLIENT_KINDS[name],
);
const https = validateClusterAuthenticatedManagementClientConfiguration(
command.configFile,
CONTEXT_COMMAND_CLIENT_KINDS[name],
);
commands.push(
Object.freeze({
name,
@@ -403,7 +401,7 @@ export async function probeQingLong3ClusterProductContext(
command.kubernetesFile!,
)
: name === 'copilot'
? await probeClusterCopilotClientReadiness(command.configFile)
? await probeClusterCopilotClientReadiness(command.configFile)
: await probeClusterAuthenticatedManagementClientReadiness(
command.configFile,
CONTEXT_COMMAND_CLIENT_KINDS[name],
@@ -1,6 +1,7 @@
/** TLS 1.3 Worker credential management HTTP adapter boundary. */
import {
CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH,
CLUSTER_WORKER_MANAGEMENT_PATH,
startClusterPluginPackageManagementHttp,
type ClusterPluginPackageManagementHttpApplication,
type ClusterPluginPackageManagementHttpLimits,
@@ -32,15 +33,17 @@ export interface StartClusterWorkerCredentialManagementHttpOptions {
}
/**
* Starts the Worker credential management endpoint on the shared Cluster Admin
* TLS 1.3/OIDC boundary. The public manager process never receives credential
* delivery or Kubernetes execution capabilities.
* Starts the Worker management endpoint on the shared Cluster Admin TLS
* 1.3/OIDC boundary. The former credential-scoped path remains an exact
* compatibility alias on this listener. The public manager process never
* receives credential delivery or Kubernetes execution capabilities.
*/
export async function startClusterWorkerCredentialManagementHttp(
options: StartClusterWorkerCredentialManagementHttpOptions,
): Promise<Readonly<ClusterWorkerCredentialManagementHttpApplication>> {
return startClusterPluginPackageManagementHttp({
...options,
managementPath: CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH,
managementPath: CLUSTER_WORKER_MANAGEMENT_PATH,
compatibleManagementPaths: [CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH],
});
}
@@ -0,0 +1,81 @@
/** Read-only product client boundary for bounded Worker session observation. */
import {
executeClusterAuthenticatedManagementClient,
type ClusterAuthenticatedManagementClientResult,
type ClusterAuthenticatedManagementCommandExecution,
type ClusterPluginPackageManagementClientConnectionOptions,
} from '../management-support/pluginPackageManagementClient';
import {
ClusterWorkerCredentialManagementTransportRequestError,
normalizeClusterWorkerCredentialManagementCommand,
type ClusterWorkerCredentialManagementCommand,
type ClusterWorkerCredentialManagementTransportResult,
} from '../worker-credential/management-server/workerCredentialManagementTransport';
import { validateClusterWorkerCredentialManagementClientResult } from '../worker-credential/workerCredentialManagementClient';
const MANAGEMENT_PATH = '/api/v3/workers/management';
export type ClusterWorkerManagementCommand = Extract<
ClusterWorkerCredentialManagementCommand,
{ readonly operation: 'worker-session.inspect' | 'worker-session.list' }
>;
export type ClusterWorkerManagementTransportResult = Extract<
ClusterWorkerCredentialManagementTransportResult,
{ readonly operation: 'worker-session.inspect' | 'worker-session.list' }
>;
export type ClusterWorkerManagementClientExecution =
ClusterAuthenticatedManagementCommandExecution<ClusterWorkerManagementCommand>;
export type ClusterWorkerManagementClientConnectionOptions =
ClusterPluginPackageManagementClientConnectionOptions;
export type ClusterWorkerManagementClientResult =
ClusterAuthenticatedManagementClientResult<ClusterWorkerManagementTransportResult>;
export function normalizeClusterWorkerManagementCommand(
value: unknown,
): Readonly<ClusterWorkerManagementCommand> {
const command = normalizeClusterWorkerCredentialManagementCommand(value);
if (
command.operation !== 'worker-session.inspect' &&
command.operation !== 'worker-session.list'
) {
throw new ClusterWorkerCredentialManagementTransportRequestError(
'operation is not available through the read-only Worker client',
);
}
return command;
}
function validateClusterWorkerManagementResult(
value: unknown,
command: Readonly<ClusterWorkerManagementCommand>,
): Readonly<ClusterWorkerManagementTransportResult> {
const result = validateClusterWorkerCredentialManagementClientResult(
value,
command,
);
if (
result.operation !== 'worker-session.inspect' &&
result.operation !== 'worker-session.list'
) {
throw new Error('Worker management response is invalid');
}
return result;
}
const PROTOCOL = Object.freeze({
managementPath: MANAGEMENT_PATH,
clientCertificate: 'required' as const,
normalizeCommand: normalizeClusterWorkerManagementCommand,
validateResult: validateClusterWorkerManagementResult,
});
export async function executeClusterWorkerManagementClient(
execution: ClusterWorkerManagementClientExecution,
connectionOptions?: ClusterWorkerManagementClientConnectionOptions,
): Promise<Readonly<ClusterWorkerManagementClientResult>> {
return executeClusterAuthenticatedManagementClient(
execution,
PROTOCOL,
connectionOptions,
);
}
@@ -0,0 +1,150 @@
#!/usr/bin/env node
import { ClusterPluginPackageManagementClientRemoteError } from '../management-support/pluginPackageManagementClient';
import { executeClusterWorkerManagementClient } from './workerManagementClient';
import {
createWorkerSessionInspectionCommand,
createWorkerSessionListCommand,
formatWorkerSessionInspectionCard,
formatWorkerSessionListCard,
projectWorkerSessionInspection,
projectWorkerSessionList,
} from './workerManagementProduct';
const USAGE = [
'Usage: ql3-worker-client inspect --config=/absolute/client.json --assertion=/absolute/assertion.jwt --project=PROJECT --worker=WORKER [--format=text|json]',
' ql3-worker-client list --config=/absolute/client.json --assertion=/absolute/assertion.jwt --project=PROJECT [--after=WORKER] [--format=text|json]',
'',
'One invocation performs one bounded read; it never retries, polls or auto-pages.',
].join('\n');
const IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
type WorkerClientArguments = Readonly<{
kind: 'inspect' | 'list';
configFile: string;
assertionFile: string;
projectId: string;
workerId?: string;
afterWorkerId?: string;
format: 'text' | 'json';
}>;
function argumentsFrom(argv: readonly string[]): WorkerClientArguments | null {
const modes = argv.filter(
(argument) => argument === 'inspect' || argument === 'list',
);
if (modes.length !== 1 || argv.length < 4 || argv.length > 6) return null;
const kind = modes[0] as 'inspect' | 'list';
const values = new Map<string, string>();
for (const argument of argv) {
if (argument === kind) continue;
const match =
/^--(config|assertion|project|worker|after|format)=(.+)$/.exec(argument);
if (!match || values.has(match[1]!)) return null;
values.set(match[1]!, match[2]!);
}
if (
!values.get('config')?.startsWith('/') ||
!values.get('assertion')?.startsWith('/') ||
!IDENTIFIER.test(values.get('project') ?? '') ||
(kind === 'inspect' &&
(!IDENTIFIER.test(values.get('worker') ?? '') || values.has('after'))) ||
(kind === 'list' &&
(values.has('worker') ||
(values.has('after') && !IDENTIFIER.test(values.get('after')!)))) ||
(values.has('format') &&
values.get('format') !== 'text' &&
values.get('format') !== 'json')
) {
return null;
}
return Object.freeze({
kind,
configFile: values.get('config')!,
assertionFile: values.get('assertion')!,
projectId: values.get('project')!,
...(kind === 'inspect' ? { workerId: values.get('worker')! } : {}),
...(kind === 'list' && values.has('after')
? { afterWorkerId: values.get('after')! }
: {}),
format: (values.get('format') ?? 'text') as 'text' | 'json',
});
}
function failureFact(error: unknown): Readonly<Record<string, unknown>> {
const candidate = error as { readonly code?: unknown };
return Object.freeze({
schemaVersion: 1,
component: 'qinglong3-worker-management-client',
event: 'inspection_failed',
code:
typeof candidate?.code === 'string'
? candidate.code
: 'QL3_WORKER_MANAGEMENT_CLIENT_FAILED',
...(error instanceof ClusterPluginPackageManagementClientRemoteError
? {
statusCode: error.statusCode,
responseCode: error.responseCode,
requestId: error.requestId,
...(error.retryAfterSeconds === null
? {}
: { retryAfterSeconds: error.retryAfterSeconds }),
}
: {}),
});
}
async function run(argv: readonly string[]): Promise<void> {
if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
process.stdout.write(`${USAGE}\n`);
return;
}
const argumentsValue = argumentsFrom(argv);
if (argumentsValue === null) {
process.stderr.write(
`${JSON.stringify({
schemaVersion: 1,
component: 'qinglong3-worker-management-client',
event: 'usage_invalid',
code: 'QL3_WORKER_MANAGEMENT_CLIENT_USAGE_INVALID',
})}\n`,
);
process.exitCode = 64;
return;
}
try {
const command =
argumentsValue.kind === 'inspect'
? createWorkerSessionInspectionCommand(
argumentsValue.projectId,
argumentsValue.workerId!,
)
: createWorkerSessionListCommand(
argumentsValue.projectId,
argumentsValue.afterWorkerId,
);
const response = await executeClusterWorkerManagementClient({
configFile: argumentsValue.configFile,
assertionFile: argumentsValue.assertionFile,
command,
});
const projection =
argumentsValue.kind === 'inspect'
? projectWorkerSessionInspection(argumentsValue.projectId, response)
: projectWorkerSessionList(argumentsValue.projectId, response);
process.stdout.write(
argumentsValue.format === 'json'
? `${JSON.stringify(projection)}\n`
: `${
projection.schema === 'qinglong/worker-session-inspection@v1'
? formatWorkerSessionInspectionCard(projection)
: formatWorkerSessionListCard(projection)
}\n`,
);
} catch (error) {
process.stderr.write(`${JSON.stringify(failureFact(error))}\n`);
process.exitCode = 1;
}
}
void run(process.argv.slice(2));
@@ -0,0 +1,192 @@
/** Bounded commands and low-sensitive product projections for Worker sessions. */
import { randomUUID } from 'node:crypto';
import type {
ClusterWorkerManagementClientResult,
ClusterWorkerManagementCommand,
ClusterWorkerManagementTransportResult,
} from './workerManagementClient';
const IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
type InspectCommand = Extract<
ClusterWorkerManagementCommand,
{ readonly operation: 'worker-session.inspect' }
>;
type ListCommand = Extract<
ClusterWorkerManagementCommand,
{ readonly operation: 'worker-session.list' }
>;
type InspectResult = Extract<
ClusterWorkerManagementTransportResult,
{ readonly operation: 'worker-session.inspect' }
>;
type ListResult = Extract<
ClusterWorkerManagementTransportResult,
{ readonly operation: 'worker-session.list' }
>;
export interface WorkerSessionInspection {
readonly schema: 'qinglong/worker-session-inspection@v1';
readonly projectId: string;
readonly observedAtMs: number;
readonly found: boolean;
readonly worker: InspectResult['worker'];
}
export interface WorkerSessionList {
readonly schema: 'qinglong/worker-session-list@v1';
readonly projectId: string;
readonly observedAtMs: number;
readonly count: number;
readonly workers: ListResult['workers'];
readonly nextAfterWorkerId: string | null;
}
export class ClusterWorkerManagementProductError extends TypeError {
readonly code = 'QL3_WORKER_MANAGEMENT_PRODUCT_INPUT_INVALID';
constructor() {
super('Worker management product input is invalid');
this.name = 'ClusterWorkerManagementProductError';
}
}
function identifier(value: string): string {
if (!IDENTIFIER.test(value)) throw new ClusterWorkerManagementProductError();
return value;
}
function inspectionId(createId: () => string): string {
const value = createId();
return identifier(value);
}
export function createWorkerSessionInspectionCommand(
projectId: string,
workerId: string,
createId: () => string = randomUUID,
): Readonly<InspectCommand> {
return Object.freeze({
schemaVersion: 1,
operation: 'worker-session.inspect',
request: Object.freeze({
authorityProjectId: identifier(projectId),
workerId: identifier(workerId),
inspectionId: inspectionId(createId),
}),
});
}
export function createWorkerSessionListCommand(
projectId: string,
afterWorkerId?: string,
createId: () => string = randomUUID,
): Readonly<ListCommand> {
return Object.freeze({
schemaVersion: 1,
operation: 'worker-session.list',
request: Object.freeze({
authorityProjectId: identifier(projectId),
afterWorkerId:
afterWorkerId === undefined ? null : identifier(afterWorkerId),
inspectionId: inspectionId(createId),
}),
});
}
function cloneWorker<
Worker extends
| NonNullable<InspectResult['worker']>
| ListResult['workers'][number],
>(worker: Worker): Worker {
return Object.freeze({
...worker,
...('runtimes' in worker
? {
runtimes: Object.freeze(
worker.runtimes.map((runtime) => Object.freeze({ ...runtime })),
),
declaredCapacity: Object.freeze({ ...worker.declaredCapacity }),
}
: {}),
}) as Worker;
}
export function projectWorkerSessionInspection(
projectId: string,
response: Readonly<ClusterWorkerManagementClientResult>,
): Readonly<WorkerSessionInspection> {
if (response.result.operation !== 'worker-session.inspect') {
throw new ClusterWorkerManagementProductError();
}
return Object.freeze({
schema: 'qinglong/worker-session-inspection@v1',
projectId: identifier(projectId),
observedAtMs: response.result.observedAtMs,
found: response.result.worker !== null,
worker:
response.result.worker === null
? null
: cloneWorker(response.result.worker),
});
}
export function projectWorkerSessionList(
projectId: string,
response: Readonly<ClusterWorkerManagementClientResult>,
): Readonly<WorkerSessionList> {
if (response.result.operation !== 'worker-session.list') {
throw new ClusterWorkerManagementProductError();
}
const workers = Object.freeze(response.result.workers.map(cloneWorker));
return Object.freeze({
schema: 'qinglong/worker-session-list@v1',
projectId: identifier(projectId),
observedAtMs: response.result.observedAtMs,
count: workers.length,
workers,
nextAfterWorkerId: response.result.nextCursor,
});
}
export function formatWorkerSessionInspectionCard(
inspection: Readonly<WorkerSessionInspection>,
): string {
if (inspection.worker === null) {
return [
`Worker session: not found`,
`Project: ${inspection.projectId}`,
`Observed: ${inspection.observedAtMs}`,
].join('\n');
}
const worker = inspection.worker;
return [
`Worker session: ${worker.workerId}`,
`Project: ${inspection.projectId}`,
`State: ${worker.lifecycle} / ${worker.compatibility} / ${worker.supportTier}`,
`Platform: ${worker.operatingSystem ?? 'unknown'} ${
worker.architecture
} / protocol ${worker.protocolVersion}`,
`Capacity: ${worker.availableSlots}/${worker.maxConcurrentRuns} slots available`,
`Heartbeat: ${worker.lastHeartbeatAtMs} / lease ${worker.leaseExpiresAtMs}`,
`Observed: ${inspection.observedAtMs}`,
].join('\n');
}
export function formatWorkerSessionListCard(
page: Readonly<WorkerSessionList>,
): string {
const lines = [
`Worker sessions: ${page.count}`,
`Project: ${page.projectId}`,
`Observed: ${page.observedAtMs}`,
];
for (const worker of page.workers) {
lines.push(
`${worker.workerId} ${worker.lifecycle} ${worker.supportTier} ${worker.architecture} slots ${worker.availableSlots}/${worker.maxConcurrentRuns}`,
);
}
lines.push(`Next after: ${page.nextAfterWorkerId ?? '-'}`);
return lines.join('\n');
}
@@ -284,6 +284,13 @@ function validContextFixture(t) {
copilot: { configFile: copilotConfig },
package: { configFile: packageConfig },
'package-kubernetes': { configFile: packageConfig, kubernetesFile },
worker: {
configFile: config(
'worker-observation-client',
'/api/v3/workers/management',
'required',
),
},
'worker-credential': {
configFile: config(
'worker-client',
@@ -329,7 +336,7 @@ function validContextFixture(t) {
test('catalog exposes only reviewed product entrypoints from the same package', () => {
assert.equal(manifest.bin['ql3-cluster-admin'], 'dist/product-cli/cli.js');
assert.equal(QINGLONG3_CLUSTER_PRODUCT_COMMANDS.length, 11);
assert.equal(QINGLONG3_CLUSTER_PRODUCT_COMMANDS.length, 12);
assert.equal(
new Set(QINGLONG3_CLUSTER_PRODUCT_COMMANDS.map(({ name }) => name)).size,
QINGLONG3_CLUSTER_PRODUCT_COMMANDS.length,
@@ -617,7 +624,7 @@ test('validates the complete operator context offline without operational author
schemaVersion: 1,
component: 'qinglong3-cluster-product-cli',
event: 'context_valid',
commandCount: 8,
commandCount: 9,
commands: [
{ name: 'copilot', transport: 'https', clientCertificate: 'forbidden' },
{ name: 'package', transport: 'https', clientCertificate: 'forbidden' },
@@ -627,6 +634,11 @@ test('validates the complete operator context offline without operational author
clientCertificate: 'forbidden',
kubernetesAuthentication: 'token',
},
{
name: 'worker',
transport: 'https',
clientCertificate: 'required',
},
{
name: 'worker-credential',
transport: 'https',
@@ -56,6 +56,7 @@ const NEXT_CLIENT_KEY = resolve(
'fixtures/management-service-key.pem',
);
const WORKER_PATH = '/api/v3/worker-credentials/management';
const CANONICAL_WORKER_PATH = '/api/v3/workers/management';
function command() {
return {
@@ -223,7 +224,7 @@ async function requestWithClientIdentity(application, certificate, key) {
});
}
test('serves only the fixed Worker credential management route', async () => {
test('serves the canonical Worker route and exact credential compatibility alias', async () => {
const calls = [];
const application = await start(async (value, authentication) => {
calls.push({ value, principal: await authentication.authenticate() });
@@ -245,12 +246,15 @@ test('serves only the fixed Worker credential management route', async () => {
type: 'user',
id: 'cluster-reviewer',
});
const canonical = await request(application, CANONICAL_WORKER_PATH);
assert.equal(canonical.statusCode, 200);
assert.equal(canonical.body.result.operation, 'worker-credential.inspect');
assert.equal(
(await request(application, '/api/v3/plugin-packages/management'))
.statusCode,
404,
);
assert.equal(calls.length, 1);
assert.equal(calls.length, 2);
} finally {
await application.close();
}
@@ -312,10 +316,18 @@ test('rejects a CRL-revoked client certificate before OIDC', async () => {
test('maps Worker credential management failures to stable HTTP errors', async () => {
for (const [failure, statusCode, code] of [
[new WorkerCredentialManagementRequestError('invalid'), 400, 'request_invalid'],
[
new WorkerCredentialManagementRequestError('invalid'),
400,
'request_invalid',
],
[new WorkerCredentialManagementAuthorizationError(), 403, 'forbidden'],
[new WorkerCredentialManagementConflictError('conflict'), 409, 'conflict'],
[new WorkerCredentialManagementQuotaExceededError(1_250), 429, 'quota_exceeded'],
[
new WorkerCredentialManagementQuotaExceededError(1_250),
429,
'quota_exceeded',
],
[new WorkerCredentialManagementUnavailableError(), 503, 'unavailable'],
]) {
const application = await start(async () => {
@@ -353,6 +365,34 @@ test('rejects arbitrary management paths at configuration time', async () => {
}
});
test('rejects arbitrary or cross-plane compatible route aliases', async () => {
for (const compatibleManagementPaths of [
['/api/v3/automations/management'],
['/api/v3/worker-credentials/management', '/api/v3/runs/management'],
]) {
const privateKey = Buffer.from(readFileSync(SERVER_KEY));
try {
await assert.rejects(
startClusterPluginPackageManagementHttp({
host: '127.0.0.1',
port: 0,
tls: {
privateKey,
certificate: Buffer.from(readFileSync(SERVER_CERT)),
},
identities: identities(),
transport: { async execute() {} },
managementPath: CANONICAL_WORKER_PATH,
compatibleManagementPaths,
}),
ClusterPluginPackageManagementHttpConfigurationError,
);
} finally {
privateKey.fill(0);
}
}
});
test('accepts both client CAs during overlap then rejects the retired CA', async () => {
const execute = async () => ({
schemaVersion: 1,
@@ -0,0 +1,240 @@
'use strict';
const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const test = require('node:test');
const {
executeClusterWorkerManagementClient,
normalizeClusterWorkerManagementCommand,
} = require('@qinglong/cluster-admin/worker-management-client');
const {
createWorkerSessionInspectionCommand,
createWorkerSessionListCommand,
formatWorkerSessionInspectionCard,
formatWorkerSessionListCard,
projectWorkerSessionInspection,
projectWorkerSessionList,
} = require('@qinglong/cluster-admin/worker-management-product');
const fixtureRoot = path.resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls',
);
const detailedWorker = Object.freeze({
workerId: 'worker-a',
sessionId: 'session-a',
generation: 2,
sessionVersion: 5,
lifecycle: 'online',
compatibility: 'default_placement',
architecture: 'arm64',
supportTier: 'tier1',
protocolVersion: '1.0.0',
operatingSystem: 'linux',
maxConcurrentRuns: 2,
availableSlots: 1,
registeredAtMs: 900,
lastHeartbeatAtMs: 1_050,
leaseExpiresAtMs: 2_000,
updatedAtMs: 1_050,
observedAtMs: 1_100,
runtimes: Object.freeze([{ name: 'node', version: '24.18.0' }]),
declaredCapacity: Object.freeze({
cpuCores: 1,
memoryBytes: 268_435_456,
diskBytes: 1_073_741_824,
gpuCount: 0,
}),
});
const summaryWorker = Object.freeze(
Object.fromEntries(
Object.entries(detailedWorker).filter(
([key]) => key !== 'runtimes' && key !== 'declaredCapacity',
),
),
);
function response(result) {
return Object.freeze({
schemaVersion: 1,
requestId: 'transport-request-must-not-project',
result: Object.freeze(result),
});
}
function privateFile(directory, name, value) {
const filePath = path.join(directory, name);
fs.writeFileSync(filePath, value, { mode: 0o600 });
return fs.realpathSync(filePath);
}
test('builds only immutable inspect and bounded list commands', () => {
const inspect = createWorkerSessionInspectionCommand(
'project-a',
'worker-a',
() => 'inspection-a',
);
assert.deepEqual(inspect, {
schemaVersion: 1,
operation: 'worker-session.inspect',
request: {
authorityProjectId: 'project-a',
workerId: 'worker-a',
inspectionId: 'inspection-a',
},
});
assert.equal(Object.isFrozen(inspect), true);
assert.equal(Object.isFrozen(inspect.request), true);
const page = createWorkerSessionListCommand(
'project-a',
'worker-a',
() => 'inspection-b',
);
assert.deepEqual(page.request, {
authorityProjectId: 'project-a',
afterWorkerId: 'worker-a',
inspectionId: 'inspection-b',
});
assert.equal(Object.hasOwn(page.request, 'limit'), false);
assert.throws(() => createWorkerSessionListCommand('../escape'));
assert.throws(() =>
normalizeClusterWorkerManagementCommand({
schemaVersion: 1,
operation: 'worker-credential.inspect',
request: {
actionRef: 'action-a',
authorityProjectId: 'project-a',
approvalRequestId: 'approval-a',
inspectionId: 'inspection-a',
},
}),
);
});
test('projects strict low-sensitive products without transport request identity', () => {
const inspection = projectWorkerSessionInspection(
'project-a',
response({
schemaVersion: 1,
operation: 'worker-session.inspect',
observedAtMs: 1_100,
worker: detailedWorker,
}),
);
assert.equal(inspection.schema, 'qinglong/worker-session-inspection@v1');
assert.equal(inspection.found, true);
assert.equal(inspection.worker.workerId, 'worker-a');
assert.equal(Object.isFrozen(inspection.worker.runtimes), true);
assert.doesNotMatch(
JSON.stringify(inspection),
/transport-request|inspectionId/,
);
assert.match(
formatWorkerSessionInspectionCard(inspection),
/slots available/,
);
const page = projectWorkerSessionList(
'project-a',
response({
schemaVersion: 1,
operation: 'worker-session.list',
observedAtMs: 1_100,
workers: [summaryWorker],
nextCursor: 'worker-a',
}),
);
assert.deepEqual(
{
schema: page.schema,
count: page.count,
nextAfterWorkerId: page.nextAfterWorkerId,
},
{
schema: 'qinglong/worker-session-list@v1',
count: 1,
nextAfterWorkerId: 'worker-a',
},
);
assert.match(formatWorkerSessionListCard(page), /worker-a online/);
assert.throws(() =>
projectWorkerSessionInspection(
'project-a',
response({
schemaVersion: 1,
operation: 'worker-session.list',
observedAtMs: 1_100,
workers: [],
nextCursor: null,
}),
),
);
});
test('requires the canonical Worker endpoint before making a connection', async (t) => {
const directory = fs.realpathSync(
fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-worker-product-client-')),
);
t.after(() => fs.rmSync(directory, { recursive: true, force: true }));
const caFile = privateFile(
directory,
'ca.pem',
fs.readFileSync(path.join(fixtureRoot, 'ca-cert.pem')),
);
const clientCertificateFile = privateFile(
directory,
'client.crt',
fs.readFileSync(path.join(fixtureRoot, 'client-cert.pem')),
);
const clientPrivateKeyFile = privateFile(
directory,
'client.key',
fs.readFileSync(path.join(fixtureRoot, 'client-key.pem')),
);
const assertionFile = privateFile(
directory,
'assertion.jwt',
'eyJhbGciOiJFZERTQSJ9.eyJzdWIiOiJ1In0.c2lnbmF0dXJl',
);
const command = createWorkerSessionListCommand(
'project-a',
undefined,
() => 'inspection-a',
);
for (const [managementPath, expectedConnections] of [
['/api/v3/workers/management', 1],
['/api/v3/worker-credentials/management', 0],
]) {
const configFile = privateFile(
directory,
`client-${expectedConnections}.json`,
JSON.stringify({
schemaVersion: 1,
endpoint: `https://manager.example.test:8443${managementPath}`,
servername: 'manager.example.test',
caFile,
clientCertificateFile,
clientPrivateKeyFile,
requestTimeoutMs: 1_000,
}),
);
let connections = 0;
await assert.rejects(
executeClusterWorkerManagementClient(
{ configFile, assertionFile, command },
{
async connect() {
connections += 1;
throw new Error('stop-after-policy-validation');
},
},
),
);
assert.equal(connections, expectedConnections);
}
});
@@ -0,0 +1,256 @@
'use strict';
const assert = require('node:assert/strict');
const { spawn } = require('node:child_process');
const fs = require('node:fs');
const { createServer } = require('node:https');
const os = require('node:os');
const path = require('node:path');
const test = require('node:test');
const packageRoot = path.resolve(__dirname, '..');
const cliPath = path.join(
packageRoot,
'dist',
'worker-management',
'workerManagementClientCli.js',
);
const fixtureRoot = path.resolve(
packageRoot,
'../ql3-cluster-control/test/fixtures/mtls',
);
function privateFile(directory, name, value) {
const filePath = path.join(directory, name);
fs.writeFileSync(filePath, value, { mode: 0o600 });
return fs.realpathSync(filePath);
}
function runCli(args) {
return new Promise((resolve, reject) => {
const child = spawn(process.execPath, [cliPath, ...args], {
cwd: packageRoot,
stdio: ['ignore', 'pipe', 'pipe'],
});
const stdout = [];
const stderr = [];
child.stdout.on('data', (chunk) => stdout.push(chunk));
child.stderr.on('data', (chunk) => stderr.push(chunk));
child.once('error', reject);
child.once('close', (status, signal) => {
resolve({
status,
signal,
stdout: Buffer.concat(stdout).toString('utf8'),
stderr: Buffer.concat(stderr).toString('utf8'),
});
});
});
}
function summary(workerId) {
return {
workerId,
sessionId: `session-${workerId}`,
generation: 1,
sessionVersion: 1,
lifecycle: 'online',
compatibility: 'default_placement',
architecture: 'arm64',
supportTier: 'tier1',
protocolVersion: '1.0.0',
operatingSystem: 'linux',
maxConcurrentRuns: 1,
availableSlots: 1,
registeredAtMs: 900,
lastHeartbeatAtMs: 1_000,
leaseExpiresAtMs: 2_000,
updatedAtMs: 1_000,
observedAtMs: 1_100,
};
}
test('CLI performs one canonical read per invocation and rejects mutation vocabulary', async (t) => {
const directory = fs.realpathSync(
fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-worker-cli-')),
);
t.after(() => fs.rmSync(directory, { recursive: true, force: true }));
const requests = [];
const server = createServer(
{
key: fs.readFileSync(path.join(fixtureRoot, 'server-key.pem')),
cert: fs.readFileSync(path.join(fixtureRoot, 'server-cert.pem')),
ca: fs.readFileSync(path.join(fixtureRoot, 'ca-cert.pem')),
requestCert: true,
rejectUnauthorized: true,
minVersion: 'TLSv1.3',
maxVersion: 'TLSv1.3',
},
(request, response) => {
const chunks = [];
request.on('data', (chunk) => chunks.push(chunk));
request.on('end', () => {
const command = JSON.parse(Buffer.concat(chunks).toString('utf8'));
requests.push({
method: request.method,
path: request.url,
authorized: request.socket.authorized,
command,
});
const worker = summary('worker-a');
const result =
command.operation === 'worker-session.inspect'
? {
schemaVersion: 1,
operation: 'worker-session.inspect',
observedAtMs: 1_100,
worker: {
...worker,
runtimes: [{ name: 'node', version: '24.18.0' }],
declaredCapacity: {
cpuCores: 1,
memoryBytes: 268_435_456,
diskBytes: 1_073_741_824,
gpuCount: 0,
},
},
}
: {
schemaVersion: 1,
operation: 'worker-session.list',
observedAtMs: 1_100,
workers: [worker],
nextCursor: null,
};
const body = Buffer.from(
JSON.stringify({
schemaVersion: 1,
requestId: 'server-request-hidden',
result,
}),
);
response.writeHead(200, {
'content-type': 'application/json; charset=utf-8',
'content-length': String(body.length),
});
response.end(body);
});
},
);
await new Promise((resolve, reject) => {
server.once('error', reject);
server.listen(0, '127.0.0.1', resolve);
});
t.after(
() =>
new Promise((resolve) => {
server.close(() => resolve());
}),
);
const address = server.address();
assert.notEqual(address, null);
assert.notEqual(typeof address, 'string');
const caFile = privateFile(
directory,
'ca.pem',
fs.readFileSync(path.join(fixtureRoot, 'ca-cert.pem')),
);
const clientCertificateFile = privateFile(
directory,
'client.crt',
fs.readFileSync(path.join(fixtureRoot, 'client-cert.pem')),
);
const clientPrivateKeyFile = privateFile(
directory,
'client.key',
fs.readFileSync(path.join(fixtureRoot, 'client-key.pem')),
);
const configFile = privateFile(
directory,
'client.json',
JSON.stringify({
schemaVersion: 1,
endpoint: `https://localhost:${address.port}/api/v3/workers/management`,
servername: 'localhost',
caFile,
clientCertificateFile,
clientPrivateKeyFile,
requestTimeoutMs: 2_000,
}),
);
const assertionFile = privateFile(
directory,
'assertion.jwt',
'eyJhbGciOiJFZERTQSJ9.eyJzdWIiOiJ1In0.c2lnbmF0dXJl',
);
const inspect = await runCli([
'inspect',
`--config=${configFile}`,
`--assertion=${assertionFile}`,
'--project=project-a',
'--worker=worker-a',
'--format=json',
]);
assert.equal(inspect.status, 0, inspect.stderr);
assert.equal(inspect.stderr, '');
const inspection = JSON.parse(inspect.stdout);
assert.equal(inspection.schema, 'qinglong/worker-session-inspection@v1');
assert.equal(inspection.worker.workerId, 'worker-a');
assert.equal(inspect.stdout.includes('server-request-hidden'), false);
const list = await runCli([
'list',
`--config=${configFile}`,
`--assertion=${assertionFile}`,
'--project=project-a',
'--format=json',
]);
assert.equal(list.status, 0, list.stderr);
assert.equal(JSON.parse(list.stdout).count, 1);
assert.equal(requests.length, 2);
assert.deepEqual(
requests.map(({ method, path, authorized, command }) => ({
method,
path,
authorized,
operation: command.operation,
})),
[
{
method: 'POST',
path: '/api/v3/workers/management',
authorized: true,
operation: 'worker-session.inspect',
},
{
method: 'POST',
path: '/api/v3/workers/management',
authorized: true,
operation: 'worker-session.list',
},
],
);
const rejected = await runCli([
'inspect',
`--config=${configFile}`,
`--assertion=${assertionFile}`,
'--project=project-a',
'--worker=worker-a',
'--command=/private/mutation.json',
]);
assert.equal(rejected.status, 64);
assert.equal(requests.length, 2);
assert.equal(rejected.stdout, '');
assert.equal(JSON.parse(rejected.stderr).event, 'usage_invalid');
});
test('CLI help states its bounded read contract', async () => {
const help = await runCli(['--help']);
assert.equal(help.status, 0);
assert.match(help.stdout, /ql3-worker-client inspect/);
assert.match(help.stdout, /ql3-worker-client list/);
assert.match(help.stdout, /never retries, polls or auto-pages/);
assert.doesNotMatch(help.stdout, /credential|secret|token/i);
});