feat(ql3): expose worker observations in console

This commit is contained in:
whyour
2026-08-20 14:02:01 +08:00
parent af5d5bfc0b
commit 344680d64a
25 changed files with 840 additions and 49 deletions
@@ -8,8 +8,9 @@
run_cancellation_status: '/api/v1/run-management/cancellation-status',
run_cancellation_blocked_list:
'/api/v1/run-management/blocked-cancellations',
run_cancellation_inspect:
'/api/v1/run-management/cancellation-inspect',
run_cancellation_inspect: '/api/v1/run-management/cancellation-inspect',
worker_list: '/api/v1/worker-management/workers',
worker_inspect: '/api/v1/worker-management/worker',
run_list: '/api/v1/observe/run-list',
run_read: '/api/v1/observe/run',
run_event_list: '/api/v1/observe/run-events',
@@ -28,6 +29,8 @@
run_cancellation_status: '取消可用性',
run_cancellation_blocked_list: 'Blocked Cancellations',
run_cancellation_inspect: '取消诊断',
worker_list: 'Worker 目录',
worker_inspect: 'Worker 详情',
run_list: 'Run 目录',
run_read: 'Run 详情',
run_event_list: 'Run Events',
@@ -124,6 +127,10 @@
result.cursor = null;
} else if (operation === 'run_cancellation_inspect') {
result.runId = value('cancellation-run-id');
} else if (operation === 'worker_list') {
result.afterWorkerId = null;
} else if (operation === 'worker_inspect') {
result.workerId = value('worker-id');
} else if (operation === 'run_list') {
result.afterCreatedAtMs = null;
result.afterRunId = null;
@@ -176,6 +183,11 @@
typeof fact.nextCursor === 'string'
) {
next.cursor = fact.nextCursor;
} else if (
operation === 'worker_list' &&
typeof fact.nextAfterWorkerId === 'string'
) {
next.afterWorkerId = fact.nextAfterWorkerId;
} else if (operation === 'run_list' && fact.hasMore === true && fact.next) {
next.afterCreatedAtMs = fact.next.createdAtMs;
next.afterRunId = fact.next.runId;
@@ -234,6 +246,20 @@
entry.append(button);
return;
}
if (operation === 'worker_list' && Array.isArray(fact.workers)) {
fact.workers.forEach(function (worker) {
if (!worker || typeof worker.workerId !== 'string') return;
const button = document.createElement('button');
button.type = 'button';
button.textContent = '显式检查 ' + worker.workerId;
button.addEventListener('click', function () {
document.getElementById('worker-id').value = worker.workerId;
void execute('worker_inspect');
});
entry.append(button);
});
return;
}
if (
operation !== 'run_cancellation_blocked_list' ||
!Array.isArray(fact.items)
@@ -31,6 +31,8 @@
'run_cancellation_status',
'run_cancellation_blocked_list',
'run_cancellation_inspect',
'worker_list',
'worker_inspect',
'run_list',
'run_read',
'run_event_list',
@@ -50,6 +52,8 @@
run_cancellation_status: ['projectId', 'requestId'],
run_cancellation_blocked_list: ['cursor', 'projectId', 'requestId'],
run_cancellation_inspect: ['projectId', 'requestId', 'runId'],
worker_list: ['afterWorkerId', 'projectId', 'requestId'],
worker_inspect: ['projectId', 'requestId', 'workerId'],
run_list: [
'afterCreatedAtMs',
'afterRunId',
@@ -117,6 +121,7 @@
afterStepKey: 'step',
afterStepRunId: 'step',
afterTaskId: 'task',
afterWorkerId: 'worker',
artifactId: 'artifact',
attemptId: 'attempt',
contentDigest: 'digest',
@@ -125,6 +130,7 @@
executionId: 'execution',
id: 'identifier',
modelId: 'model',
nextAfterWorkerId: 'worker',
outputRef: 'artifact',
packageName: 'package',
projectId: 'project',
@@ -160,6 +166,10 @@
'target',
'task',
'tasks',
'declaredCapacity',
'runtimes',
'worker',
'workers',
'usage',
'workflow',
'workflows',
@@ -170,6 +180,7 @@
'available',
'cancelRequested',
'enabled',
'found',
'hasMore',
'outputAvailable',
'ready',
@@ -183,6 +194,9 @@
'assessment',
'cancelReason',
'kind',
'architecture',
'compatibility',
'lifecycle',
'lastResult',
'operation',
'operatorAction',
@@ -191,6 +205,8 @@
'severity',
'stage',
'status',
'supportTier',
'operatingSystem',
]);
const safeEnumValues = new Set([
'accepted',
@@ -198,6 +214,13 @@
'admission',
'attention_required',
'available',
'amd64',
'arm64',
'ppc64le',
's390x',
'arm/v7',
'arm/v6',
'386',
'blocked',
'cancelled',
'completed',
@@ -208,6 +231,22 @@
'dispatch',
'dispatching',
'disabled',
'default_placement',
'explicit_placement_required',
'protocol_incompatible',
'online',
'draining',
'offline',
'lease_expired',
'tier1',
'candidate',
'experimental',
'legacy-only',
'linux',
'darwin',
'win32',
'freebsd',
'aix',
'enabled',
'execution',
'failed',
@@ -269,7 +308,7 @@
const freeTextKey =
/text|content|stdout|stderr|command|input|output|environment|reason|error|message|description|name|path|url|uri|host|endpoint/iu;
const numericKey =
/^(?:schemaVersion|version|revision|sequence|attempt|priority|limit|offset|size|total|count|exitCode|pending|leased|retryWait|dispatched|blocked|due|expiredLease|identityMismatch|pidMismatch|unsupported|invalid|[A-Za-z0-9_]*(?:AtMs|TimeMs|DurationMs|Bytes|Tokens|Micros|Sequence|Version|Count|Limit|Offset|Size|Total))$/u;
/^(?:schemaVersion|version|revision|sequence|attempt|priority|limit|offset|size|total|count|exitCode|pending|leased|retryWait|dispatched|blocked|due|expiredLease|identityMismatch|pidMismatch|unsupported|invalid|availableSlots|maxConcurrentRuns|cpuCores|[A-Za-z0-9_]*(?:AtMs|TimeMs|DurationMs|Bytes|Tokens|Micros|Sequence|Version|Count|Limit|Offset|Size|Total))$/u;
const schemaValue = /^[a-z0-9][a-z0-9./_-]{0,126}@[a-z0-9._-]{1,16}$/u;
class ClusterConsoleEvidenceBundleError extends TypeError {
@@ -24,7 +24,7 @@
<div class="boundary" aria-label="当前权限边界">
<span class="boundary-dot" aria-hidden="true"></span>
<span>本机只读 BFF</span>
<strong>Run · Task · Workflow · Copilot · Optional management</strong>
<strong>Run · Task · Workflow · Worker · Copilot</strong>
</div>
</header>
@@ -52,6 +52,7 @@
<nav class="mode-tabs" aria-label="观察面">
<button type="button" class="mode-tab active" data-panel="runtime-panel" aria-pressed="true">运行态</button>
<button type="button" class="mode-tab" data-panel="management-panel" aria-pressed="false">取消可用性</button>
<button type="button" class="mode-tab" data-panel="worker-panel" aria-pressed="false">Worker</button>
<button type="button" class="mode-tab" data-panel="workflow-panel" aria-pressed="false">工作流</button>
<button type="button" class="mode-tab" data-panel="copilot-panel" aria-pressed="false">Copilot</button>
</nav>
@@ -80,6 +81,20 @@
</div>
</section>
<section id="worker-panel" class="mode-panel" hidden>
<div class="control-group">
<div class="control-title"><span>01</span><strong>Worker 目录</strong></div>
<button type="button" class="primary" data-read="worker_list">读取首屏 Workers</button>
<p class="field-note">只有启动进程显式提供独立 Worker management 配置与短期 assertion 时可用;固定 16 项,下一页必须再次点击。</p>
</div>
<div class="control-group">
<div class="control-title"><span>02</span><strong>单 Worker 状态</strong></div>
<input id="worker-id" type="text" maxlength="128" autocomplete="off" spellcheck="false" placeholder="worker-id" />
<button type="button" data-read="worker_inspect">读取 Worker 详情</button>
<p class="field-note">只投影在线状态、兼容性、架构与容量;没有 credential、drain、revoke 或调度 mutation。</p>
</div>
</section>
<section id="management-panel" class="mode-panel" hidden>
<div class="control-group">
<div class="control-title"><span>01</span><strong>Project 状态</strong></div>
@@ -130,7 +145,7 @@
<aside class="trust-note">
<span>Authority boundary</span>
<p>Project credential可选 Run management authority 只由本机进程从彼此独立的私有文件读取。浏览器无法提交任意路径,也没有 start、stop、retry、rearm 或 diagnose 权限入口。脱敏导出只处理本页已读事实,不补读或上传。</p>
<p>Project credential可选 Run authority 与可选 Worker authority 只由本机进程从彼此独立的私有文件读取。浏览器无法提交任意路径,也没有 start、stop、retry、rearm、drain、revoke 或 diagnose 权限入口。脱敏导出只处理本页已读事实,不补读或上传。</p>
</aside>
</aside>
@@ -161,7 +176,7 @@
<footer>
<span>Loopback only · explicit reads · zero polling</span>
<span>QingLong 3.0 incubation / D-369</span>
<span>QingLong 3.0 incubation / D-375</span>
</footer>
</div>
</body>
@@ -24,7 +24,7 @@ const ASSETS = Object.freeze([
name: 'index.html',
field: 'html',
maximumBytes: 32 * 1024,
digest: 'a5a3d46a8493a27b53bd4a253ef38ebaf00d204a1454f4b47a1f1ceff668855f',
digest: '363fcf2d52ff86e5b2a1c9ed8b7810226920a9270b6d0c41a1f61ce24f957832',
}),
Object.freeze({
name: 'app.css',
@@ -36,13 +36,13 @@ const ASSETS = Object.freeze([
name: 'evidence-bundle.js',
field: 'evidenceBundle',
maximumBytes: 32 * 1024,
digest: '739ff786b651de23876fc5f4df5073e211085dfdfa1d2ecb79f53d5c871c6c1d',
digest: 'ae4a08572cfc3296284c56549a3850f530474151573e2705401996730bf0466e',
}),
Object.freeze({
name: 'app.js',
field: 'javascript',
maximumBytes: 32 * 1024,
digest: '7ed994d8f2f5b151a247c5dec1d2841d45d30ff05b14dd1f41c12c5582acf9e6',
digest: '4b13da1a85d59e29a606da3b1a3327419926a496a498a06ddc323365ce5230c1',
}),
] as const);
@@ -24,6 +24,13 @@ import {
projectRunCancellationStatus,
} from '../run-management/runCancellationStatus';
import { executeClusterRunManagementCommand } from '../run-management/runManagementClient';
import { executeClusterWorkerManagementClient } from '../worker-management/workerManagementClient';
import {
createWorkerSessionInspectionCommand,
createWorkerSessionListCommand,
projectWorkerSessionInspection,
projectWorkerSessionList,
} from '../worker-management/workerManagementProduct';
import { loadClusterCopilotConsoleAssets } from './assets';
import {
CLUSTER_COPILOT_CONSOLE_READ_OPERATIONS,
@@ -42,6 +49,7 @@ const USAGE = [
' ql3-copilot-console --check --config /absolute/client.json --credential /absolute/credential --session /absolute/session',
' ql3-copilot-console --container-published-loopback --port=1024..65535 --config /absolute/client.json --credential /absolute/credential --session /absolute/session [--check]',
' Optional Run reads: --run-management-config /absolute/run-client.json --run-management-assertion /absolute/assertion.jwt',
' Optional Worker reads: --worker-management-config /absolute/worker-client.json --worker-management-assertion /absolute/assertion.jwt',
'',
'Native mode binds 127.0.0.1. Container mode requires host-loopback port publication.',
'The browser session key remains in a separate owner-private 0600 file.',
@@ -54,6 +62,8 @@ interface ClusterCopilotConsoleCliArguments {
readonly networkBoundary: 'host-loopback' | 'container-published-loopback';
readonly runManagementAssertionFile?: string;
readonly runManagementConfigFile?: string;
readonly workerManagementAssertionFile?: string;
readonly workerManagementConfigFile?: string;
readonly sessionFile: string;
readonly port: number;
}
@@ -67,6 +77,7 @@ const RUN_MANAGEMENT_OPERATIONS = new Set([
'run_cancellation_blocked_list',
'run_cancellation_inspect',
]);
const WORKER_MANAGEMENT_OPERATIONS = new Set(['worker_list', 'worker_inspect']);
function usageFailure(): never {
process.stderr.write(USAGE + '\n');
@@ -105,6 +116,8 @@ export function parseClusterCopilotConsoleCliArguments(
let sessionFile: string | undefined;
let runManagementConfigFile: string | undefined;
let runManagementAssertionFile: string | undefined;
let workerManagementConfigFile: string | undefined;
let workerManagementAssertionFile: string | undefined;
let port = 0;
let portSeen = false;
let containerPublishedLoopback = false;
@@ -166,6 +179,28 @@ export function parseClusterCopilotConsoleCliArguments(
index += runManagementAssertion.consumed;
continue;
}
const workerManagementConfig = argumentValue(
argv,
index,
'--worker-management-config',
);
if (workerManagementConfig) {
if (workerManagementConfigFile !== undefined) return usageFailure();
workerManagementConfigFile = workerManagementConfig.value;
index += workerManagementConfig.consumed;
continue;
}
const workerManagementAssertion = argumentValue(
argv,
index,
'--worker-management-assertion',
);
if (workerManagementAssertion) {
if (workerManagementAssertionFile !== undefined) return usageFailure();
workerManagementAssertionFile = workerManagementAssertion.value;
index += workerManagementAssertion.consumed;
continue;
}
const portArgument = argumentValue(argv, index, '--port');
if (portArgument) {
if (portSeen || !/^(?:0|[1-9][0-9]{0,4})$/.test(portArgument.value)) {
@@ -190,6 +225,8 @@ export function parseClusterCopilotConsoleCliArguments(
sessionFile === undefined ||
(runManagementConfigFile === undefined) !==
(runManagementAssertionFile === undefined) ||
(workerManagementConfigFile === undefined) !==
(workerManagementAssertionFile === undefined) ||
(containerPublishedLoopback && port === 0) ||
(!containerPublishedLoopback && check && port !== 0)
) {
@@ -206,28 +243,26 @@ export function parseClusterCopilotConsoleCliArguments(
runManagementAssertionFile !== undefined
? { runManagementConfigFile, runManagementAssertionFile }
: {}),
...(workerManagementConfigFile !== undefined &&
workerManagementAssertionFile !== undefined
? { workerManagementConfigFile, workerManagementAssertionFile }
: {}),
sessionFile,
port,
});
}
function validateRunManagementAuthority(
parsed: Readonly<ClusterCopilotConsoleCliArguments>,
function validateManagementAuthority(
configFile: string | undefined,
assertionFile: string | undefined,
kind: 'run' | 'worker',
): boolean {
if (
parsed.runManagementConfigFile === undefined ||
parsed.runManagementAssertionFile === undefined
) {
return false;
}
validateClusterAuthenticatedManagementClientConfiguration(
parsed.runManagementConfigFile,
'run',
);
if (configFile === undefined || assertionFile === undefined) return false;
validateClusterAuthenticatedManagementClientConfiguration(configFile, kind);
let bytes: Buffer | undefined;
try {
bytes = readCanonicalFile(
parsed.runManagementAssertionFile,
assertionFile,
MAXIMUM_MANAGEMENT_ASSERTION_BYTES,
'private',
);
@@ -235,7 +270,7 @@ function validateRunManagementAuthority(
bytes.some((byte) => byte > 0x7f) ||
!MANAGEMENT_ASSERTION.test(bytes.toString('ascii'))
) {
throw new Error('invalid Run management assertion');
throw new Error('invalid management assertion');
}
return true;
} finally {
@@ -243,12 +278,16 @@ function validateRunManagementAuthority(
}
}
function availableOperations(runManagementAuthority: boolean) {
return runManagementAuthority
? CLUSTER_COPILOT_CONSOLE_READ_OPERATIONS
: CLUSTER_COPILOT_CONSOLE_READ_OPERATIONS.filter(
(operation) => !RUN_MANAGEMENT_OPERATIONS.has(operation),
);
function availableOperations(
runManagementAuthority: boolean,
workerManagementAuthority: boolean,
) {
return CLUSTER_COPILOT_CONSOLE_READ_OPERATIONS.filter(
(operation) =>
(runManagementAuthority || !RUN_MANAGEMENT_OPERATIONS.has(operation)) &&
(workerManagementAuthority ||
!WORKER_MANAGEMENT_OPERATIONS.has(operation)),
);
}
function commandIdSource(requestId: string): () => string {
@@ -312,7 +351,45 @@ async function executeConsoleRead(
: projectRunCancellationInspection(result);
return Object.freeze({
schemaVersion: 1 as const,
requestId: result.requestId,
requestId: request.requestId,
result: projected as unknown as Readonly<Record<string, unknown>>,
});
}
if (
request.operation === 'worker_list' ||
request.operation === 'worker_inspect'
) {
if (
parsed.workerManagementConfigFile === undefined ||
parsed.workerManagementAssertionFile === undefined
) {
throw new Error('Worker management authority is disabled');
}
const createUuid = commandIdSource(request.requestId);
const command =
request.operation === 'worker_list'
? createWorkerSessionListCommand(
request.projectId,
request.afterWorkerId ?? undefined,
createUuid,
)
: createWorkerSessionInspectionCommand(
request.projectId,
request.workerId,
createUuid,
);
const result = await executeClusterWorkerManagementClient({
configFile: parsed.workerManagementConfigFile,
assertionFile: parsed.workerManagementAssertionFile,
command,
});
const projected =
request.operation === 'worker_list'
? projectWorkerSessionList(request.projectId, result)
: projectWorkerSessionInspection(request.projectId, result);
return Object.freeze({
schemaVersion: 1 as const,
requestId: request.requestId,
result: projected as unknown as Readonly<Record<string, unknown>>,
});
}
@@ -352,8 +429,20 @@ async function main(): Promise<void> {
const assets = loadClusterCopilotConsoleAssets(__dirname);
validateClusterCopilotClientConfiguration(parsed.configFile);
validateClusterCopilotClientCredentialFile(parsed.credentialFile);
const runManagementAuthority = validateRunManagementAuthority(parsed);
const operations = availableOperations(runManagementAuthority);
const runManagementAuthority = validateManagementAuthority(
parsed.runManagementConfigFile,
parsed.runManagementAssertionFile,
'run',
);
const workerManagementAuthority = validateManagementAuthority(
parsed.workerManagementConfigFile,
parsed.workerManagementAssertionFile,
'worker',
);
const operations = availableOperations(
runManagementAuthority,
workerManagementAuthority,
);
const sessionDigest = readSessionDigest(parsed.sessionFile);
if (parsed.check) {
try {
@@ -373,6 +462,9 @@ async function main(): Promise<void> {
runManagementAuthority: runManagementAuthority
? 'server_only'
: 'disabled',
workerManagementAuthority: workerManagementAuthority
? 'server_only'
: 'disabled',
operations,
mutation: false,
}) + '\n',
@@ -409,6 +501,9 @@ async function main(): Promise<void> {
runManagementAuthority: runManagementAuthority
? 'server_only'
: 'disabled',
workerManagementAuthority: workerManagementAuthority
? 'server_only'
: 'disabled',
operations,
mutation: false,
}) + '\n',
@@ -14,6 +14,8 @@ export const CLUSTER_COPILOT_CONSOLE_READ_OPERATIONS = Object.freeze([
'run_cancellation_status',
'run_cancellation_blocked_list',
'run_cancellation_inspect',
'worker_list',
'worker_inspect',
'run_list',
'run_read',
'run_event_list',
@@ -46,6 +48,9 @@ export type ClusterCopilotConsoleReadRequest =
| (BaseReadRequest<'run_cancellation_blocked_list'> &
Readonly<{ cursor: string | null }>)
| (BaseReadRequest<'run_cancellation_inspect'> & Readonly<{ runId: string }>)
| (BaseReadRequest<'worker_list'> &
Readonly<{ afterWorkerId: string | null }>)
| (BaseReadRequest<'worker_inspect'> & Readonly<{ workerId: string }>)
| (BaseReadRequest<'run_list'> &
Readonly<{
afterCreatedAtMs: number | null;
@@ -243,6 +248,25 @@ export function normalizeClusterCopilotConsoleReadRequest(
runId: record.runId,
});
}
if (op === 'worker_list') {
exact(record, op, ['afterWorkerId']);
if (record.afterWorkerId !== null && !identifier(record.afterWorkerId))
invalid();
return Object.freeze({
...common(record),
operation: op,
afterWorkerId: record.afterWorkerId as string | null,
});
}
if (op === 'worker_inspect') {
exact(record, op, ['workerId']);
if (!identifier(record.workerId)) invalid();
return Object.freeze({
...common(record),
operation: op,
workerId: record.workerId,
});
}
if (op === 'run_list') {
exact(record, op, ['afterCreatedAtMs', 'afterRunId', 'limit']);
if (
@@ -476,7 +500,9 @@ export function clusterCopilotConsoleProjectReadPath(
normalized.operation === 'output' ||
normalized.operation === 'run_cancellation_status' ||
normalized.operation === 'run_cancellation_blocked_list' ||
normalized.operation === 'run_cancellation_inspect'
normalized.operation === 'run_cancellation_inspect' ||
normalized.operation === 'worker_list' ||
normalized.operation === 'worker_inspect'
)
invalid();
const project = '/api/v3/projects/' + encoded(normalized.projectId);
@@ -33,6 +33,8 @@ const OPERATIONS = Object.freeze([
'run_cancellation_status',
'run_cancellation_blocked_list',
'run_cancellation_inspect',
'worker_list',
'worker_inspect',
'run_list',
'run_read',
'run_event_list',
@@ -62,6 +64,8 @@ const REQUEST_FIELDS: Readonly<Record<EvidenceOperation, readonly string[]>> =
'requestId',
'runId',
]),
worker_list: Object.freeze(['afterWorkerId', 'projectId', 'requestId']),
worker_inspect: Object.freeze(['projectId', 'requestId', 'workerId']),
run_list: Object.freeze([
'afterCreatedAtMs',
'afterRunId',
@@ -134,6 +138,7 @@ const IDENTIFIER_DOMAINS: Readonly<Record<string, string>> = Object.freeze({
afterStepKey: 'step',
afterStepRunId: 'step',
afterTaskId: 'task',
afterWorkerId: 'worker',
artifactId: 'artifact',
attemptId: 'attempt',
contentDigest: 'digest',
@@ -142,6 +147,7 @@ const IDENTIFIER_DOMAINS: Readonly<Record<string, string>> = Object.freeze({
executionId: 'execution',
id: 'identifier',
modelId: 'model',
nextAfterWorkerId: 'worker',
outputRef: 'artifact',
packageName: 'package',
projectId: 'project',
@@ -177,6 +183,10 @@ const SAFE_CONTAINERS = new Set([
'target',
'task',
'tasks',
'declaredCapacity',
'runtimes',
'worker',
'workers',
'usage',
'workflow',
'workflows',
@@ -187,6 +197,7 @@ const SAFE_BOOLEANS = new Set([
'available',
'cancelRequested',
'enabled',
'found',
'hasMore',
'outputAvailable',
'ready',
@@ -199,8 +210,11 @@ const SAFE_ENUM_KEYS = new Set([
'assessment',
'cancelReason',
'finishReason',
'architecture',
'compatibility',
'kind',
'lastResult',
'lifecycle',
'operation',
'operatorAction',
'outcome',
@@ -208,6 +222,8 @@ const SAFE_ENUM_KEYS = new Set([
'severity',
'stage',
'status',
'supportTier',
'operatingSystem',
]);
const SAFE_ENUM_VALUES = new Set([
'accepted',
@@ -215,6 +231,13 @@ const SAFE_ENUM_VALUES = new Set([
'admission',
'attention_required',
'available',
'amd64',
'arm64',
'ppc64le',
's390x',
'arm/v7',
'arm/v6',
'386',
'blocked',
'cancelled',
'completed',
@@ -225,6 +248,22 @@ const SAFE_ENUM_VALUES = new Set([
'dispatch',
'dispatching',
'disabled',
'default_placement',
'explicit_placement_required',
'protocol_incompatible',
'online',
'draining',
'offline',
'lease_expired',
'tier1',
'candidate',
'experimental',
'legacy-only',
'linux',
'darwin',
'win32',
'freebsd',
'aix',
'enabled',
'execution',
'failed',
@@ -282,7 +321,7 @@ const SAFE_ENUM_VALUES = new Set([
'workflow',
]);
const NUMERIC_KEY =
/^(?:schemaVersion|version|revision|sequence|attempt|priority|limit|offset|size|total|count|exitCode|pending|leased|retryWait|dispatched|blocked|due|expiredLease|identityMismatch|pidMismatch|unsupported|invalid|[A-Za-z0-9_]*(?:AtMs|TimeMs|DurationMs|Bytes|Tokens|Micros|Sequence|Version|Count|Limit|Offset|Size|Total))$/u;
/^(?:schemaVersion|version|revision|sequence|attempt|priority|limit|offset|size|total|count|exitCode|pending|leased|retryWait|dispatched|blocked|due|expiredLease|identityMismatch|pidMismatch|unsupported|invalid|availableSlots|maxConcurrentRuns|cpuCores|[A-Za-z0-9_]*(?:AtMs|TimeMs|DurationMs|Bytes|Tokens|Micros|Sequence|Version|Count|Limit|Offset|Size|Total))$/u;
const SCHEMA_VALUE = /^[a-z0-9][a-z0-9./_-]{0,126}@[a-z0-9._-]{1,16}$/u;
const SHA256 = /^[0-9a-f]{64}$/u;
const CONTROL = /[\0-\x1f\x7f]/u;
@@ -195,6 +195,8 @@ const READ_ROUTES: Readonly<
'/api/v1/run-management/blocked-cancellations':
'run_cancellation_blocked_list',
'/api/v1/run-management/cancellation-inspect': 'run_cancellation_inspect',
'/api/v1/worker-management/workers': 'worker_list',
'/api/v1/worker-management/worker': 'worker_inspect',
'/api/v1/observe/run-list': 'run_list',
'/api/v1/observe/run': 'run_read',
'/api/v1/observe/run-events': 'run_event_list',
@@ -248,6 +248,23 @@ test('normalizes Copilot and fixed Project observation operations without arbitr
}),
{ code: 'QL3_CLUSTER_COPILOT_CONSOLE_READ_REQUEST_INVALID' },
);
const workerList = normalizeClusterCopilotConsoleReadRequest({
schema: CLUSTER_COPILOT_CONSOLE_READ_REQUEST_SCHEMA,
operation: 'worker_list',
projectId: 'project-main',
requestId: 'console-worker-list',
afterWorkerId: 'worker-16',
});
assert.deepEqual(workerList, {
schema: CLUSTER_COPILOT_CONSOLE_READ_REQUEST_SCHEMA,
operation: 'worker_list',
projectId: 'project-main',
requestId: 'console-worker-list',
afterWorkerId: 'worker-16',
});
assert.throws(() => clusterCopilotConsoleProjectReadPath(workerList), {
code: 'QL3_CLUSTER_COPILOT_CONSOLE_READ_REQUEST_INVALID',
});
assert.throws(
() =>
normalizeClusterCopilotConsoleReadRequest({
@@ -617,6 +634,62 @@ test('routes only the three fixed Run management reads and validates their curso
assert.equal(reads.length, 3);
});
test('routes only fixed Worker list and inspect reads with a bounded cursor', async (t) => {
const reads = [];
const { server, headers } = await fixture(async (read) => {
reads.push(read);
return {
schemaVersion: 1,
requestId: read.requestId,
result: { operation: read.operation },
};
});
t.after(() => server.close());
const cases = [
[
'/api/v1/worker-management/workers',
{
schema: CLUSTER_COPILOT_CONSOLE_READ_REQUEST_SCHEMA,
operation: 'worker_list',
projectId: 'project-main',
requestId: 'console-worker-list-1',
afterWorkerId: null,
},
],
[
'/api/v1/worker-management/worker',
{
schema: CLUSTER_COPILOT_CONSOLE_READ_REQUEST_SCHEMA,
operation: 'worker_inspect',
projectId: 'project-main',
requestId: 'console-worker-inspect-1',
workerId: 'worker-a',
},
],
];
for (const [path, body] of cases) {
const response = await request(server.origin, {
method: 'POST',
path,
headers,
body,
});
assert.equal(response.statusCode, 200);
}
assert.deepEqual(
reads,
cases.map(([, body]) => body),
);
const invalidCursor = await request(server.origin, {
method: 'POST',
path: '/api/v1/worker-management/workers',
headers,
body: { ...cases[0][1], afterWorkerId: 'contains space' },
});
assert.equal(invalidCursor.statusCode, 400);
assert.equal(reads.length, 2);
});
test('returns model text as JSON data only after an explicit output read', async (t) => {
const { server, headers } = await fixture(async (command) => {
assert.equal(command.operation, 'output');
@@ -39,6 +39,29 @@ const runManagementOperations = [
'run_cancellation_blocked_list',
'run_cancellation_inspect',
];
const workerManagementOperations = ['worker_list', 'worker_inspect'];
function workerSummary(workerId = 'worker-a') {
return {
workerId,
sessionId: 'session-a',
generation: 2,
sessionVersion: 5,
lifecycle: 'online',
compatibility: 'default_placement',
architecture: 'arm64',
supportTier: 'tier1',
protocolVersion: '1.0.0',
operatingSystem: 'linux',
maxConcurrentRuns: 2,
availableSlots: 1,
registeredAtMs: 900,
lastHeartbeatAtMs: 1_050,
leaseExpiresAtMs: 2_000,
updatedAtMs: 1_050,
observedAtMs: 1_100,
};
}
function privateFile(directory, name, contents) {
const filePath = path.join(directory, name);
@@ -156,6 +179,9 @@ async function fixture(t) {
{
key: fs.readFileSync(path.join(tlsFixture, 'server-key.pem')),
cert: fs.readFileSync(path.join(tlsFixture, 'server-cert.pem')),
ca: fs.readFileSync(path.join(tlsFixture, 'ca-cert.pem')),
requestCert: true,
rejectUnauthorized: false,
minVersion: 'TLSv1.3',
maxVersion: 'TLSv1.3',
},
@@ -175,7 +201,39 @@ async function fixture(t) {
command,
});
const body =
command?.operation === 'run.cancellation.summary'
command?.operation === 'worker-session.list'
? {
schemaVersion: 1,
requestId: 'worker-transport-hidden',
result: {
schemaVersion: 1,
operation: 'worker-session.list',
observedAtMs: 1_100,
workers: [workerSummary()],
nextCursor: 'worker-a',
},
}
: command?.operation === 'worker-session.inspect'
? {
schemaVersion: 1,
requestId: 'worker-transport-hidden',
result: {
schemaVersion: 1,
operation: 'worker-session.inspect',
observedAtMs: 1_100,
worker: {
...workerSummary(),
runtimes: [{ name: 'node', version: '24.18.0' }],
declaredCapacity: {
cpuCores: 1,
memoryBytes: 268_435_456,
diskBytes: 1_073_741_824,
gpuCount: 0,
},
},
},
}
: command?.operation === 'run.cancellation.summary'
? {
schemaVersion: 1,
requestId: command.request.requestId,
@@ -268,6 +326,21 @@ async function fixture(t) {
requestTimeoutMs: 2_000,
}),
);
const workerManagementConfigFile = privateFile(
directory,
'worker-client.json',
JSON.stringify({
schemaVersion: 1,
endpoint: `https://localhost:${
server.address().port
}/api/v3/workers/management`,
servername: 'localhost',
caFile,
clientCertificateFile,
clientPrivateKeyFile,
requestTimeoutMs: 2_000,
}),
);
const sessionToken = randomBytes(32).toString('base64url');
return {
requests,
@@ -281,6 +354,12 @@ async function fixture(t) {
'run-assertion.jwt',
'eyJhbGciOiJFZERTQSJ9.eyJzdWIiOiJvcGVyYXRvci0xIn0.c2lnbmF0dXJl',
),
workerManagementConfigFile,
workerManagementAssertionFile: privateFile(
directory,
'worker-assertion.jwt',
'eyJhbGciOiJFZERTQSJ9.eyJzdWIiOiJvcGVyYXRvci0xIn0.c2lnbmF0dXJl',
),
};
}
@@ -291,6 +370,7 @@ test('CLI exposes deterministic help and a low-sensitive failure surface', async
' ql3-copilot-console --check --config /absolute/client.json --credential /absolute/credential --session /absolute/session',
' ql3-copilot-console --container-published-loopback --port=1024..65535 --config /absolute/client.json --credential /absolute/credential --session /absolute/session [--check]',
' Optional Run reads: --run-management-config /absolute/run-client.json --run-management-assertion /absolute/assertion.jwt',
' Optional Worker reads: --worker-management-config /absolute/worker-client.json --worker-management-assertion /absolute/assertion.jwt',
'',
'Native mode binds 127.0.0.1. Container mode requires host-loopback port publication.',
'The browser session key remains in a separate owner-private 0600 file.',
@@ -345,6 +425,7 @@ test('preflight proves private authority and unauthenticated TLS 1.3 readiness',
browserCredential: 'forbidden',
clusterCredential: 'server_only',
runManagementAuthority: 'disabled',
workerManagementAuthority: 'disabled',
operations: consoleOperations,
mutation: false,
});
@@ -400,6 +481,46 @@ test('preflight enables exactly three optional Run management reads only when bo
assert.doesNotMatch(incomplete.stderr, /ql3-copilot-console-cli-/);
});
test('preflight enables exactly two Worker reads only with canonical config and assertion', async (t) => {
const value = await fixture(t);
const result = await runCli([
'--check',
'--config',
value.configFile,
'--credential',
value.credentialFile,
'--session',
value.sessionFile,
'--worker-management-config',
value.workerManagementConfigFile,
'--worker-management-assertion',
value.workerManagementAssertionFile,
]);
assert.equal(result.status, 0, result.stderr);
const fact = JSON.parse(result.stdout);
assert.equal(fact.runManagementAuthority, 'disabled');
assert.equal(fact.workerManagementAuthority, 'server_only');
assert.deepEqual(fact.operations, [
'inspect',
'output',
...workerManagementOperations,
...consoleOperations.slice(2),
]);
assert.equal(value.requests.length, 1);
const incomplete = await runCli([
'--config',
value.configFile,
'--credential',
value.credentialFile,
'--session',
value.sessionFile,
'--worker-management-config',
value.workerManagementConfigFile,
]);
assert.equal(incomplete.status, 64);
});
test('serve mode starts an ephemeral loopback origin and shuts down cleanly', async (t) => {
const value = await fixture(t);
const child = spawn(
@@ -426,6 +547,7 @@ test('serve mode starts an ephemeral loopback origin and shuts down cleanly', as
assert.deepEqual(started.operations, consoleOperations);
assert.equal(started.mutation, false);
assert.equal(started.runManagementAuthority, 'disabled');
assert.equal(started.workerManagementAuthority, 'disabled');
assert.equal(started.networkBoundary, 'host-loopback');
assert.equal(started.publishedHostAddress, '127.0.0.1');
const shell = await get(started.origin);
@@ -499,6 +621,72 @@ test('serve mode forwards one explicit status click through the optional mTLS Ru
assert.deepEqual(exit, { status: 0, signal: null });
});
test('serve mode performs one canonical Worker page read without exposing transport identity', async (t) => {
const value = await fixture(t);
const child = spawn(
process.execPath,
[
cliPath,
'--config',
value.configFile,
'--credential',
value.credentialFile,
'--session',
value.sessionFile,
'--worker-management-config',
value.workerManagementConfigFile,
'--worker-management-assertion',
value.workerManagementAssertionFile,
'--port=0',
],
{ cwd: packageRoot, stdio: ['ignore', 'pipe', 'pipe'] },
);
t.after(() => {
if (child.exitCode === null && child.signalCode === null)
child.kill('SIGKILL');
});
const started = JSON.parse(await firstLine(child.stdout));
assert.equal(started.workerManagementAuthority, 'server_only');
const response = await post(
started.origin,
value.sessionToken,
'/api/v1/worker-management/workers',
{
schema: 'qinglong/cluster-copilot-console-read-request@v1',
operation: 'worker_list',
projectId: 'project-main',
requestId: 'console-worker-list-1',
afterWorkerId: null,
},
);
assert.equal(response.statusCode, 200);
assert.equal(
response.body.result.result.schema,
'qinglong/worker-session-list@v1',
);
assert.equal(response.body.result.result.count, 1);
assert.equal(response.body.result.result.nextAfterWorkerId, 'worker-a');
assert.equal(
JSON.stringify(response.body).includes('worker-transport-hidden'),
false,
);
const management = value.requests.find(
(request) => request.path === '/api/v3/workers/management',
);
assert.equal(management.method, 'POST');
assert.equal(management.command.operation, 'worker-session.list');
assert.equal(
management.command.request.inspectionId,
'console-worker-list-1',
);
child.kill('SIGTERM');
const exit = await new Promise((resolve, reject) => {
child.once('error', reject);
child.once('close', (status, signal) => resolve({ status, signal }));
});
assert.deepEqual(exit, { status: 0, signal: null });
});
test('container mode requires an explicit publish port before any authority read', async () => {
const result = await runCli([
'--container-published-loopback',
@@ -233,6 +233,67 @@ test('redacts optional Run management observations while preserving fixed availa
assert.doesNotMatch(JSON.stringify(bundle), /project-sensitive/);
});
test('redacts Worker identity and preserves only bounded placement and capacity facts', async () => {
const bundle = await createClusterConsoleEvidenceBundle(
[
{
operation: 'worker_list',
observedAtMs: 1_700_000_003_000,
request: {
schema: requestSchema,
operation: 'worker_list',
afterWorkerId: null,
projectId: 'project-sensitive',
requestId: 'worker-request-sensitive',
},
fact: {
schema: 'qinglong/worker-session-list@v1',
projectId: 'project-sensitive',
observedAtMs: 1_700_000_003_000,
count: 1,
workers: [
{
workerId: 'worker-sensitive',
sessionId: 'session-must-not-export',
generation: 2,
sessionVersion: 5,
lifecycle: 'online',
compatibility: 'default_placement',
architecture: 'arm64',
supportTier: 'tier1',
protocolVersion: '1.0.0',
operatingSystem: 'linux',
maxConcurrentRuns: 2,
availableSlots: 1,
lastHeartbeatAtMs: 1_700_000_002_000,
leaseExpiresAtMs: 1_700_000_004_000,
},
],
nextAfterWorkerId: 'worker-sensitive',
},
},
],
1_700_000_004_000,
webcrypto,
);
const entry = bundle.entries[0];
assert.equal(entry.operation, 'worker_list');
assert.equal(entry.target.afterWorkerId, null);
assert.equal(entry.fact.workers[0].workerId, 'worker-001');
assert.equal(entry.fact.nextAfterWorkerId, 'worker-001');
assert.equal(entry.fact.workers[0].lifecycle, 'online');
assert.equal(entry.fact.workers[0].compatibility, 'default_placement');
assert.equal(entry.fact.workers[0].architecture, 'arm64');
assert.equal(entry.fact.workers[0].supportTier, 'tier1');
assert.equal(entry.fact.workers[0].availableSlots, 1);
assert.equal(entry.fact.workers[0].sessionId, undefined);
assert.equal(entry.fact.workers[0].protocolVersion, undefined);
assert.doesNotMatch(
JSON.stringify(bundle),
/worker-sensitive|session-must-not-export|project-sensitive/,
);
});
test('fails closed on widened records, unsafe JSON and every capacity ceiling', async () => {
const error = { code: 'QL3_CLUSTER_CONSOLE_EVIDENCE_BUNDLE_INVALID' };
assert.throws(() => measureClusterConsoleEvidenceRecord(null), error);
@@ -137,6 +137,16 @@ test('cross-verifies every fixed Console read operation', async (t) => {
requestId: 'q-management-inspect',
runId: 'r-management',
},
worker_list: {
afterWorkerId: null,
projectId: 'p-worker',
requestId: 'q-worker-list',
},
worker_inspect: {
projectId: 'p-worker',
requestId: 'q-worker-inspect',
workerId: 'worker-1',
},
run_list: {
afterCreatedAtMs: null,
afterRunId: null,
@@ -221,18 +231,23 @@ test('cross-verifies every fixed Console read operation', async (t) => {
},
}),
);
const bundle = await createClusterConsoleEvidenceBundle(
records,
1_700_000_001_000,
webcrypto,
);
const { filePath } = fixture(
t,
serializeClusterConsoleEvidenceBundle(bundle),
);
const result = verifyClusterConsoleEvidenceBundleFile(filePath);
assert.equal(result.status, 'verified');
assert.equal(result.bundle.entryCount, 16);
let verified = 0;
for (let index = 0; index < records.length; index += 16) {
const bundle = await createClusterConsoleEvidenceBundle(
records.slice(index, index + 16),
1_700_000_001_000 + index,
webcrypto,
);
const { filePath } = fixture(
t,
serializeClusterConsoleEvidenceBundle(bundle),
`operations-${index}.json`,
);
const result = verifyClusterConsoleEvidenceBundleFile(filePath);
assert.equal(result.status, 'verified');
verified += result.bundle.entryCount;
}
assert.equal(verified, 18);
});
test('CLI is secret-free on success, invalid input and usage errors', async (t) => {