feat(ql3): add optional console run drilldown

This commit is contained in:
whyour
2026-08-20 09:23:19 +08:00
parent cf21e984cb
commit 0a5f1448f1
29 changed files with 1193 additions and 68 deletions
@@ -24,25 +24,25 @@ const ASSETS = Object.freeze([
name: 'index.html',
field: 'html',
maximumBytes: 32 * 1024,
digest: '5d452c947a9f1266e4920cf48e7d5116b3f5ef8f9120f681124ed61f0217f5ff',
digest: 'a5a3d46a8493a27b53bd4a253ef38ebaf00d204a1454f4b47a1f1ceff668855f',
}),
Object.freeze({
name: 'app.css',
field: 'css',
maximumBytes: 64 * 1024,
digest: '5cf82b0a88920d106530603a7d407f852312138e5b7af5c422b3bccee785f144',
digest: 'ddfe85971df0b8acfaed8b4bb5f5bcdf679347106294987d928bbb82dc6610ec',
}),
Object.freeze({
name: 'evidence-bundle.js',
field: 'evidenceBundle',
maximumBytes: 32 * 1024,
digest: '6ecb14d2f59d872b889bb42c22bf0c0d2c150c90ea708fb1662d47f17f2e2095',
digest: '739ff786b651de23876fc5f4df5073e211085dfdfa1d2ecb79f53d5c871c6c1d',
}),
Object.freeze({
name: 'app.js',
field: 'javascript',
maximumBytes: 32 * 1024,
digest: 'f109c5b0491ba9a473e3129e35773edf38ac745b403e1f547f8252aa2932cdff',
digest: '7ed994d8f2f5b151a247c5dec1d2841d45d30ff05b14dd1f41c12c5582acf9e6',
}),
] as const);
@@ -1,5 +1,7 @@
#!/usr/bin/env node
import { randomUUID } from 'node:crypto';
import {
executeClusterCopilotCommand,
executeClusterProjectApiRead,
@@ -8,6 +10,20 @@ import {
validateClusterCopilotClientCredentialFile,
} from '../copilot-client/client';
import { readCanonicalFile } from '../management-support/managementClientConfiguration';
import { validateClusterAuthenticatedManagementClientConfiguration } from '../management-support/pluginPackageManagementClient';
import {
createRunCancellationBlockedListCommand,
projectRunCancellationBlockedList,
} from '../run-management/runCancellationBlockedList';
import {
createRunCancellationInspectionCommand,
projectRunCancellationInspection,
} from '../run-management/runCancellationInspection';
import {
createRunCancellationStatusCommand,
projectRunCancellationStatus,
} from '../run-management/runCancellationStatus';
import { executeClusterRunManagementCommand } from '../run-management/runManagementClient';
import { loadClusterCopilotConsoleAssets } from './assets';
import {
CLUSTER_COPILOT_CONSOLE_READ_OPERATIONS,
@@ -25,6 +41,7 @@ const USAGE = [
' ql3-copilot-console --config /absolute/client.json --credential /absolute/credential --session /absolute/session [--port=0..65535]',
' ql3-copilot-console --check --config /absolute/client.json --credential /absolute/credential --session /absolute/session',
' ql3-copilot-console --container-published-loopback --port=1024..65535 --config /absolute/client.json --credential /absolute/credential --session /absolute/session [--check]',
' Optional Run reads: --run-management-config /absolute/run-client.json --run-management-assertion /absolute/assertion.jwt',
'',
'Native mode binds 127.0.0.1. Container mode requires host-loopback port publication.',
'The browser session key remains in a separate owner-private 0600 file.',
@@ -35,12 +52,21 @@ interface ClusterCopilotConsoleCliArguments {
readonly configFile: string;
readonly credentialFile: string;
readonly networkBoundary: 'host-loopback' | 'container-published-loopback';
readonly runManagementAssertionFile?: string;
readonly runManagementConfigFile?: string;
readonly sessionFile: string;
readonly port: number;
}
const SESSION_TOKEN = /^[A-Za-z0-9_-]{43}$/;
const MANAGEMENT_ASSERTION = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/;
const MAXIMUM_SESSION_BYTES = 128;
const MAXIMUM_MANAGEMENT_ASSERTION_BYTES = 16 * 1024;
const RUN_MANAGEMENT_OPERATIONS = new Set([
'run_cancellation_status',
'run_cancellation_blocked_list',
'run_cancellation_inspect',
]);
function usageFailure(): never {
process.stderr.write(USAGE + '\n');
@@ -77,6 +103,8 @@ export function parseClusterCopilotConsoleCliArguments(
let configFile: string | undefined;
let credentialFile: string | undefined;
let sessionFile: string | undefined;
let runManagementConfigFile: string | undefined;
let runManagementAssertionFile: string | undefined;
let port = 0;
let portSeen = false;
let containerPublishedLoopback = false;
@@ -116,6 +144,28 @@ export function parseClusterCopilotConsoleCliArguments(
index += session.consumed;
continue;
}
const runManagementConfig = argumentValue(
argv,
index,
'--run-management-config',
);
if (runManagementConfig) {
if (runManagementConfigFile !== undefined) return usageFailure();
runManagementConfigFile = runManagementConfig.value;
index += runManagementConfig.consumed;
continue;
}
const runManagementAssertion = argumentValue(
argv,
index,
'--run-management-assertion',
);
if (runManagementAssertion) {
if (runManagementAssertionFile !== undefined) return usageFailure();
runManagementAssertionFile = runManagementAssertion.value;
index += runManagementAssertion.consumed;
continue;
}
const portArgument = argumentValue(argv, index, '--port');
if (portArgument) {
if (portSeen || !/^(?:0|[1-9][0-9]{0,4})$/.test(portArgument.value)) {
@@ -138,6 +188,8 @@ export function parseClusterCopilotConsoleCliArguments(
configFile === undefined ||
credentialFile === undefined ||
sessionFile === undefined ||
(runManagementConfigFile === undefined) !==
(runManagementAssertionFile === undefined) ||
(containerPublishedLoopback && port === 0) ||
(!containerPublishedLoopback && check && port !== 0)
) {
@@ -150,11 +202,128 @@ export function parseClusterCopilotConsoleCliArguments(
networkBoundary: containerPublishedLoopback
? 'container-published-loopback'
: 'host-loopback',
...(runManagementConfigFile !== undefined &&
runManagementAssertionFile !== undefined
? { runManagementConfigFile, runManagementAssertionFile }
: {}),
sessionFile,
port,
});
}
function validateRunManagementAuthority(
parsed: Readonly<ClusterCopilotConsoleCliArguments>,
): boolean {
if (
parsed.runManagementConfigFile === undefined ||
parsed.runManagementAssertionFile === undefined
) {
return false;
}
validateClusterAuthenticatedManagementClientConfiguration(
parsed.runManagementConfigFile,
'run',
);
let bytes: Buffer | undefined;
try {
bytes = readCanonicalFile(
parsed.runManagementAssertionFile,
MAXIMUM_MANAGEMENT_ASSERTION_BYTES,
'private',
);
if (
bytes.some((byte) => byte > 0x7f) ||
!MANAGEMENT_ASSERTION.test(bytes.toString('ascii'))
) {
throw new Error('invalid Run management assertion');
}
return true;
} finally {
bytes?.fill(0);
}
}
function availableOperations(runManagementAuthority: boolean) {
return runManagementAuthority
? CLUSTER_COPILOT_CONSOLE_READ_OPERATIONS
: CLUSTER_COPILOT_CONSOLE_READ_OPERATIONS.filter(
(operation) => !RUN_MANAGEMENT_OPERATIONS.has(operation),
);
}
function commandIdSource(requestId: string): () => string {
let first = true;
return () => {
if (first) {
first = false;
return requestId;
}
return randomUUID();
};
}
async function executeConsoleRead(
request: Readonly<ClusterCopilotConsoleReadRequest>,
parsed: Readonly<ClusterCopilotConsoleCliArguments>,
) {
if (request.operation === 'inspect' || request.operation === 'output') {
return executeClusterCopilotCommand({
configFile: parsed.configFile,
credentialFile: parsed.credentialFile,
command: clusterCopilotConsoleClientCommand(request),
});
}
if (
request.operation === 'run_cancellation_status' ||
request.operation === 'run_cancellation_blocked_list' ||
request.operation === 'run_cancellation_inspect'
) {
if (
parsed.runManagementConfigFile === undefined ||
parsed.runManagementAssertionFile === undefined
) {
throw new Error('Run management authority is disabled');
}
const createUuid = commandIdSource(request.requestId);
const command =
request.operation === 'run_cancellation_status'
? createRunCancellationStatusCommand(request.projectId, createUuid)
: request.operation === 'run_cancellation_blocked_list'
? createRunCancellationBlockedListCommand(
request.projectId,
request.cursor ?? undefined,
createUuid,
)
: createRunCancellationInspectionCommand(
request.projectId,
request.runId,
createUuid,
);
const result = await executeClusterRunManagementCommand({
configFile: parsed.runManagementConfigFile,
assertionFile: parsed.runManagementAssertionFile,
command,
});
const projected =
request.operation === 'run_cancellation_status'
? projectRunCancellationStatus(result)
: request.operation === 'run_cancellation_blocked_list'
? projectRunCancellationBlockedList(result)
: projectRunCancellationInspection(result);
return Object.freeze({
schemaVersion: 1 as const,
requestId: result.requestId,
result: projected as unknown as Readonly<Record<string, unknown>>,
});
}
return executeClusterProjectApiRead({
configFile: parsed.configFile,
credentialFile: parsed.credentialFile,
path: clusterCopilotConsoleProjectReadPath(request),
requestId: request.requestId,
});
}
function readSessionDigest(sessionFile: string): Buffer {
let bytes: Buffer | undefined;
try {
@@ -183,6 +352,8 @@ async function main(): Promise<void> {
const assets = loadClusterCopilotConsoleAssets(__dirname);
validateClusterCopilotClientConfiguration(parsed.configFile);
validateClusterCopilotClientCredentialFile(parsed.credentialFile);
const runManagementAuthority = validateRunManagementAuthority(parsed);
const operations = availableOperations(runManagementAuthority);
const sessionDigest = readSessionDigest(parsed.sessionFile);
if (parsed.check) {
try {
@@ -199,7 +370,10 @@ async function main(): Promise<void> {
publishedHostAddress: '127.0.0.1',
browserCredential: 'forbidden',
clusterCredential: 'server_only',
operations: CLUSTER_COPILOT_CONSOLE_READ_OPERATIONS,
runManagementAuthority: runManagementAuthority
? 'server_only'
: 'disabled',
operations,
mutation: false,
}) + '\n',
);
@@ -214,19 +388,7 @@ async function main(): Promise<void> {
assets,
executor: Object.freeze({
execute(request: Readonly<ClusterCopilotConsoleReadRequest>) {
if (request.operation === 'inspect' || request.operation === 'output') {
return executeClusterCopilotCommand({
configFile: parsed.configFile,
credentialFile: parsed.credentialFile,
command: clusterCopilotConsoleClientCommand(request),
});
}
return executeClusterProjectApiRead({
configFile: parsed.configFile,
credentialFile: parsed.credentialFile,
path: clusterCopilotConsoleProjectReadPath(request),
requestId: request.requestId,
});
return executeConsoleRead(request, parsed);
},
}),
networkBoundary: parsed.networkBoundary,
@@ -244,7 +406,10 @@ async function main(): Promise<void> {
publishedHostAddress: '127.0.0.1',
browserCredential: 'forbidden',
clusterCredential: 'server_only',
operations: CLUSTER_COPILOT_CONSOLE_READ_OPERATIONS,
runManagementAuthority: runManagementAuthority
? 'server_only'
: 'disabled',
operations,
mutation: false,
}) + '\n',
);
@@ -11,6 +11,9 @@ export const CLUSTER_COPILOT_CONSOLE_READ_RESPONSE_SCHEMA =
export const CLUSTER_COPILOT_CONSOLE_READ_OPERATIONS = Object.freeze([
'inspect',
'output',
'run_cancellation_status',
'run_cancellation_blocked_list',
'run_cancellation_inspect',
'run_list',
'run_read',
'run_event_list',
@@ -39,6 +42,10 @@ interface BaseReadRequest<
export type ClusterCopilotConsoleReadRequest =
| (BaseReadRequest<'inspect'> & Readonly<{ sourceRunId: string }>)
| (BaseReadRequest<'output'> & Readonly<{ sourceRunId: string }>)
| BaseReadRequest<'run_cancellation_status'>
| (BaseReadRequest<'run_cancellation_blocked_list'> &
Readonly<{ cursor: string | null }>)
| (BaseReadRequest<'run_cancellation_inspect'> & Readonly<{ runId: string }>)
| (BaseReadRequest<'run_list'> &
Readonly<{
afterCreatedAtMs: number | null;
@@ -98,6 +105,7 @@ export class InvalidClusterCopilotConsoleReadRequestError extends TypeError {
const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
const COPILOT_RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
const RUN_CANCELLATION_CURSOR = /^v1\.[A-Za-z0-9_-]{1,512}$/;
const PACKAGE_NAME = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
const WORKFLOW_ID = /^[a-z][a-z0-9-]{0,62}$/;
const UUID_V4 =
@@ -205,6 +213,36 @@ export function normalizeClusterCopilotConsoleReadRequest(
sourceRunId: record.sourceRunId,
});
}
if (op === 'run_cancellation_status') {
exact(record, op, []);
return Object.freeze({
...common(record),
operation: op,
});
}
if (op === 'run_cancellation_blocked_list') {
exact(record, op, ['cursor']);
if (
record.cursor !== null &&
(typeof record.cursor !== 'string' ||
!RUN_CANCELLATION_CURSOR.test(record.cursor))
)
invalid();
return Object.freeze({
...common(record),
operation: op,
cursor: record.cursor as string | null,
});
}
if (op === 'run_cancellation_inspect') {
exact(record, op, ['runId']);
if (!identifier(record.runId)) invalid();
return Object.freeze({
...common(record),
operation: op,
runId: record.runId,
});
}
if (op === 'run_list') {
exact(record, op, ['afterCreatedAtMs', 'afterRunId', 'limit']);
if (
@@ -433,7 +471,13 @@ export function clusterCopilotConsoleProjectReadPath(
request: Readonly<ClusterCopilotConsoleReadRequest>,
): string {
const normalized = normalizeClusterCopilotConsoleReadRequest(request);
if (normalized.operation === 'inspect' || normalized.operation === 'output')
if (
normalized.operation === 'inspect' ||
normalized.operation === 'output' ||
normalized.operation === 'run_cancellation_status' ||
normalized.operation === 'run_cancellation_blocked_list' ||
normalized.operation === 'run_cancellation_inspect'
)
invalid();
const project = '/api/v3/projects/' + encoded(normalized.projectId);
if (normalized.operation === 'run_list') {
@@ -30,6 +30,9 @@ const LIMITS = Object.freeze({
const OPERATIONS = Object.freeze([
'inspect',
'output',
'run_cancellation_status',
'run_cancellation_blocked_list',
'run_cancellation_inspect',
'run_list',
'run_read',
'run_event_list',
@@ -48,6 +51,17 @@ const REQUEST_FIELDS: Readonly<Record<EvidenceOperation, readonly string[]>> =
Object.freeze({
inspect: Object.freeze(['projectId', 'requestId', 'sourceRunId']),
output: Object.freeze(['projectId', 'requestId', 'sourceRunId']),
run_cancellation_status: Object.freeze(['projectId', 'requestId']),
run_cancellation_blocked_list: Object.freeze([
'cursor',
'projectId',
'requestId',
]),
run_cancellation_inspect: Object.freeze([
'projectId',
'requestId',
'runId',
]),
run_list: Object.freeze([
'afterCreatedAtMs',
'afterRunId',
@@ -121,7 +135,9 @@ const IDENTIFIER_DOMAINS: Readonly<Record<string, string>> = Object.freeze({
afterStepRunId: 'step',
afterTaskId: 'task',
artifactId: 'artifact',
attemptId: 'attempt',
contentDigest: 'digest',
cursor: 'cursor',
diagnosisRunId: 'run',
executionId: 'execution',
id: 'identifier',
@@ -142,7 +158,10 @@ const IDENTIFIER_DOMAINS: Readonly<Record<string, string>> = Object.freeze({
});
const SAFE_CONTAINERS = new Set([
'attempts',
'blockingResults',
'counts',
'dispatch',
'dispatches',
'events',
'items',
'metadata',
@@ -151,6 +170,7 @@ const SAFE_CONTAINERS = new Set([
'run',
'runs',
'source',
'signals',
'step',
'steps',
'summary',
@@ -176,10 +196,16 @@ const SAFE_BOOLEANS = new Set([
'truncated',
]);
const SAFE_ENUM_KEYS = new Set([
'assessment',
'cancelReason',
'finishReason',
'kind',
'lastResult',
'operation',
'operatorAction',
'outcome',
'runStatus',
'severity',
'stage',
'status',
]);
@@ -187,11 +213,15 @@ const SAFE_ENUM_VALUES = new Set([
'accepted',
'active',
'admission',
'attention_required',
'available',
'blocked',
'cancelled',
'completed',
'completion',
'converging',
'critical',
'clear',
'dispatch',
'dispatching',
'disabled',
@@ -200,12 +230,18 @@ const SAFE_ENUM_VALUES = new Set([
'failed',
'finalization',
'installed',
'inspect',
'invalid',
'identity_mismatch',
'local',
'lost',
'missing',
'model',
'none',
'not_found',
'pending',
'pid_mismatch',
'policy',
'post_model',
'pre_model',
'prompt',
@@ -213,6 +249,8 @@ const SAFE_ENUM_VALUES = new Set([
'queued',
'ready',
'recovery',
'rearm',
'reconcile',
'rejected',
'remote',
'retained',
@@ -226,18 +264,25 @@ const SAFE_ENUM_VALUES = new Set([
'step',
'stop',
'succeeded',
'shutdown',
'system',
'task',
'terminal',
'timed_out',
'timeout',
'tool',
'trigger',
'unknown',
'unsupported',
'user',
'wait',
'warning',
'ok',
'unavailable',
'workflow',
]);
const NUMERIC_KEY =
/^(?:schemaVersion|version|revision|sequence|attempt|priority|limit|offset|size|total|count|[A-Za-z0-9_]*(?:AtMs|TimeMs|DurationMs|Bytes|Tokens|Micros|Sequence|Version|Count|Limit|Offset|Size|Total))$/u;
/^(?:schemaVersion|version|revision|sequence|attempt|priority|limit|offset|size|total|count|exitCode|pending|leased|retryWait|dispatched|blocked|due|expiredLease|identityMismatch|pidMismatch|unsupported|invalid|[A-Za-z0-9_]*(?:AtMs|TimeMs|DurationMs|Bytes|Tokens|Micros|Sequence|Version|Count|Limit|Offset|Size|Total))$/u;
const SCHEMA_VALUE = /^[a-z0-9][a-z0-9./_-]{0,126}@[a-z0-9._-]{1,16}$/u;
const SHA256 = /^[0-9a-f]{64}$/u;
const CONTROL = /[\0-\x1f\x7f]/u;
@@ -10,6 +10,11 @@ import {
ClusterCopilotClientRemoteError,
ClusterCopilotClientRequestError,
} from '../copilot-client/client';
import {
ClusterPluginPackageManagementClientConfigurationError,
ClusterPluginPackageManagementClientRemoteError,
ClusterPluginPackageManagementClientRequestError,
} from '../management-support/pluginPackageManagementClient';
import { type ClusterCopilotConsoleAssets } from './assets';
import {
CLUSTER_COPILOT_CONSOLE_READ_RESPONSE_SCHEMA,
@@ -186,6 +191,10 @@ const READ_ROUTES: Readonly<
> = Object.freeze({
'/api/v1/copilot/inspect': 'inspect',
'/api/v1/copilot/output': 'output',
'/api/v1/run-management/cancellation-status': 'run_cancellation_status',
'/api/v1/run-management/blocked-cancellations':
'run_cancellation_blocked_list',
'/api/v1/run-management/cancellation-inspect': 'run_cancellation_inspect',
'/api/v1/observe/run-list': 'run_list',
'/api/v1/observe/run': 'run_read',
'/api/v1/observe/run-events': 'run_event_list',
@@ -294,7 +303,9 @@ async function readJsonBody(request: IncomingMessage): Promise<unknown> {
function remoteFailure(
response: ServerResponse,
error: ClusterCopilotClientRemoteError,
error:
| ClusterCopilotClientRemoteError
| ClusterPluginPackageManagementClientRemoteError,
): void {
const statusCode =
error.statusCode === 404 ? 404 : error.statusCode === 429 ? 429 : 502;
@@ -457,11 +468,17 @@ export async function startClusterCopilotConsoleServer(
code: 'invalid_cluster_copilot_console_read_request',
}),
);
} else if (error instanceof ClusterCopilotClientRemoteError) {
} else if (
error instanceof ClusterCopilotClientRemoteError ||
error instanceof ClusterPluginPackageManagementClientRemoteError
) {
remoteFailure(response, error);
} else if (
error instanceof ClusterCopilotClientConfigurationError ||
error instanceof ClusterCopilotClientRequestError
error instanceof ClusterCopilotClientRequestError ||
error instanceof
ClusterPluginPackageManagementClientConfigurationError ||
error instanceof ClusterPluginPackageManagementClientRequestError
) {
sendJson(
response,
@@ -0,0 +1,73 @@
import { randomUUID } from 'node:crypto';
import type { ClusterRunManagementClientResult } from './runManagementClient';
import {
RUN_CANCELLATION_DISPATCH_INSPECT_REQUEST_SCHEMA,
normalizeClusterRunManagementCommand,
type ClusterRunManagementCancellationInspectCommand,
type ClusterRunManagementCancellationInspectTransportResult,
} from './runManagementTransport';
export const RUN_CANCELLATION_INSPECTION_SCHEMA =
'qinglong/run-cancellation-inspection@v1' as const;
type CancellationDiagnostic =
ClusterRunManagementCancellationInspectTransportResult['diagnostic'];
export type RunCancellationInspectionObservation = Readonly<
{
schemaVersion: 1;
schema: typeof RUN_CANCELLATION_INSPECTION_SCHEMA;
component: 'qinglong3-run-management-client';
event: 'cancellation_inspected';
requestId: string;
} & Omit<CancellationDiagnostic, 'schema'>
>;
export function createRunCancellationInspectionCommand(
projectId: string,
runId: string,
createUuid: () => string = randomUUID,
): Readonly<ClusterRunManagementCancellationInspectCommand> {
const requestId = createUuid();
const auditEventId = createUuid();
let failureAuditEventId = createUuid();
for (
let attempts = 0;
failureAuditEventId === auditEventId && attempts < 3;
attempts += 1
) {
failureAuditEventId = createUuid();
}
return normalizeClusterRunManagementCommand({
schemaVersion: 1,
operation: 'run.cancellation.inspect',
request: {
projectId,
runId,
requestId,
auditEventId,
failureAuditEventId,
body: { schema: RUN_CANCELLATION_DISPATCH_INSPECT_REQUEST_SCHEMA },
},
}) as Readonly<ClusterRunManagementCancellationInspectCommand>;
}
export function projectRunCancellationInspection(
result: Readonly<ClusterRunManagementClientResult>,
): Readonly<RunCancellationInspectionObservation> {
if (result.result.operation !== 'run.cancellation.inspect') {
throw new TypeError(
'Run cancellation inspection requires an inspect result',
);
}
const { schema: _schema, ...diagnostic } = result.result.diagnostic;
return Object.freeze({
schemaVersion: 1,
schema: RUN_CANCELLATION_INSPECTION_SCHEMA,
component: 'qinglong3-run-management-client',
event: 'cancellation_inspected',
requestId: result.requestId,
...diagnostic,
});
}