mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): add strong cluster run management
This commit is contained in:
whyour
@@ -20,6 +20,31 @@
|
||||
"require": "./dist/approval-management/approvalDecisionManagement.js",
|
||||
"default": "./dist/approval-management/approvalDecisionManagement.js"
|
||||
},
|
||||
"./run-management": {
|
||||
"types": "./dist/run-management/runManagement.d.ts",
|
||||
"require": "./dist/run-management/runManagement.js",
|
||||
"default": "./dist/run-management/runManagement.js"
|
||||
},
|
||||
"./run-management-transport": {
|
||||
"types": "./dist/run-management/runManagementTransport.d.ts",
|
||||
"require": "./dist/run-management/runManagementTransport.js",
|
||||
"default": "./dist/run-management/runManagementTransport.js"
|
||||
},
|
||||
"./run-management-http": {
|
||||
"types": "./dist/run-management/runManagementHttp.d.ts",
|
||||
"require": "./dist/run-management/runManagementHttp.js",
|
||||
"default": "./dist/run-management/runManagementHttp.js"
|
||||
},
|
||||
"./run-management-process": {
|
||||
"types": "./dist/run-management/runManagementProcess.d.ts",
|
||||
"require": "./dist/run-management/runManagementProcess.js",
|
||||
"default": "./dist/run-management/runManagementProcess.js"
|
||||
},
|
||||
"./run-management-client": {
|
||||
"types": "./dist/run-management/runManagementClient.d.ts",
|
||||
"require": "./dist/run-management/runManagementClient.js",
|
||||
"default": "./dist/run-management/runManagementClient.js"
|
||||
},
|
||||
"./approval-management": {
|
||||
"types": "./dist/approval-management/approvalManagement.d.ts",
|
||||
"require": "./dist/approval-management/approvalManagement.js",
|
||||
@@ -331,6 +356,8 @@
|
||||
"ql3-worker-credential-client": "dist/worker-credential/workerCredentialManagementClientCli.js",
|
||||
"ql3-approval-manage": "dist/approval-management/approvalManagementCli.js",
|
||||
"ql3-approval-client": "dist/approval-management/approvalManagementClientCli.js",
|
||||
"ql3-run-manage": "dist/run-management/runManagementCli.js",
|
||||
"ql3-run-client": "dist/run-management/runManagementClientCli.js",
|
||||
"ql3-automation-manage": "dist/automation-management/automationManagementCli.js",
|
||||
"ql3-automation-client": "dist/automation-management/automationManagementClientCli.js",
|
||||
"ql3-provider-credential-manage": "dist/model-provider-credential/modelProviderCredentialManagementCli.js",
|
||||
|
||||
@@ -80,6 +80,11 @@ export const CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_ASSERTION_PRO
|
||||
purpose: 'model-provider-credential-management',
|
||||
});
|
||||
|
||||
export const CLUSTER_RUN_MANAGEMENT_IDENTITY_ASSERTION_PROFILE = Object.freeze({
|
||||
type: 'ql3-run-management+jwt',
|
||||
purpose: 'run-management',
|
||||
});
|
||||
|
||||
export interface ClusterPluginPackageIdentityAssertionVerifierOptions {
|
||||
readonly issuer: string;
|
||||
readonly audience: string;
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
CLUSTER_AUTOMATION_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
CLUSTER_APPROVAL_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
CLUSTER_RUN_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
createClusterPluginPackageIdentityAssertionVerifier,
|
||||
type ClusterManagementIdentityAssertionProfile,
|
||||
type ClusterPluginPackageIdentityAssertionAuthentication,
|
||||
@@ -507,3 +508,12 @@ export function createClusterModelProviderCredentialIdentityKeysetFile(
|
||||
CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
});
|
||||
}
|
||||
|
||||
export function createClusterRunIdentityKeysetFile(
|
||||
options: ClusterWorkerCredentialIdentityKeysetFileOptions,
|
||||
): Readonly<ClusterPluginPackageIdentityKeysetFile> {
|
||||
return createClusterPluginPackageIdentityKeysetFile({
|
||||
...options,
|
||||
assertionProfile: CLUSTER_RUN_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -67,6 +67,19 @@ import {
|
||||
ClusterModelProviderCredentialManagementTransportRequestError,
|
||||
ClusterModelProviderCredentialManagementTransportUnavailableError,
|
||||
} from '../model-provider-credential/modelProviderCredentialManagementTransport';
|
||||
import {
|
||||
ClusterRunManagementAuthorizationError,
|
||||
ClusterRunManagementConflictError,
|
||||
ClusterRunManagementRateLimitedError,
|
||||
ClusterRunManagementRequestError,
|
||||
ClusterRunManagementTargetUnavailableError,
|
||||
ClusterRunManagementUnavailableError,
|
||||
} from '../run-management/runManagement';
|
||||
import {
|
||||
ClusterRunManagementTransportAuthenticationError,
|
||||
ClusterRunManagementTransportRequestError,
|
||||
ClusterRunManagementTransportUnavailableError,
|
||||
} from '../run-management/runManagementTransport';
|
||||
|
||||
export const CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH =
|
||||
'/api/v3/plugin-packages/management';
|
||||
@@ -77,18 +90,21 @@ export const CLUSTER_AUTOMATION_MANAGEMENT_PATH =
|
||||
export const CLUSTER_APPROVAL_MANAGEMENT_PATH = '/api/v3/approvals/management';
|
||||
export const CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH =
|
||||
'/api/v3/provider-credentials/management';
|
||||
export const CLUSTER_RUN_MANAGEMENT_PATH = '/api/v3/runs/management';
|
||||
export type ClusterAuthenticatedManagementPath =
|
||||
| typeof CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH
|
||||
| typeof CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH
|
||||
| typeof CLUSTER_AUTOMATION_MANAGEMENT_PATH
|
||||
| typeof CLUSTER_APPROVAL_MANAGEMENT_PATH
|
||||
| typeof CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH;
|
||||
| typeof CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH
|
||||
| typeof CLUSTER_RUN_MANAGEMENT_PATH;
|
||||
const MANAGEMENT_PATHS = new Set<ClusterAuthenticatedManagementPath>([
|
||||
CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH,
|
||||
CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH,
|
||||
CLUSTER_AUTOMATION_MANAGEMENT_PATH,
|
||||
CLUSTER_APPROVAL_MANAGEMENT_PATH,
|
||||
CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH,
|
||||
CLUSTER_RUN_MANAGEMENT_PATH,
|
||||
]);
|
||||
const DEFAULT_MAX_BODY_BYTES = 64 * 1024;
|
||||
const DEFAULT_MAX_CONNECTIONS = 64;
|
||||
@@ -531,7 +547,8 @@ function responseError(error: unknown): HttpRequestError {
|
||||
error instanceof ClusterApprovalManagementTransportAuthenticationError ||
|
||||
error instanceof
|
||||
ClusterModelProviderCredentialManagementTransportAuthenticationError ||
|
||||
error instanceof ClusterModelProviderCredentialManagementAuthenticationError
|
||||
error instanceof ClusterModelProviderCredentialManagementAuthenticationError ||
|
||||
error instanceof ClusterRunManagementTransportAuthenticationError
|
||||
) {
|
||||
return new HttpRequestError(401, 'authentication_required');
|
||||
}
|
||||
@@ -544,6 +561,8 @@ function responseError(error: unknown): HttpRequestError {
|
||||
error instanceof
|
||||
ClusterModelProviderCredentialManagementTransportRequestError ||
|
||||
error instanceof ClusterModelProviderCredentialManagementRequestError ||
|
||||
error instanceof ClusterRunManagementTransportRequestError ||
|
||||
error instanceof ClusterRunManagementRequestError ||
|
||||
error instanceof PluginPackageManagementRequestError ||
|
||||
error instanceof WorkerCredentialManagementRequestError
|
||||
) {
|
||||
@@ -554,7 +573,8 @@ function responseError(error: unknown): HttpRequestError {
|
||||
error instanceof WorkerCredentialManagementAuthorizationError ||
|
||||
error instanceof ClusterAutomationManagementAuthorizationError ||
|
||||
error instanceof ClusterApprovalManagementTransportAuthorizationError ||
|
||||
error instanceof ClusterModelProviderCredentialManagementAuthorizationError
|
||||
error instanceof ClusterModelProviderCredentialManagementAuthorizationError ||
|
||||
error instanceof ClusterRunManagementAuthorizationError
|
||||
) {
|
||||
return new HttpRequestError(403, 'forbidden');
|
||||
}
|
||||
@@ -563,7 +583,8 @@ function responseError(error: unknown): HttpRequestError {
|
||||
error instanceof WorkerCredentialManagementConflictError ||
|
||||
error instanceof ClusterAutomationManagementConflictError ||
|
||||
error instanceof ClusterApprovalManagementTransportConflictError ||
|
||||
error instanceof ClusterModelProviderCredentialManagementConflictError
|
||||
error instanceof ClusterModelProviderCredentialManagementConflictError ||
|
||||
error instanceof ClusterRunManagementConflictError
|
||||
) {
|
||||
return new HttpRequestError(409, 'conflict');
|
||||
}
|
||||
@@ -578,8 +599,12 @@ function responseError(error: unknown): HttpRequestError {
|
||||
) {
|
||||
return new HttpRequestError(429, 'quota_exceeded', error.retryAfterMs);
|
||||
}
|
||||
if (error instanceof ClusterRunManagementRateLimitedError) {
|
||||
return new HttpRequestError(429, 'rate_limited', error.retryAfterMs);
|
||||
}
|
||||
if (
|
||||
error instanceof ClusterApprovalManagementTransportTargetUnavailableError
|
||||
error instanceof ClusterApprovalManagementTransportTargetUnavailableError ||
|
||||
error instanceof ClusterRunManagementTargetUnavailableError
|
||||
) {
|
||||
return new HttpRequestError(404, 'not_found');
|
||||
}
|
||||
@@ -594,6 +619,8 @@ function responseError(error: unknown): HttpRequestError {
|
||||
error instanceof
|
||||
ClusterModelProviderCredentialManagementTransportUnavailableError ||
|
||||
error instanceof ClusterModelProviderCredentialManagementUnavailableError ||
|
||||
error instanceof ClusterRunManagementTransportUnavailableError ||
|
||||
error instanceof ClusterRunManagementUnavailableError ||
|
||||
error instanceof PluginPackageManagementUnavailableError ||
|
||||
error instanceof WorkerCredentialManagementUnavailableError
|
||||
) {
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
import {
|
||||
PostgresProjectPolicyRepository,
|
||||
PostgresRunManualRetryRepository,
|
||||
PostgresSecurityAuditRepository,
|
||||
} from '@qinglong/cluster-postgres/run-manager';
|
||||
import type { PostgresPool } from '@qinglong/runtime-core';
|
||||
import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy';
|
||||
import {
|
||||
InvalidRunManualRetryError,
|
||||
RunManualRetryFenceRejectedError,
|
||||
RunManualRetryNotFoundError,
|
||||
RunManualRetryRateLimitedError,
|
||||
RunManualRetryUnavailableError,
|
||||
type RunManualRetryResult,
|
||||
type RunManualRetrySourceStatus,
|
||||
} from '@qinglong/runtime-core/run-manual-retry';
|
||||
import {
|
||||
normalizeSecurityPrincipal,
|
||||
type SecurityPolicyFence,
|
||||
type SecurityPrincipal,
|
||||
} from '@qinglong/runtime-core/security';
|
||||
import { normalizeSecurityAuditRecord } from '@qinglong/runtime-core/security-audit';
|
||||
|
||||
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const UUID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
|
||||
export interface ClusterRunManagementRetryRequest {
|
||||
readonly projectId: string;
|
||||
readonly sourceRunId: string;
|
||||
readonly mutationId: string;
|
||||
readonly expectedRunVersion: number;
|
||||
readonly expectedRunStatus: RunManualRetrySourceStatus;
|
||||
readonly requestId: string;
|
||||
readonly auditEventId: string;
|
||||
readonly failureAuditEventId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
}
|
||||
|
||||
export interface ClusterRunManagementService {
|
||||
retry(
|
||||
request: Readonly<ClusterRunManagementRetryRequest>,
|
||||
): Promise<Readonly<RunManualRetryResult>>;
|
||||
}
|
||||
|
||||
export interface ClusterRunManagementOptions {
|
||||
readonly pool: PostgresPool;
|
||||
readonly now?: () => number;
|
||||
readonly randomUuid?: () => string;
|
||||
}
|
||||
|
||||
export class ClusterRunManagementConfigurationError extends TypeError {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_CONFIGURATION_INVALID';
|
||||
constructor() {
|
||||
super('Cluster Run management configuration is invalid');
|
||||
this.name = 'ClusterRunManagementConfigurationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementRequestError extends TypeError {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_REQUEST_INVALID';
|
||||
constructor() {
|
||||
super('Cluster Run management request is invalid');
|
||||
this.name = 'ClusterRunManagementRequestError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementAuthorizationError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_FORBIDDEN';
|
||||
constructor() {
|
||||
super('Cluster Run management is forbidden');
|
||||
this.name = 'ClusterRunManagementAuthorizationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementTargetUnavailableError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_TARGET_UNAVAILABLE';
|
||||
constructor() {
|
||||
super('Cluster Run management target is unavailable');
|
||||
this.name = 'ClusterRunManagementTargetUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementConflictError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_CONFLICT';
|
||||
constructor() {
|
||||
super('Cluster Run management conflicts with durable state');
|
||||
this.name = 'ClusterRunManagementConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementRateLimitedError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_RATE_LIMITED';
|
||||
constructor(readonly retryAfterMs: number) {
|
||||
super('Cluster Run management rate limit is exhausted');
|
||||
this.name = 'ClusterRunManagementRateLimitedError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementUnavailableError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_UNAVAILABLE';
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('Cluster Run management is unavailable', options);
|
||||
this.name = 'ClusterRunManagementUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function exactRequest(
|
||||
value: unknown,
|
||||
): asserts value is Readonly<ClusterRunManagementRetryRequest> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
Object.keys(value).sort().join('\0') !==
|
||||
[
|
||||
'auditEventId',
|
||||
'expectedRunStatus',
|
||||
'expectedRunVersion',
|
||||
'failureAuditEventId',
|
||||
'mutationId',
|
||||
'principal',
|
||||
'projectId',
|
||||
'requestId',
|
||||
'sourceRunId',
|
||||
]
|
||||
.sort()
|
||||
.join('\0')
|
||||
) {
|
||||
throw new ClusterRunManagementRequestError();
|
||||
}
|
||||
}
|
||||
|
||||
function validUuid(value: unknown): value is string {
|
||||
return typeof value === 'string' && UUID_PATTERN.test(value);
|
||||
}
|
||||
|
||||
function failureReason(error: unknown): string {
|
||||
if (error instanceof ClusterRunManagementAuthorizationError) {
|
||||
return 'authorization_rejected';
|
||||
}
|
||||
if (error instanceof RunManualRetryNotFoundError) return 'run_not_found';
|
||||
if (error instanceof RunManualRetryRateLimitedError) return 'rate_limited';
|
||||
if (error instanceof RunManualRetryFenceRejectedError) return error.reason;
|
||||
return 'management_unavailable';
|
||||
}
|
||||
|
||||
/** Strong OIDC Run management composition over one run-manager Pool. */
|
||||
export function createClusterRunManagementService(
|
||||
options: ClusterRunManagementOptions,
|
||||
): Readonly<ClusterRunManagementService> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some(
|
||||
(key) => key !== 'pool' && key !== 'now' && key !== 'randomUuid',
|
||||
) ||
|
||||
!options.pool ||
|
||||
typeof options.pool.query !== 'function' ||
|
||||
typeof options.pool.connect !== 'function' ||
|
||||
(options.now !== undefined && typeof options.now !== 'function') ||
|
||||
(options.randomUuid !== undefined && typeof options.randomUuid !== 'function')
|
||||
) {
|
||||
throw new ClusterRunManagementConfigurationError();
|
||||
}
|
||||
const now = options.now ?? Date.now;
|
||||
const createId = options.randomUuid ?? randomUUID;
|
||||
const policy = new ProjectPolicyEngine(
|
||||
new PostgresProjectPolicyRepository(options.pool),
|
||||
);
|
||||
const retries = new PostgresRunManualRetryRepository(options.pool);
|
||||
const audit = new PostgresSecurityAuditRepository(options.pool);
|
||||
|
||||
return Object.freeze({
|
||||
async retry(requestValue: Readonly<ClusterRunManagementRetryRequest>) {
|
||||
exactRequest(requestValue);
|
||||
const observedAtMs = now();
|
||||
let principal: Readonly<SecurityPrincipal>;
|
||||
if (
|
||||
!Number.isSafeInteger(observedAtMs) ||
|
||||
observedAtMs < 0 ||
|
||||
!IDENTIFIER_PATTERN.test(requestValue.projectId) ||
|
||||
!IDENTIFIER_PATTERN.test(requestValue.sourceRunId) ||
|
||||
!IDENTIFIER_PATTERN.test(requestValue.requestId) ||
|
||||
!validUuid(requestValue.mutationId) ||
|
||||
!validUuid(requestValue.auditEventId) ||
|
||||
!validUuid(requestValue.failureAuditEventId) ||
|
||||
requestValue.auditEventId === requestValue.failureAuditEventId
|
||||
) {
|
||||
throw new ClusterRunManagementRequestError();
|
||||
}
|
||||
try {
|
||||
principal = normalizeSecurityPrincipal(
|
||||
requestValue.principal,
|
||||
observedAtMs,
|
||||
);
|
||||
} catch {
|
||||
throw new ClusterRunManagementRequestError();
|
||||
}
|
||||
|
||||
let fence: Readonly<SecurityPolicyFence> | null = null;
|
||||
try {
|
||||
const decision = await policy.authorize(
|
||||
principal,
|
||||
requestValue.projectId,
|
||||
'run.retry',
|
||||
);
|
||||
fence = decision.fence;
|
||||
if (
|
||||
decision.effect !== 'allow' ||
|
||||
!fence ||
|
||||
fence.bindingVersion === null
|
||||
) {
|
||||
throw new ClusterRunManagementAuthorizationError();
|
||||
}
|
||||
return await retries.retryRun({
|
||||
projectId: requestValue.projectId,
|
||||
sourceRunId: requestValue.sourceRunId,
|
||||
mutationId: requestValue.mutationId,
|
||||
expectedRunVersion: requestValue.expectedRunVersion,
|
||||
expectedRunStatus: requestValue.expectedRunStatus,
|
||||
runId: createId(),
|
||||
attemptId: createId(),
|
||||
createdEventId: createId(),
|
||||
queuedEventId: createId(),
|
||||
auditEventId: requestValue.auditEventId,
|
||||
requestId: requestValue.requestId,
|
||||
principal,
|
||||
policyFence: fence,
|
||||
});
|
||||
} catch (error) {
|
||||
try {
|
||||
await audit.record(
|
||||
normalizeSecurityAuditRecord({
|
||||
eventId: requestValue.failureAuditEventId,
|
||||
requestId: requestValue.requestId,
|
||||
operationId: 'run.retry',
|
||||
projectId: requestValue.projectId,
|
||||
subject: principal.subject,
|
||||
authenticationId: principal.authenticationId,
|
||||
outcome: 'denied',
|
||||
reasons: [failureReason(error)],
|
||||
fence,
|
||||
occurredAtMs: observedAtMs,
|
||||
}),
|
||||
);
|
||||
} catch (auditError) {
|
||||
throw new ClusterRunManagementUnavailableError({ cause: auditError });
|
||||
}
|
||||
if (error instanceof ClusterRunManagementAuthorizationError) throw error;
|
||||
if (error instanceof InvalidRunManualRetryError) {
|
||||
throw new ClusterRunManagementRequestError();
|
||||
}
|
||||
if (error instanceof RunManualRetryNotFoundError) {
|
||||
throw new ClusterRunManagementTargetUnavailableError();
|
||||
}
|
||||
if (error instanceof RunManualRetryFenceRejectedError) {
|
||||
throw new ClusterRunManagementConflictError();
|
||||
}
|
||||
if (error instanceof RunManualRetryRateLimitedError) {
|
||||
throw new ClusterRunManagementRateLimitedError(error.retryAfterMs);
|
||||
}
|
||||
if (error instanceof RunManualRetryUnavailableError) {
|
||||
throw new ClusterRunManagementUnavailableError({ cause: error });
|
||||
}
|
||||
throw new ClusterRunManagementUnavailableError({ cause: error });
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import {
|
||||
startClusterRunManagementProcess,
|
||||
type ClusterRunManagementProcessRuntime,
|
||||
} from './runManagementProcess';
|
||||
|
||||
const USAGE = 'Usage: ql3-run-manage';
|
||||
|
||||
function fact(error: unknown): Readonly<Record<string, unknown>> {
|
||||
const candidate = error as { readonly name?: unknown; readonly code?: unknown };
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-run-management',
|
||||
event: 'management_failed',
|
||||
name: typeof candidate?.name === 'string' ? candidate.name : 'Error',
|
||||
...(typeof candidate?.code === 'string' ? { code: candidate.code } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
function emit(value: Readonly<Record<string, unknown>>): void {
|
||||
process.stdout.write(`${JSON.stringify(value)}\n`);
|
||||
}
|
||||
|
||||
async function run(argv: readonly string[]): Promise<void> {
|
||||
if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
|
||||
process.stdout.write(`${USAGE}\n`);
|
||||
return;
|
||||
}
|
||||
if (argv.length !== 0) {
|
||||
process.stderr.write(`${JSON.stringify({ code: 'QL3_RUN_MANAGEMENT_CLI_USAGE_INVALID', message: USAGE })}\n`);
|
||||
process.exitCode = 64;
|
||||
return;
|
||||
}
|
||||
let runtime: Readonly<ClusterRunManagementProcessRuntime>;
|
||||
try {
|
||||
runtime = await startClusterRunManagementProcess({
|
||||
environment: process.env,
|
||||
onError: () => emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_unavailable' }),
|
||||
});
|
||||
} catch (error) {
|
||||
process.stderr.write(`${JSON.stringify(fact(error))}\n`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
if (runtime.status === 'disabled') {
|
||||
emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_disabled' });
|
||||
return;
|
||||
}
|
||||
emit({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-run-management',
|
||||
event: 'management_started',
|
||||
address: runtime.address,
|
||||
identityGeneration: runtime.identity.generation,
|
||||
databaseContractVersion: runtime.database.contractVersion,
|
||||
databaseMigrationCount: runtime.database.migrationIds.length,
|
||||
});
|
||||
let stopping: Promise<void> | undefined;
|
||||
const stop = (): Promise<void> => {
|
||||
stopping ??= runtime.close().then(() => emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_stopped' }));
|
||||
return stopping;
|
||||
};
|
||||
for (const signal of ['SIGINT', 'SIGTERM'] as const) {
|
||||
process.once(signal, () => {
|
||||
void stop().then(
|
||||
() => { process.exitCode = 0; },
|
||||
(error) => { process.stderr.write(`${JSON.stringify(fact(error))}\n`); process.exitCode = 1; },
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void run(process.argv.slice(2));
|
||||
@@ -0,0 +1,106 @@
|
||||
import {
|
||||
RUN_MANUAL_RETRY_SCHEMA,
|
||||
normalizeRunManualRetryResult,
|
||||
} from '@qinglong/runtime-core/run-manual-retry';
|
||||
import {
|
||||
ClusterPluginPackageManagementClientRequestError,
|
||||
executeClusterAuthenticatedManagementClient,
|
||||
type ClusterAuthenticatedManagementClientResult,
|
||||
type ClusterPluginPackageManagementClientConnectionOptions,
|
||||
type ClusterPluginPackageManagementClientPaths,
|
||||
} from '../management-support/pluginPackageManagementClient';
|
||||
import {
|
||||
normalizeClusterRunManagementCommand,
|
||||
type ClusterRunManagementCommand,
|
||||
type ClusterRunManagementTransportResult,
|
||||
} from './runManagementTransport';
|
||||
|
||||
const MANAGEMENT_PATH = '/api/v3/runs/management';
|
||||
|
||||
export type ClusterRunManagementClientPaths =
|
||||
ClusterPluginPackageManagementClientPaths;
|
||||
export type ClusterRunManagementClientConnectionOptions =
|
||||
ClusterPluginPackageManagementClientConnectionOptions;
|
||||
export type ClusterRunManagementClientResult =
|
||||
ClusterAuthenticatedManagementClientResult<ClusterRunManagementTransportResult>;
|
||||
|
||||
function invalid(): never {
|
||||
throw new ClusterPluginPackageManagementClientRequestError();
|
||||
}
|
||||
|
||||
function exact(value: unknown, keys: readonly string[]): Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) invalid();
|
||||
const actual = Object.keys(value as object).sort();
|
||||
const expected = [...keys].sort();
|
||||
if (
|
||||
actual.length !== expected.length ||
|
||||
actual.some((key, index) => key !== expected[index])
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
export function validateClusterRunManagementClientResult(
|
||||
value: unknown,
|
||||
command: Readonly<ClusterRunManagementCommand>,
|
||||
): Readonly<ClusterRunManagementTransportResult> {
|
||||
const envelope = exact(value, ['schemaVersion', 'operation', 'retry']);
|
||||
if (envelope.schemaVersion !== 1 || envelope.operation !== 'run.retry') invalid();
|
||||
const retry = exact(envelope.retry, [
|
||||
'schema',
|
||||
'status',
|
||||
'projectId',
|
||||
'sourceRunId',
|
||||
'sourceRunStatus',
|
||||
'sourceRunVersion',
|
||||
'runId',
|
||||
'retryOfRunId',
|
||||
'taskId',
|
||||
'taskRevision',
|
||||
'attemptId',
|
||||
'runStatus',
|
||||
'runVersion',
|
||||
'eventSequence',
|
||||
'executorType',
|
||||
'executionRevisionDigest',
|
||||
'createdAtMs',
|
||||
]);
|
||||
if (retry.schema !== RUN_MANUAL_RETRY_SCHEMA) invalid();
|
||||
try {
|
||||
const { schema: _schema, ...result } = retry;
|
||||
const normalized = normalizeRunManualRetryResult(result as never);
|
||||
if (
|
||||
normalized.projectId !== command.request.projectId ||
|
||||
normalized.sourceRunId !== command.request.sourceRunId ||
|
||||
normalized.sourceRunVersion !== command.request.body.expectedRunVersion ||
|
||||
normalized.sourceRunStatus !== command.request.body.expectedRunStatus ||
|
||||
normalized.executorType !== 'remote_worker'
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
} catch {
|
||||
invalid();
|
||||
}
|
||||
return Object.freeze(
|
||||
envelope as unknown as ClusterRunManagementTransportResult,
|
||||
);
|
||||
}
|
||||
|
||||
const PROTOCOL = Object.freeze({
|
||||
managementPath: MANAGEMENT_PATH,
|
||||
clientCertificate: 'required' as const,
|
||||
normalizeCommand: normalizeClusterRunManagementCommand,
|
||||
validateResult: validateClusterRunManagementClientResult,
|
||||
});
|
||||
|
||||
export function executeClusterRunManagementClient(
|
||||
paths: ClusterRunManagementClientPaths,
|
||||
connectionOptions?: ClusterRunManagementClientConnectionOptions,
|
||||
): Promise<Readonly<ClusterRunManagementClientResult>> {
|
||||
return executeClusterAuthenticatedManagementClient(
|
||||
paths,
|
||||
PROTOCOL,
|
||||
connectionOptions,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { ClusterPluginPackageManagementClientRemoteError } from '../management-support/pluginPackageManagementClient';
|
||||
import { executeClusterRunManagementClient } from './runManagementClient';
|
||||
|
||||
const USAGE =
|
||||
'Usage: ql3-run-client --config=/absolute/client.json --command=/absolute/command.json --assertion=/absolute/assertion.jwt';
|
||||
|
||||
function argumentsFrom(argv: readonly string[]): Readonly<{
|
||||
configFile: string;
|
||||
commandFile: string;
|
||||
assertionFile: string;
|
||||
}> | null {
|
||||
if (argv.length !== 3) return null;
|
||||
const values = new Map<string, string>();
|
||||
for (const argument of argv) {
|
||||
const match = /^--(config|command|assertion)=(\/.+)$/.exec(argument);
|
||||
if (!match || values.has(match[1]!)) return null;
|
||||
values.set(match[1]!, match[2]!);
|
||||
}
|
||||
if (!values.has('config') || !values.has('command') || !values.has('assertion')) return null;
|
||||
return Object.freeze({
|
||||
configFile: values.get('config')!,
|
||||
commandFile: values.get('command')!,
|
||||
assertionFile: values.get('assertion')!,
|
||||
});
|
||||
}
|
||||
|
||||
function failureFact(error: unknown): Readonly<Record<string, unknown>> {
|
||||
const candidate = error as { readonly code?: unknown };
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-run-management-client',
|
||||
event: 'command_failed',
|
||||
code: typeof candidate?.code === 'string' ? candidate.code : 'QL3_RUN_MANAGEMENT_CLIENT_FAILED',
|
||||
...(error instanceof ClusterPluginPackageManagementClientRemoteError
|
||||
? {
|
||||
statusCode: error.statusCode,
|
||||
responseCode: error.responseCode,
|
||||
requestId: error.requestId,
|
||||
...(error.retryAfterSeconds === null ? {} : { retryAfterSeconds: error.retryAfterSeconds }),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
async function run(argv: readonly string[]): Promise<void> {
|
||||
if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
|
||||
process.stdout.write(`${USAGE}\n`);
|
||||
return;
|
||||
}
|
||||
const paths = argumentsFrom(argv);
|
||||
if (!paths) {
|
||||
process.stderr.write(`${JSON.stringify({ schemaVersion: 1, component: 'qinglong3-run-management-client', event: 'usage_invalid', code: 'QL3_RUN_MANAGEMENT_CLIENT_USAGE_INVALID' })}\n`);
|
||||
process.exitCode = 64;
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const result = await executeClusterRunManagementClient(paths);
|
||||
process.stdout.write(`${JSON.stringify({ schemaVersion: 1, component: 'qinglong3-run-management-client', event: 'command_completed', requestId: result.requestId, result: result.result })}\n`);
|
||||
} catch (error) {
|
||||
process.stderr.write(`${JSON.stringify(failureFact(error))}\n`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
void run(process.argv.slice(2));
|
||||
@@ -0,0 +1,24 @@
|
||||
import {
|
||||
CLUSTER_RUN_MANAGEMENT_PATH,
|
||||
startClusterPluginPackageManagementHttp,
|
||||
type ClusterPluginPackageManagementHttpApplication,
|
||||
type StartClusterPluginPackageManagementHttpOptions,
|
||||
} from '../management-support/pluginPackageManagementHttp';
|
||||
|
||||
export type ClusterRunManagementHttpApplication =
|
||||
ClusterPluginPackageManagementHttpApplication;
|
||||
|
||||
export type StartClusterRunManagementHttpOptions = Omit<
|
||||
StartClusterPluginPackageManagementHttpOptions,
|
||||
'managementPath'
|
||||
>;
|
||||
|
||||
/** Starts the shared bounded OIDC/mTLS HTTPS adapter on the Run-only path. */
|
||||
export function startClusterRunManagementHttp(
|
||||
options: StartClusterRunManagementHttpOptions,
|
||||
): Promise<Readonly<ClusterRunManagementHttpApplication>> {
|
||||
return startClusterPluginPackageManagementHttp({
|
||||
...options,
|
||||
managementPath: CLUSTER_RUN_MANAGEMENT_PATH,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,467 @@
|
||||
import type {
|
||||
OpenPostgresDatabase,
|
||||
PostgresDatabaseResource,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
PostgresRunManagementIdentityKeysetLedgerRepository,
|
||||
assertPostgresRunManagerSchemaReady,
|
||||
createPostgresDatabaseOpener,
|
||||
isPostgresTlsDnsServername,
|
||||
loadPostgresCertificateAuthorityFile,
|
||||
loadPostgresConnectionEnvironment,
|
||||
type PostgresConnectionOptions,
|
||||
type PostgresPoolOptions,
|
||||
type PostgresSchemaReadinessReport,
|
||||
} from '@qinglong/cluster-postgres/run-manager';
|
||||
import {
|
||||
absoluteManagementEnvironmentFile,
|
||||
booleanManagementEnvironmentValue,
|
||||
boundedManagementEnvironmentValue,
|
||||
integerManagementEnvironmentValue,
|
||||
readManagementTlsFile,
|
||||
} from '../management-support/managementProcessSupport';
|
||||
import {
|
||||
createClusterRunIdentityKeysetFile,
|
||||
type ClusterPluginPackageIdentityKeysetFile,
|
||||
type ClusterPluginPackageIdentityKeysetSnapshot,
|
||||
} from '../management-support/pluginPackageIdentityKeyset';
|
||||
import { validateClusterManagementClientTrust } from '../worker-credential/management-server/workerCredentialManagementMutualTls';
|
||||
import { createClusterRunManagementService } from './runManagement';
|
||||
import {
|
||||
startClusterRunManagementHttp,
|
||||
type ClusterRunManagementHttpApplication,
|
||||
type StartClusterRunManagementHttpOptions,
|
||||
} from './runManagementHttp';
|
||||
import { createClusterRunManagementTransport } from './runManagementTransport';
|
||||
|
||||
const SAFE_HOST = /^[A-Za-z0-9][A-Za-z0-9.:-]{0,254}$/;
|
||||
const SAFE_APPLICATION_NAME = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,62}$/;
|
||||
|
||||
export type ClusterRunManagementProcessEnvironment = Readonly<
|
||||
Record<string, string | undefined>
|
||||
>;
|
||||
|
||||
export type ClusterRunManagementProcessConfig =
|
||||
| Readonly<{ enabled: false }>
|
||||
| Readonly<{
|
||||
enabled: true;
|
||||
profile: 'cluster-admin';
|
||||
host: string;
|
||||
port: number;
|
||||
certificateFile: string;
|
||||
privateKeyFile: string;
|
||||
clientCertificateAuthorityFile: string;
|
||||
clientCertificateRevocationListFile: string;
|
||||
identityKeysetFile: string;
|
||||
http: Readonly<{
|
||||
maxBodyBytes: number;
|
||||
maxConnections: number;
|
||||
maxConcurrentRequests: number;
|
||||
requestTimeoutMs: number;
|
||||
drainTimeoutMs: number;
|
||||
rateWindowMs: number;
|
||||
peerRequestLimit: number;
|
||||
globalRequestLimit: number;
|
||||
maxRateLimitPeers: number;
|
||||
}>;
|
||||
database: Readonly<{
|
||||
connection: PostgresConnectionOptions;
|
||||
pool: PostgresPoolOptions;
|
||||
}>;
|
||||
}>;
|
||||
|
||||
export type ClusterRunManagementProcessRuntime =
|
||||
| Readonly<{ status: 'disabled'; close(): Promise<void> }>
|
||||
| Readonly<{
|
||||
status: 'active';
|
||||
address: Readonly<{ host: string; port: number }>;
|
||||
database: PostgresSchemaReadinessReport;
|
||||
identity: ClusterPluginPackageIdentityKeysetSnapshot;
|
||||
availabilityStatus(): 'ready' | 'unavailable' | 'stopped';
|
||||
close(): Promise<void>;
|
||||
}>;
|
||||
|
||||
export interface StartClusterRunManagementProcessOptions {
|
||||
readonly environment: ClusterRunManagementProcessEnvironment;
|
||||
readonly openDatabase?: OpenPostgresDatabase;
|
||||
readonly identities?: ClusterPluginPackageIdentityKeysetFile;
|
||||
readonly assertReady?: (
|
||||
pool: PostgresDatabaseResource['pool'],
|
||||
) => Promise<PostgresSchemaReadinessReport>;
|
||||
readonly startHttp?: (
|
||||
options: StartClusterRunManagementHttpOptions,
|
||||
) => Promise<Readonly<ClusterRunManagementHttpApplication>>;
|
||||
readonly now?: () => number;
|
||||
readonly randomUuid?: () => string;
|
||||
readonly onError?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export class ClusterRunManagementProcessConfigError extends TypeError {
|
||||
readonly code = 'QL3_RUN_MANAGEMENT_PROCESS_CONFIG_INVALID';
|
||||
constructor(message: string) {
|
||||
super(`Run management process configuration is invalid: ${message}`);
|
||||
this.name = 'ClusterRunManagementProcessConfigError';
|
||||
}
|
||||
}
|
||||
|
||||
function failure(message: string): ClusterRunManagementProcessConfigError {
|
||||
return new ClusterRunManagementProcessConfigError(message);
|
||||
}
|
||||
|
||||
function bounded(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
name: string,
|
||||
maximumLength: number,
|
||||
required = false,
|
||||
): string | undefined {
|
||||
return boundedManagementEnvironmentValue(
|
||||
environment,
|
||||
name,
|
||||
maximumLength,
|
||||
failure,
|
||||
required,
|
||||
);
|
||||
}
|
||||
|
||||
function bool(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
name: string,
|
||||
): boolean {
|
||||
return booleanManagementEnvironmentValue(environment, name, failure);
|
||||
}
|
||||
|
||||
function integer(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
name: string,
|
||||
fallback: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): number {
|
||||
return integerManagementEnvironmentValue(
|
||||
environment,
|
||||
name,
|
||||
fallback,
|
||||
minimum,
|
||||
maximum,
|
||||
failure,
|
||||
);
|
||||
}
|
||||
|
||||
function absolute(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
name: string,
|
||||
): string {
|
||||
return absoluteManagementEnvironmentFile(environment, name, failure);
|
||||
}
|
||||
|
||||
function loadDatabase(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
): Readonly<{
|
||||
connection: PostgresConnectionOptions;
|
||||
pool: PostgresPoolOptions;
|
||||
}> {
|
||||
let connection: PostgresConnectionOptions;
|
||||
try {
|
||||
connection = loadPostgresConnectionEnvironment(environment, {
|
||||
connectionString: 'QL3_POSTGRES_RUN_MANAGER_URL',
|
||||
host: 'QL3_POSTGRES_RUN_MANAGER_HOST',
|
||||
port: 'QL3_POSTGRES_RUN_MANAGER_PORT',
|
||||
database: 'QL3_POSTGRES_RUN_MANAGER_DATABASE',
|
||||
user: 'QL3_POSTGRES_RUN_MANAGER_USER',
|
||||
password: 'QL3_POSTGRES_RUN_MANAGER_PASSWORD',
|
||||
});
|
||||
} catch (error) {
|
||||
throw failure(
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: 'PostgreSQL run manager connection is invalid',
|
||||
);
|
||||
}
|
||||
const mode = environment.QL3_POSTGRES_RUN_MANAGER_TLS_MODE ?? 'verify-full';
|
||||
if (mode !== 'verify-full' && mode !== 'disable') {
|
||||
throw failure('QL3_POSTGRES_RUN_MANAGER_TLS_MODE must be verify-full or disable');
|
||||
}
|
||||
if (
|
||||
mode === 'disable' &&
|
||||
!bool(environment, 'QL3_POSTGRES_RUN_MANAGER_ALLOW_INSECURE')
|
||||
) {
|
||||
throw failure(
|
||||
'disabling run manager PostgreSQL TLS requires QL3_POSTGRES_RUN_MANAGER_ALLOW_INSECURE=true',
|
||||
);
|
||||
}
|
||||
const servername = bounded(
|
||||
environment,
|
||||
'QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME',
|
||||
253,
|
||||
);
|
||||
if (mode === 'verify-full' && !isPostgresTlsDnsServername(servername)) {
|
||||
throw failure(
|
||||
'QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME must be an explicit DNS name',
|
||||
);
|
||||
}
|
||||
const caFile = bounded(
|
||||
environment,
|
||||
'QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE',
|
||||
4_096,
|
||||
);
|
||||
if (mode === 'disable' && caFile !== undefined) {
|
||||
throw failure(
|
||||
'QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE cannot be used when TLS is disabled',
|
||||
);
|
||||
}
|
||||
let ca: string | undefined;
|
||||
if (caFile !== undefined) {
|
||||
try {
|
||||
ca = loadPostgresCertificateAuthorityFile(caFile);
|
||||
} catch {
|
||||
throw failure('QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE is invalid');
|
||||
}
|
||||
}
|
||||
const applicationName =
|
||||
bounded(environment, 'QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME', 63) ??
|
||||
'qinglong3-run-manager';
|
||||
if (!SAFE_APPLICATION_NAME.test(applicationName)) {
|
||||
throw failure('QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
connection: Object.freeze({
|
||||
...connection,
|
||||
tls:
|
||||
mode === 'disable'
|
||||
? Object.freeze({ mode: 'disable' as const })
|
||||
: Object.freeze({
|
||||
mode: 'verify-full' as const,
|
||||
servername: servername!,
|
||||
...(ca === undefined ? {} : { ca }),
|
||||
}),
|
||||
}),
|
||||
pool: Object.freeze({
|
||||
applicationName,
|
||||
maxConnections: integer(
|
||||
environment,
|
||||
'QL3_POSTGRES_RUN_MANAGER_POOL_MAX',
|
||||
2,
|
||||
1,
|
||||
4,
|
||||
),
|
||||
idleTimeoutMs: integer(
|
||||
environment,
|
||||
'QL3_POSTGRES_RUN_MANAGER_IDLE_TIMEOUT_MS',
|
||||
10_000,
|
||||
1_000,
|
||||
60_000,
|
||||
),
|
||||
connectionTimeoutMs: integer(
|
||||
environment,
|
||||
'QL3_POSTGRES_RUN_MANAGER_CONNECTION_TIMEOUT_MS',
|
||||
5_000,
|
||||
100,
|
||||
60_000,
|
||||
),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export function loadClusterRunManagementProcessConfig(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
): Readonly<ClusterRunManagementProcessConfig> {
|
||||
if (!environment || typeof environment !== 'object') {
|
||||
throw failure('environment is invalid');
|
||||
}
|
||||
if (!bool(environment, 'QL3_RUN_MANAGEMENT_ENABLED')) {
|
||||
return Object.freeze({ enabled: false as const });
|
||||
}
|
||||
if (environment.QL3_PROFILE !== 'cluster-admin') {
|
||||
throw failure('QL3_PROFILE must be cluster-admin when Run management is enabled');
|
||||
}
|
||||
const host = bounded(environment, 'QL3_RUN_MANAGEMENT_HOST', 255) ?? '0.0.0.0';
|
||||
if (!SAFE_HOST.test(host)) throw failure('QL3_RUN_MANAGEMENT_HOST is invalid');
|
||||
const http = Object.freeze({
|
||||
maxBodyBytes: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_BODY_BYTES', 32 * 1024, 1_024, 256 * 1024),
|
||||
maxConnections: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_CONNECTIONS', 32, 1, 512),
|
||||
maxConcurrentRequests: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_CONCURRENT_REQUESTS', 16, 1, 256),
|
||||
requestTimeoutMs: integer(environment, 'QL3_RUN_MANAGEMENT_REQUEST_TIMEOUT_MS', 10_000, 1_000, 60_000),
|
||||
drainTimeoutMs: integer(environment, 'QL3_RUN_MANAGEMENT_DRAIN_TIMEOUT_MS', 5_000, 100, 60_000),
|
||||
rateWindowMs: integer(environment, 'QL3_RUN_MANAGEMENT_RATE_WINDOW_MS', 60_000, 1_000, 5 * 60_000),
|
||||
peerRequestLimit: integer(environment, 'QL3_RUN_MANAGEMENT_PEER_REQUEST_LIMIT', 30, 1, 10_000),
|
||||
globalRequestLimit: integer(environment, 'QL3_RUN_MANAGEMENT_GLOBAL_REQUEST_LIMIT', 300, 1, 100_000),
|
||||
maxRateLimitPeers: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_RATE_LIMIT_PEERS', 1_024, 1, 16_384),
|
||||
});
|
||||
if (http.globalRequestLimit < http.peerRequestLimit) {
|
||||
throw failure('global request limit cannot be below peer request limit');
|
||||
}
|
||||
return Object.freeze({
|
||||
enabled: true as const,
|
||||
profile: 'cluster-admin' as const,
|
||||
host,
|
||||
port: integer(environment, 'QL3_RUN_MANAGEMENT_PORT', 8_448, 1, 65_535),
|
||||
certificateFile: absolute(environment, 'QL3_RUN_MANAGEMENT_TLS_CERT_FILE'),
|
||||
privateKeyFile: absolute(environment, 'QL3_RUN_MANAGEMENT_TLS_KEY_FILE'),
|
||||
clientCertificateAuthorityFile: absolute(environment, 'QL3_RUN_MANAGEMENT_CLIENT_CA_FILE'),
|
||||
clientCertificateRevocationListFile: absolute(environment, 'QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE'),
|
||||
identityKeysetFile: absolute(environment, 'QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE'),
|
||||
http,
|
||||
database: loadDatabase(environment),
|
||||
});
|
||||
}
|
||||
|
||||
export async function startClusterRunManagementProcess(
|
||||
options: StartClusterRunManagementProcessOptions,
|
||||
): Promise<Readonly<ClusterRunManagementProcessRuntime>> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some(
|
||||
(key) =>
|
||||
![
|
||||
'environment',
|
||||
'openDatabase',
|
||||
'identities',
|
||||
'assertReady',
|
||||
'startHttp',
|
||||
'now',
|
||||
'randomUuid',
|
||||
'onError',
|
||||
].includes(key),
|
||||
) ||
|
||||
!options.environment ||
|
||||
typeof options.environment !== 'object' ||
|
||||
(options.openDatabase !== undefined && typeof options.openDatabase !== 'function') ||
|
||||
(options.identities !== undefined &&
|
||||
(typeof options.identities.reload !== 'function' ||
|
||||
typeof options.identities.bind !== 'function')) ||
|
||||
(options.assertReady !== undefined && typeof options.assertReady !== 'function') ||
|
||||
(options.startHttp !== undefined && typeof options.startHttp !== 'function') ||
|
||||
(options.now !== undefined && typeof options.now !== 'function') ||
|
||||
(options.randomUuid !== undefined && typeof options.randomUuid !== 'function') ||
|
||||
(options.onError !== undefined && typeof options.onError !== 'function')
|
||||
) {
|
||||
throw failure('options are invalid');
|
||||
}
|
||||
const config = loadClusterRunManagementProcessConfig(options.environment);
|
||||
if (!config.enabled) {
|
||||
return Object.freeze({ status: 'disabled' as const, close: () => Promise.resolve() });
|
||||
}
|
||||
const now = options.now ?? Date.now;
|
||||
let http: Readonly<ClusterRunManagementHttpApplication> | undefined;
|
||||
let database: PostgresDatabaseResource | undefined;
|
||||
let unavailableError: unknown;
|
||||
let closePromise: Promise<void> | undefined;
|
||||
const report = (error: unknown): void => {
|
||||
try {
|
||||
options.onError?.(error);
|
||||
} catch {
|
||||
// Diagnostics never own availability or cleanup.
|
||||
}
|
||||
};
|
||||
const openDatabase =
|
||||
options.openDatabase ??
|
||||
createPostgresDatabaseOpener({
|
||||
role: 'run-manager',
|
||||
connection: config.database.connection,
|
||||
pool: config.database.pool,
|
||||
onPoolError(error) {
|
||||
const first = unavailableError === undefined;
|
||||
unavailableError ??= error;
|
||||
http?.withdraw(error);
|
||||
if (first) report(error);
|
||||
},
|
||||
});
|
||||
try {
|
||||
database = await openDatabase();
|
||||
const evidence = await (options.assertReady ?? assertPostgresRunManagerSchemaReady)(database.pool);
|
||||
if (unavailableError !== undefined) throw unavailableError;
|
||||
const identities =
|
||||
options.identities ??
|
||||
createClusterRunIdentityKeysetFile({
|
||||
filePath: config.identityKeysetFile,
|
||||
now,
|
||||
ledger: new PostgresRunManagementIdentityKeysetLedgerRepository(
|
||||
database.pool,
|
||||
'run-management',
|
||||
),
|
||||
});
|
||||
const identity = await identities.reload();
|
||||
const service = createClusterRunManagementService({
|
||||
pool: database.pool,
|
||||
now,
|
||||
...(options.randomUuid === undefined ? {} : { randomUuid: options.randomUuid }),
|
||||
});
|
||||
const transport = createClusterRunManagementTransport({ service, now });
|
||||
const privateKey = readManagementTlsFile(config.privateKeyFile, true, failure);
|
||||
try {
|
||||
const certificate = readManagementTlsFile(config.certificateFile, false, failure);
|
||||
const clientCertificateAuthority = readManagementTlsFile(
|
||||
config.clientCertificateAuthorityFile,
|
||||
false,
|
||||
failure,
|
||||
);
|
||||
const clientCertificateRevocationList = readManagementTlsFile(
|
||||
config.clientCertificateRevocationListFile,
|
||||
false,
|
||||
failure,
|
||||
);
|
||||
validateClusterManagementClientTrust(
|
||||
clientCertificateAuthority,
|
||||
clientCertificateRevocationList,
|
||||
now(),
|
||||
failure,
|
||||
);
|
||||
http = await (options.startHttp ?? startClusterRunManagementHttp)({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
tls: {
|
||||
privateKey,
|
||||
certificate,
|
||||
clientCertificateAuthority,
|
||||
clientCertificateRevocationList,
|
||||
},
|
||||
transport,
|
||||
identities,
|
||||
limits: config.http,
|
||||
now,
|
||||
onError: report,
|
||||
});
|
||||
} finally {
|
||||
privateKey.fill(0);
|
||||
}
|
||||
if (unavailableError !== undefined) http.withdraw(unavailableError);
|
||||
return Object.freeze({
|
||||
status: 'active' as const,
|
||||
address: http.address,
|
||||
database: evidence,
|
||||
identity,
|
||||
availabilityStatus: () => http?.availabilityStatus() ?? 'stopped',
|
||||
close(): Promise<void> {
|
||||
closePromise ??= (async () => {
|
||||
let primaryError: unknown;
|
||||
try {
|
||||
await http?.close();
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
}
|
||||
try {
|
||||
await database?.close();
|
||||
} catch (error) {
|
||||
primaryError ??= error;
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
})();
|
||||
return closePromise;
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
try {
|
||||
await http?.close();
|
||||
} catch {
|
||||
// Preserve startup failure.
|
||||
}
|
||||
try {
|
||||
await database?.close();
|
||||
} catch {
|
||||
// Preserve startup failure.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
import {
|
||||
createRunManualRetryResponseBody,
|
||||
parseRunManualRetryRequestBody,
|
||||
type RunManualRetryResponseBody,
|
||||
} from '@qinglong/runtime-core/run-manual-retry';
|
||||
import {
|
||||
normalizeSecurityPrincipal,
|
||||
type SecurityPrincipal,
|
||||
} from '@qinglong/runtime-core/security';
|
||||
import type { ClusterRunManagementService } from './runManagement';
|
||||
|
||||
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const UUID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
const STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']);
|
||||
|
||||
export type ClusterRunManagementCommand = Readonly<{
|
||||
schemaVersion: 1;
|
||||
operation: 'run.retry';
|
||||
request: Readonly<{
|
||||
projectId: string;
|
||||
sourceRunId: string;
|
||||
requestId: string;
|
||||
auditEventId: string;
|
||||
failureAuditEventId: string;
|
||||
body: Readonly<{
|
||||
schema: 'qinglong/run-manual-retry@v1';
|
||||
mutationId: string;
|
||||
expectedRunVersion: number;
|
||||
expectedRunStatus: 'failed' | 'cancelled' | 'timed_out';
|
||||
}>;
|
||||
}>;
|
||||
}>;
|
||||
|
||||
export type ClusterRunManagementTransportResult = Readonly<{
|
||||
schemaVersion: 1;
|
||||
operation: 'run.retry';
|
||||
retry: Readonly<RunManualRetryResponseBody>;
|
||||
}>;
|
||||
|
||||
export interface ClusterRunManagementAuthentication {
|
||||
authenticate(): Promise<Readonly<SecurityPrincipal> | null>;
|
||||
}
|
||||
|
||||
export interface ClusterRunManagementTransport {
|
||||
execute(
|
||||
command: unknown,
|
||||
authentication: ClusterRunManagementAuthentication,
|
||||
): Promise<Readonly<ClusterRunManagementTransportResult>>;
|
||||
}
|
||||
|
||||
export class ClusterRunManagementTransportConfigurationError extends TypeError {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_CONFIGURATION_INVALID';
|
||||
constructor() {
|
||||
super('Cluster Run management transport configuration is invalid');
|
||||
this.name = 'ClusterRunManagementTransportConfigurationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementTransportRequestError extends TypeError {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_REQUEST_INVALID';
|
||||
constructor() {
|
||||
super('Cluster Run management transport request is invalid');
|
||||
this.name = 'ClusterRunManagementTransportRequestError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementTransportAuthenticationError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_AUTHENTICATION_REQUIRED';
|
||||
constructor() {
|
||||
super('Cluster Run management transport requires a strong User principal');
|
||||
this.name = 'ClusterRunManagementTransportAuthenticationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementTransportUnavailableError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_UNAVAILABLE';
|
||||
constructor() {
|
||||
super('Cluster Run management transport is unavailable');
|
||||
this.name = 'ClusterRunManagementTransportUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function invalid(): never {
|
||||
throw new ClusterRunManagementTransportRequestError();
|
||||
}
|
||||
|
||||
function exact(value: unknown, keys: readonly string[]): Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) invalid();
|
||||
const actual = Object.keys(value as object).sort();
|
||||
const expected = [...keys].sort();
|
||||
if (
|
||||
actual.length !== expected.length ||
|
||||
actual.some((key, index) => key !== expected[index])
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function identifier(value: unknown): string {
|
||||
if (typeof value !== 'string' || !IDENTIFIER_PATTERN.test(value)) invalid();
|
||||
return value;
|
||||
}
|
||||
|
||||
function uuid(value: unknown): string {
|
||||
if (typeof value !== 'string' || !UUID_PATTERN.test(value)) invalid();
|
||||
return value;
|
||||
}
|
||||
|
||||
export function normalizeClusterRunManagementCommand(
|
||||
value: unknown,
|
||||
): Readonly<ClusterRunManagementCommand> {
|
||||
const envelope = exact(value, ['schemaVersion', 'operation', 'request']);
|
||||
if (envelope.schemaVersion !== 1 || envelope.operation !== 'run.retry') invalid();
|
||||
const request = exact(envelope.request, [
|
||||
'projectId',
|
||||
'sourceRunId',
|
||||
'requestId',
|
||||
'auditEventId',
|
||||
'failureAuditEventId',
|
||||
'body',
|
||||
]);
|
||||
let body: ReturnType<typeof parseRunManualRetryRequestBody>;
|
||||
try {
|
||||
body = parseRunManualRetryRequestBody(request.body);
|
||||
} catch {
|
||||
invalid();
|
||||
}
|
||||
const auditEventId = uuid(request.auditEventId);
|
||||
const failureAuditEventId = uuid(request.failureAuditEventId);
|
||||
if (auditEventId === failureAuditEventId) invalid();
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
request: Object.freeze({
|
||||
projectId: identifier(request.projectId),
|
||||
sourceRunId: identifier(request.sourceRunId),
|
||||
requestId: identifier(request.requestId),
|
||||
auditEventId,
|
||||
failureAuditEventId,
|
||||
body,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export function createClusterRunManagementTransport(options: Readonly<{
|
||||
service: ClusterRunManagementService;
|
||||
now?: () => number;
|
||||
}>): Readonly<ClusterRunManagementTransport> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some((key) => key !== 'service' && key !== 'now') ||
|
||||
!options.service ||
|
||||
typeof options.service.retry !== 'function' ||
|
||||
(options.now !== undefined && typeof options.now !== 'function')
|
||||
) {
|
||||
throw new ClusterRunManagementTransportConfigurationError();
|
||||
}
|
||||
const now = options.now ?? Date.now;
|
||||
return Object.freeze({
|
||||
async execute(
|
||||
commandValue: unknown,
|
||||
authentication: ClusterRunManagementAuthentication,
|
||||
) {
|
||||
const command = normalizeClusterRunManagementCommand(commandValue);
|
||||
if (
|
||||
!authentication ||
|
||||
typeof authentication !== 'object' ||
|
||||
Array.isArray(authentication) ||
|
||||
Object.keys(authentication).length !== 1 ||
|
||||
typeof authentication.authenticate !== 'function'
|
||||
) {
|
||||
throw new ClusterRunManagementTransportConfigurationError();
|
||||
}
|
||||
let candidate: Readonly<SecurityPrincipal> | null;
|
||||
try {
|
||||
candidate = await authentication.authenticate();
|
||||
} catch {
|
||||
throw new ClusterRunManagementTransportUnavailableError();
|
||||
}
|
||||
let principal: Readonly<SecurityPrincipal>;
|
||||
try {
|
||||
principal = normalizeSecurityPrincipal(candidate as SecurityPrincipal, now());
|
||||
} catch {
|
||||
throw new ClusterRunManagementTransportAuthenticationError();
|
||||
}
|
||||
if (
|
||||
principal.subject.type !== 'user' ||
|
||||
!STRONG_ASSURANCES.has(principal.assurance)
|
||||
) {
|
||||
throw new ClusterRunManagementTransportAuthenticationError();
|
||||
}
|
||||
const result = await options.service.retry({
|
||||
projectId: command.request.projectId,
|
||||
sourceRunId: command.request.sourceRunId,
|
||||
mutationId: command.request.body.mutationId,
|
||||
expectedRunVersion: command.request.body.expectedRunVersion,
|
||||
expectedRunStatus: command.request.body.expectedRunStatus,
|
||||
requestId: command.request.requestId,
|
||||
auditEventId: command.request.auditEventId,
|
||||
failureAuditEventId: command.request.failureAuditEventId,
|
||||
principal,
|
||||
});
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
retry: createRunManualRetryResponseBody(result),
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -163,6 +163,7 @@ function database(serverVersionNum = '160014') {
|
||||
'enforce_plugin_package_stage_provenance',
|
||||
'lock_active_plugin_package_project',
|
||||
'lock_approval_policy_fence',
|
||||
'lock_run_management_policy_fence',
|
||||
'plugin_package_lifecycle_blocking_runs',
|
||||
'plugin_package_automation_start_allowed',
|
||||
'plugin_package_run_start_allowed',
|
||||
|
||||
@@ -12,6 +12,7 @@ const {
|
||||
createClusterAutomationIdentityKeysetFile,
|
||||
createClusterApprovalIdentityKeysetFile,
|
||||
createClusterModelProviderCredentialIdentityKeysetFile,
|
||||
createClusterRunIdentityKeysetFile,
|
||||
} = require('@qinglong/cluster-admin/plugin-package-identity-keyset');
|
||||
|
||||
const NOW_MS = 1_700_000_000_000;
|
||||
@@ -216,6 +217,38 @@ function providerCredentialAssertion(key, overrides = {}) {
|
||||
).toString('base64url')}`;
|
||||
}
|
||||
|
||||
function runAssertion(key, overrides = {}) {
|
||||
const header = Buffer.from(
|
||||
JSON.stringify({
|
||||
alg: 'EdDSA',
|
||||
kid: key.kid,
|
||||
typ: 'ql3-run-management+jwt',
|
||||
}),
|
||||
).toString('base64url');
|
||||
const now = Math.floor(NOW_MS / 1000);
|
||||
const payload = Buffer.from(
|
||||
JSON.stringify({
|
||||
acr: 'urn:ql3:mfa',
|
||||
amr: ['pwd', 'otp'],
|
||||
aud: 'qinglong3-run-management',
|
||||
auth_time: now - 10,
|
||||
exp: now + 120,
|
||||
iat: now,
|
||||
iss: ISSUER,
|
||||
jti: `run-assertion-${key.kid}`,
|
||||
ql3_purpose: 'run-management',
|
||||
sub: 'run-operator-1',
|
||||
...overrides,
|
||||
}),
|
||||
).toString('base64url');
|
||||
const signed = `${header}.${payload}`;
|
||||
return `${signed}.${sign(
|
||||
null,
|
||||
Buffer.from(signed, 'ascii'),
|
||||
key.privateKey,
|
||||
).toString('base64url')}`;
|
||||
}
|
||||
|
||||
async function atomicWrite(filePath, document) {
|
||||
const nextPath = `${filePath}.next`;
|
||||
await writeFile(nextPath, `${JSON.stringify(document)}\n`, { mode: 0o644 });
|
||||
@@ -374,6 +407,34 @@ test('loads a provider credential keyset isolated by type, purpose and audience'
|
||||
});
|
||||
});
|
||||
|
||||
test('loads a Run keyset isolated from every other management purpose', async () => {
|
||||
await fixture(async ({ filePath }) => {
|
||||
const key = reviewedKey('run-identity-key-1');
|
||||
await atomicWrite(filePath, {
|
||||
...keyset(1, [key]),
|
||||
audience: 'qinglong3-run-management',
|
||||
});
|
||||
const provider = createClusterRunIdentityKeysetFile({
|
||||
filePath,
|
||||
now: () => NOW_MS,
|
||||
});
|
||||
const principal = await provider.bind(runAssertion(key)).authenticate();
|
||||
assert.deepEqual(principal.subject, {
|
||||
type: 'user',
|
||||
id: 'run-operator-1',
|
||||
});
|
||||
await assert.rejects(provider.bind(approvalAssertion(key)).authenticate(), {
|
||||
code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID',
|
||||
});
|
||||
await assert.rejects(
|
||||
provider
|
||||
.bind(runAssertion(key, { ql3_purpose: 'approval-management' }))
|
||||
.authenticate(),
|
||||
{ code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID' },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('supports overlap rotation then immediately revokes the previous key', async () => {
|
||||
await fixture(async ({ filePath }) => {
|
||||
const first = reviewedKey('issuer-key-1');
|
||||
|
||||
@@ -197,6 +197,7 @@ function database(serverVersionNum = '160014') {
|
||||
'plugin_package_tool_start_allowed',
|
||||
'plugin_package_workflow_admission_snapshot',
|
||||
'plugin_package_workflow_task_attempt_snapshot',
|
||||
'lock_run_management_policy_fence',
|
||||
].includes(functionName),
|
||||
isOwner: false,
|
||||
})),
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ClusterRunManagementAuthorizationError,
|
||||
createClusterRunManagementService,
|
||||
} = require('@qinglong/cluster-admin/run-management');
|
||||
|
||||
const NOW = 1_000_000;
|
||||
const SOURCE_DIGEST = 'a'.repeat(64);
|
||||
const EXECUTION_DIGEST = 'b'.repeat(64);
|
||||
const TASK_REVISION = `qltd:v1:7:${SOURCE_DIGEST}`;
|
||||
const GENERATED = [
|
||||
'019f9500-0000-4000-8000-000000000010',
|
||||
'019f9500-0000-4000-8000-000000000011',
|
||||
'019f9500-0000-4000-8000-000000000012',
|
||||
'019f9500-0000-4000-8000-000000000013',
|
||||
];
|
||||
|
||||
function request() {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
mutationId: '019f9500-0000-4000-8000-000000000001',
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
requestId: 'request-1',
|
||||
auditEventId: '019f9500-0000-4000-8000-000000000002',
|
||||
failureAuditEventId: '019f9500-0000-4000-8000-000000000003',
|
||||
principal: {
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
authenticationId: 'oidc:run-management-1',
|
||||
authenticatedAtMs: 999_000,
|
||||
expiresAtMs: 1_100_000,
|
||||
assurance: 'multi_factor',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function policyRow(role = 'operator') {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
projectName: 'Project 1',
|
||||
projectSlug: 'project-1',
|
||||
projectStatus: 'active',
|
||||
projectVersion: 2,
|
||||
projectCreatedAtMs: '1',
|
||||
projectUpdatedAtMs: '2',
|
||||
bindingProjectId: 'project-1',
|
||||
bindingSubjectType: 'user',
|
||||
bindingSubjectId: 'operator-1',
|
||||
bindingVersion: 3,
|
||||
bindingState: 'active',
|
||||
bindingRole: role,
|
||||
bindingMutationId: 'binding-3',
|
||||
bindingChangedByType: 'user',
|
||||
bindingChangedById: 'owner-1',
|
||||
bindingCreatedAtMs: '3',
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(role = 'operator') {
|
||||
const calls = [];
|
||||
const pool = {
|
||||
async query(sql, params = []) {
|
||||
const text = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ scope: 'pool', sql: text, params });
|
||||
if (text.includes('LEFT JOIN LATERAL')) return { rows: [policyRow(role)] };
|
||||
if (text.startsWith('INSERT INTO "ql3"."security_audit_events"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`unexpected pool query: ${text}`);
|
||||
},
|
||||
async connect() {
|
||||
return {
|
||||
async query(sql, params = []) {
|
||||
const text = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ scope: 'client', sql: text, params });
|
||||
if (
|
||||
text === 'BEGIN ISOLATION LEVEL SERIALIZABLE' ||
|
||||
text === 'COMMIT' ||
|
||||
text === 'ROLLBACK' ||
|
||||
text.startsWith('SELECT set_config')
|
||||
) return { rows: [], rowCount: 0 };
|
||||
if (text.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: NOW }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('lock_run_management_policy_fence')) {
|
||||
return { rows: [{ matches: true }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('idempotency_key = $2')) return { rows: [] };
|
||||
if (text.includes('WHERE run.id = $1')) {
|
||||
return {
|
||||
rows: [{
|
||||
projectId: 'project-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: TASK_REVISION,
|
||||
taskName: 'Task 1',
|
||||
taskSnapshotRef: TASK_REVISION,
|
||||
parentRunId: null,
|
||||
triggerType: 'task_start',
|
||||
executionOwner: 'runtime',
|
||||
inputRef: null,
|
||||
priority: 1,
|
||||
runStatus: 'failed',
|
||||
runVersion: 7,
|
||||
attemptExecutorType: 'remote_worker',
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."task_definitions"')) {
|
||||
return { rows: [{ enabled: true }] };
|
||||
}
|
||||
if (text.includes('task_execution_revisions')) {
|
||||
return { rows: [{ sourceContentDigest: SOURCE_DIGEST, contentDigest: EXECUTION_DIGEST }] };
|
||||
}
|
||||
if (text.startsWith('SELECT') && text.includes("trigger_type = 'run_manual_retry'")) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text.startsWith('INSERT INTO')) return { rows: [], rowCount: 1 };
|
||||
throw new Error(`unexpected client query: ${text}`);
|
||||
},
|
||||
release() {},
|
||||
};
|
||||
},
|
||||
};
|
||||
let index = 0;
|
||||
return {
|
||||
calls,
|
||||
service: createClusterRunManagementService({
|
||||
pool,
|
||||
now: () => NOW,
|
||||
randomUuid: () => GENERATED[index++],
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
test('authorizes run.retry and keeps all generated aggregate identities server-side', async () => {
|
||||
const { calls, service } = fixture();
|
||||
const result = await service.retry(request());
|
||||
assert.equal(result.status, 'accepted');
|
||||
assert.equal(result.runId, GENERATED[0]);
|
||||
assert.equal(result.attemptId, GENERATED[1]);
|
||||
const runInsert = calls.find(({ sql }) => sql.startsWith('INSERT INTO "ql3"."runs"'));
|
||||
assert.equal(runInsert.params[0], GENERATED[0]);
|
||||
assert.equal(runInsert.params.includes(GENERATED[2]), false);
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql.includes('lock_run_management_policy_fence')),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('denied policy writes only the caller-supplied failure audit', async () => {
|
||||
const { calls, service } = fixture('viewer');
|
||||
await assert.rejects(service.retry(request()), ClusterRunManagementAuthorizationError);
|
||||
const audits = calls.filter(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."security_audit_events"'),
|
||||
);
|
||||
assert.equal(audits.length, 1);
|
||||
assert.equal(audits[0].params[0], request().failureAuditEventId);
|
||||
assert.equal(calls.some(({ scope }) => scope === 'client'), false);
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
validateClusterRunManagementClientResult,
|
||||
} = require('@qinglong/cluster-admin/run-management-client');
|
||||
const {
|
||||
normalizeClusterRunManagementCommand,
|
||||
} = require('@qinglong/cluster-admin/run-management-transport');
|
||||
const {
|
||||
ClusterPluginPackageManagementClientRequestError,
|
||||
} = require('@qinglong/cluster-admin/plugin-package-management-client');
|
||||
|
||||
const command = normalizeClusterRunManagementCommand({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
requestId: 'request-1',
|
||||
auditEventId: '019f9400-0000-4000-8000-000000000001',
|
||||
failureAuditEventId: '019f9400-0000-4000-8000-000000000002',
|
||||
body: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
mutationId: '019f9400-0000-4000-8000-000000000003',
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
function response(overrides = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
retry: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
status: 'accepted',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
sourceRunStatus: 'failed',
|
||||
sourceRunVersion: 7,
|
||||
runId: '019f9400-0000-4000-8000-000000000010',
|
||||
retryOfRunId: 'source-run-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: `qltd:v1:1:${'a'.repeat(64)}`,
|
||||
attemptId: '019f9400-0000-4000-8000-000000000011',
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'remote_worker',
|
||||
executionRevisionDigest: 'b'.repeat(64),
|
||||
createdAtMs: 1_000_000,
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('validates one low-sensitive retry response against the request fence', () => {
|
||||
assert.deepEqual(validateClusterRunManagementClientResult(response(), command), response());
|
||||
});
|
||||
|
||||
test('rejects response target, execution placement and shape drift', () => {
|
||||
for (const candidate of [
|
||||
response({ projectId: 'project-2' }),
|
||||
response({ executorType: 'local_process' }),
|
||||
response({ sourceRunVersion: 8 }),
|
||||
{ ...response(), principal: { type: 'user', id: 'operator-1' } },
|
||||
]) {
|
||||
assert.throws(
|
||||
() => validateClusterRunManagementClientResult(candidate, command),
|
||||
ClusterPluginPackageManagementClientRequestError,
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,108 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { readFileSync } = require('node:fs');
|
||||
const { request: httpsRequest } = require('node:https');
|
||||
const { resolve } = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ClusterRunManagementRateLimitedError,
|
||||
} = require('@qinglong/cluster-admin/run-management');
|
||||
const {
|
||||
startClusterRunManagementHttp,
|
||||
} = require('@qinglong/cluster-admin/run-management-http');
|
||||
|
||||
const SERVER_KEY = resolve(__dirname, '../../ql3-cluster-control/test/fixtures/mtls/server-key.pem');
|
||||
const SERVER_CERT = resolve(__dirname, '../../ql3-cluster-control/test/fixtures/mtls/server-cert.pem');
|
||||
const PATH = '/api/v3/runs/management';
|
||||
|
||||
function post(port, path = PATH) {
|
||||
const body = Buffer.from(JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
requestId: 'request-1',
|
||||
auditEventId: '019f9600-0000-4000-8000-000000000001',
|
||||
failureAuditEventId: '019f9600-0000-4000-8000-000000000002',
|
||||
body: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
mutationId: '019f9600-0000-4000-8000-000000000003',
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
},
|
||||
},
|
||||
}));
|
||||
return new Promise((resolvePromise, reject) => {
|
||||
const outgoing = httpsRequest({
|
||||
host: '127.0.0.1',
|
||||
port,
|
||||
path,
|
||||
method: 'POST',
|
||||
rejectUnauthorized: false,
|
||||
agent: false,
|
||||
headers: {
|
||||
authorization: 'Bearer assertion',
|
||||
'content-type': 'application/json',
|
||||
'content-length': String(body.length),
|
||||
},
|
||||
}, (incoming) => {
|
||||
const chunks = [];
|
||||
incoming.on('data', (chunk) => chunks.push(chunk));
|
||||
incoming.once('end', () => {
|
||||
const bytes = Buffer.concat(chunks);
|
||||
resolvePromise({
|
||||
statusCode: incoming.statusCode,
|
||||
headers: incoming.headers,
|
||||
body: bytes.length ? JSON.parse(bytes.toString('utf8')) : null,
|
||||
});
|
||||
});
|
||||
});
|
||||
outgoing.once('error', reject);
|
||||
outgoing.end(body);
|
||||
});
|
||||
}
|
||||
|
||||
test('serves only the Run path and maps durable quota to bounded HTTP facts', async () => {
|
||||
const application = await startClusterRunManagementHttp({
|
||||
host: '127.0.0.1',
|
||||
port: 0,
|
||||
tls: {
|
||||
privateKey: Buffer.from(readFileSync(SERVER_KEY)),
|
||||
certificate: Buffer.from(readFileSync(SERVER_CERT)),
|
||||
},
|
||||
identities: {
|
||||
async reload() { throw new Error('not used'); },
|
||||
bind() {
|
||||
return { authenticate: async () => ({
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
authenticationId: 'oidc:run-management-1',
|
||||
authenticatedAtMs: 900,
|
||||
expiresAtMs: 10_000,
|
||||
assurance: 'hardware',
|
||||
}) };
|
||||
},
|
||||
},
|
||||
transport: {
|
||||
async execute(_command, authentication) {
|
||||
await authentication.authenticate();
|
||||
throw new ClusterRunManagementRateLimitedError(1_500);
|
||||
},
|
||||
},
|
||||
now: () => 1_000,
|
||||
});
|
||||
try {
|
||||
const limited = await post(application.address.port);
|
||||
assert.equal(limited.statusCode, 429);
|
||||
assert.equal(limited.headers['retry-after'], '2');
|
||||
assert.equal(limited.body.error.code, 'rate_limited');
|
||||
assert.match(limited.body.requestId, /^[0-9a-f-]{36}$/);
|
||||
const absent = await post(application.address.port, '/api/v3/approvals/management');
|
||||
assert.equal(absent.statusCode, 404);
|
||||
assert.equal(absent.body.error.code, 'not_found');
|
||||
} finally {
|
||||
await application.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ClusterRunManagementProcessConfigError,
|
||||
loadClusterRunManagementProcessConfig,
|
||||
startClusterRunManagementProcess,
|
||||
} = require('@qinglong/cluster-admin/run-management-process');
|
||||
|
||||
function enabled(overrides = {}) {
|
||||
return {
|
||||
QL3_RUN_MANAGEMENT_ENABLED: 'true',
|
||||
QL3_PROFILE: 'cluster-admin',
|
||||
QL3_RUN_MANAGEMENT_TLS_CERT_FILE: '/run/ql3/run/tls.crt',
|
||||
QL3_RUN_MANAGEMENT_TLS_KEY_FILE: '/run/ql3/run/tls.key',
|
||||
QL3_RUN_MANAGEMENT_CLIENT_CA_FILE: '/run/ql3/run/client-ca.crt',
|
||||
QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE: '/run/ql3/run/client.crl',
|
||||
QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE: '/run/ql3/run/identity.json',
|
||||
QL3_POSTGRES_RUN_MANAGER_HOST: 'postgres.qinglong3-system.svc',
|
||||
QL3_POSTGRES_RUN_MANAGER_DATABASE: 'qinglong3',
|
||||
QL3_POSTGRES_RUN_MANAGER_USER: 'ql3_run_manager',
|
||||
QL3_POSTGRES_RUN_MANAGER_PASSWORD: 'secret',
|
||||
QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME: 'postgres.qinglong3-system.svc',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('disabled Run manager acquires no PostgreSQL or file authority', async () => {
|
||||
let opened = false;
|
||||
const runtime = await startClusterRunManagementProcess({
|
||||
environment: {},
|
||||
openDatabase: async () => {
|
||||
opened = true;
|
||||
throw new Error('must not open');
|
||||
},
|
||||
});
|
||||
assert.equal(runtime.status, 'disabled');
|
||||
assert.equal(opened, false);
|
||||
await runtime.close();
|
||||
});
|
||||
|
||||
test('loads a bounded opt-in Run-only process configuration', () => {
|
||||
const config = loadClusterRunManagementProcessConfig(enabled());
|
||||
assert.equal(config.enabled, true);
|
||||
assert.equal(config.port, 8448);
|
||||
assert.equal(config.http.maxConcurrentRequests, 16);
|
||||
assert.equal(config.database.pool.maxConnections, 2);
|
||||
assert.equal(config.database.connection.user, 'ql3_run_manager');
|
||||
assert.deepEqual(config.database.connection.tls, {
|
||||
mode: 'verify-full',
|
||||
servername: 'postgres.qinglong3-system.svc',
|
||||
});
|
||||
});
|
||||
|
||||
test('rejects profile drift and implicit insecure PostgreSQL', () => {
|
||||
assert.throws(
|
||||
() => loadClusterRunManagementProcessConfig(enabled({ QL3_PROFILE: 'cluster-control' })),
|
||||
ClusterRunManagementProcessConfigError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
loadClusterRunManagementProcessConfig(
|
||||
enabled({
|
||||
QL3_POSTGRES_RUN_MANAGER_TLS_MODE: 'disable',
|
||||
QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME: undefined,
|
||||
}),
|
||||
),
|
||||
ClusterRunManagementProcessConfigError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,141 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ClusterRunManagementTransportAuthenticationError,
|
||||
ClusterRunManagementTransportRequestError,
|
||||
createClusterRunManagementTransport,
|
||||
normalizeClusterRunManagementCommand,
|
||||
} = require('@qinglong/cluster-admin/run-management-transport');
|
||||
|
||||
const NOW = 1_000_000;
|
||||
|
||||
function principal(overrides = {}) {
|
||||
return {
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
authenticationId: 'oidc:run-management-1',
|
||||
authenticatedAtMs: 999_000,
|
||||
expiresAtMs: 1_100_000,
|
||||
assurance: 'multi_factor',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function command(overrides = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
requestId: 'request-1',
|
||||
auditEventId: '019f9300-0000-4000-8000-000000000001',
|
||||
failureAuditEventId: '019f9300-0000-4000-8000-000000000002',
|
||||
body: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
mutationId: '019f9300-0000-4000-8000-000000000003',
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
},
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function retryResult() {
|
||||
return {
|
||||
status: 'accepted',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
sourceRunStatus: 'failed',
|
||||
sourceRunVersion: 7,
|
||||
runId: '019f9300-0000-4000-8000-000000000010',
|
||||
retryOfRunId: 'source-run-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: `qltd:v1:1:${'a'.repeat(64)}`,
|
||||
attemptId: '019f9300-0000-4000-8000-000000000011',
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'remote_worker',
|
||||
executionRevisionDigest: 'b'.repeat(64),
|
||||
createdAtMs: NOW,
|
||||
};
|
||||
}
|
||||
|
||||
test('routes one exact strong User retry and emits the shared response', async () => {
|
||||
const calls = [];
|
||||
const transport = createClusterRunManagementTransport({
|
||||
now: () => NOW,
|
||||
service: {
|
||||
async retry(request) {
|
||||
calls.push(request);
|
||||
return retryResult();
|
||||
},
|
||||
},
|
||||
});
|
||||
const result = await transport.execute(command(), {
|
||||
authenticate: async () => principal(),
|
||||
});
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(calls[0].mutationId, command().request.body.mutationId);
|
||||
assert.equal(calls[0].principal.assurance, 'multi_factor');
|
||||
assert.deepEqual(result, {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
retry: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
...retryResult(),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test('rejects weak or non-User identity before service authority', async () => {
|
||||
let called = false;
|
||||
const transport = createClusterRunManagementTransport({
|
||||
now: () => NOW,
|
||||
service: {
|
||||
async retry() {
|
||||
called = true;
|
||||
return retryResult();
|
||||
},
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
transport.execute(command(), {
|
||||
authenticate: async () => principal({ assurance: 'single_factor' }),
|
||||
}),
|
||||
ClusterRunManagementTransportAuthenticationError,
|
||||
);
|
||||
await assert.rejects(
|
||||
transport.execute(command(), {
|
||||
authenticate: async () =>
|
||||
principal({ subject: { type: 'agent', id: 'agent-1' } }),
|
||||
}),
|
||||
ClusterRunManagementTransportAuthenticationError,
|
||||
);
|
||||
assert.equal(called, false);
|
||||
});
|
||||
|
||||
test('rejects widened commands and ambiguous audit identity', () => {
|
||||
assert.throws(
|
||||
() => normalizeClusterRunManagementCommand({ ...command(), principal: principal() }),
|
||||
ClusterRunManagementTransportRequestError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeClusterRunManagementCommand(
|
||||
command({ failureAuditEventId: command().request.auditEventId }),
|
||||
),
|
||||
ClusterRunManagementTransportRequestError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeClusterRunManagementCommand(
|
||||
command({ body: { ...command().request.body, expectedRunStatus: 'lost' } }),
|
||||
),
|
||||
ClusterRunManagementTransportRequestError,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user