mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-23 03:18:09 +08:00
feat(ql3): add strong cluster run management
This commit is contained in:
whyour
@@ -20,6 +20,31 @@
|
||||
"require": "./dist/approval-management/approvalDecisionManagement.js",
|
||||
"default": "./dist/approval-management/approvalDecisionManagement.js"
|
||||
},
|
||||
"./run-management": {
|
||||
"types": "./dist/run-management/runManagement.d.ts",
|
||||
"require": "./dist/run-management/runManagement.js",
|
||||
"default": "./dist/run-management/runManagement.js"
|
||||
},
|
||||
"./run-management-transport": {
|
||||
"types": "./dist/run-management/runManagementTransport.d.ts",
|
||||
"require": "./dist/run-management/runManagementTransport.js",
|
||||
"default": "./dist/run-management/runManagementTransport.js"
|
||||
},
|
||||
"./run-management-http": {
|
||||
"types": "./dist/run-management/runManagementHttp.d.ts",
|
||||
"require": "./dist/run-management/runManagementHttp.js",
|
||||
"default": "./dist/run-management/runManagementHttp.js"
|
||||
},
|
||||
"./run-management-process": {
|
||||
"types": "./dist/run-management/runManagementProcess.d.ts",
|
||||
"require": "./dist/run-management/runManagementProcess.js",
|
||||
"default": "./dist/run-management/runManagementProcess.js"
|
||||
},
|
||||
"./run-management-client": {
|
||||
"types": "./dist/run-management/runManagementClient.d.ts",
|
||||
"require": "./dist/run-management/runManagementClient.js",
|
||||
"default": "./dist/run-management/runManagementClient.js"
|
||||
},
|
||||
"./approval-management": {
|
||||
"types": "./dist/approval-management/approvalManagement.d.ts",
|
||||
"require": "./dist/approval-management/approvalManagement.js",
|
||||
@@ -331,6 +356,8 @@
|
||||
"ql3-worker-credential-client": "dist/worker-credential/workerCredentialManagementClientCli.js",
|
||||
"ql3-approval-manage": "dist/approval-management/approvalManagementCli.js",
|
||||
"ql3-approval-client": "dist/approval-management/approvalManagementClientCli.js",
|
||||
"ql3-run-manage": "dist/run-management/runManagementCli.js",
|
||||
"ql3-run-client": "dist/run-management/runManagementClientCli.js",
|
||||
"ql3-automation-manage": "dist/automation-management/automationManagementCli.js",
|
||||
"ql3-automation-client": "dist/automation-management/automationManagementClientCli.js",
|
||||
"ql3-provider-credential-manage": "dist/model-provider-credential/modelProviderCredentialManagementCli.js",
|
||||
|
||||
@@ -80,6 +80,11 @@ export const CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_ASSERTION_PRO
|
||||
purpose: 'model-provider-credential-management',
|
||||
});
|
||||
|
||||
export const CLUSTER_RUN_MANAGEMENT_IDENTITY_ASSERTION_PROFILE = Object.freeze({
|
||||
type: 'ql3-run-management+jwt',
|
||||
purpose: 'run-management',
|
||||
});
|
||||
|
||||
export interface ClusterPluginPackageIdentityAssertionVerifierOptions {
|
||||
readonly issuer: string;
|
||||
readonly audience: string;
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
CLUSTER_AUTOMATION_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
CLUSTER_APPROVAL_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
CLUSTER_RUN_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
createClusterPluginPackageIdentityAssertionVerifier,
|
||||
type ClusterManagementIdentityAssertionProfile,
|
||||
type ClusterPluginPackageIdentityAssertionAuthentication,
|
||||
@@ -507,3 +508,12 @@ export function createClusterModelProviderCredentialIdentityKeysetFile(
|
||||
CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
});
|
||||
}
|
||||
|
||||
export function createClusterRunIdentityKeysetFile(
|
||||
options: ClusterWorkerCredentialIdentityKeysetFileOptions,
|
||||
): Readonly<ClusterPluginPackageIdentityKeysetFile> {
|
||||
return createClusterPluginPackageIdentityKeysetFile({
|
||||
...options,
|
||||
assertionProfile: CLUSTER_RUN_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -67,6 +67,19 @@ import {
|
||||
ClusterModelProviderCredentialManagementTransportRequestError,
|
||||
ClusterModelProviderCredentialManagementTransportUnavailableError,
|
||||
} from '../model-provider-credential/modelProviderCredentialManagementTransport';
|
||||
import {
|
||||
ClusterRunManagementAuthorizationError,
|
||||
ClusterRunManagementConflictError,
|
||||
ClusterRunManagementRateLimitedError,
|
||||
ClusterRunManagementRequestError,
|
||||
ClusterRunManagementTargetUnavailableError,
|
||||
ClusterRunManagementUnavailableError,
|
||||
} from '../run-management/runManagement';
|
||||
import {
|
||||
ClusterRunManagementTransportAuthenticationError,
|
||||
ClusterRunManagementTransportRequestError,
|
||||
ClusterRunManagementTransportUnavailableError,
|
||||
} from '../run-management/runManagementTransport';
|
||||
|
||||
export const CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH =
|
||||
'/api/v3/plugin-packages/management';
|
||||
@@ -77,18 +90,21 @@ export const CLUSTER_AUTOMATION_MANAGEMENT_PATH =
|
||||
export const CLUSTER_APPROVAL_MANAGEMENT_PATH = '/api/v3/approvals/management';
|
||||
export const CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH =
|
||||
'/api/v3/provider-credentials/management';
|
||||
export const CLUSTER_RUN_MANAGEMENT_PATH = '/api/v3/runs/management';
|
||||
export type ClusterAuthenticatedManagementPath =
|
||||
| typeof CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH
|
||||
| typeof CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH
|
||||
| typeof CLUSTER_AUTOMATION_MANAGEMENT_PATH
|
||||
| typeof CLUSTER_APPROVAL_MANAGEMENT_PATH
|
||||
| typeof CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH;
|
||||
| typeof CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH
|
||||
| typeof CLUSTER_RUN_MANAGEMENT_PATH;
|
||||
const MANAGEMENT_PATHS = new Set<ClusterAuthenticatedManagementPath>([
|
||||
CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH,
|
||||
CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH,
|
||||
CLUSTER_AUTOMATION_MANAGEMENT_PATH,
|
||||
CLUSTER_APPROVAL_MANAGEMENT_PATH,
|
||||
CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH,
|
||||
CLUSTER_RUN_MANAGEMENT_PATH,
|
||||
]);
|
||||
const DEFAULT_MAX_BODY_BYTES = 64 * 1024;
|
||||
const DEFAULT_MAX_CONNECTIONS = 64;
|
||||
@@ -531,7 +547,8 @@ function responseError(error: unknown): HttpRequestError {
|
||||
error instanceof ClusterApprovalManagementTransportAuthenticationError ||
|
||||
error instanceof
|
||||
ClusterModelProviderCredentialManagementTransportAuthenticationError ||
|
||||
error instanceof ClusterModelProviderCredentialManagementAuthenticationError
|
||||
error instanceof ClusterModelProviderCredentialManagementAuthenticationError ||
|
||||
error instanceof ClusterRunManagementTransportAuthenticationError
|
||||
) {
|
||||
return new HttpRequestError(401, 'authentication_required');
|
||||
}
|
||||
@@ -544,6 +561,8 @@ function responseError(error: unknown): HttpRequestError {
|
||||
error instanceof
|
||||
ClusterModelProviderCredentialManagementTransportRequestError ||
|
||||
error instanceof ClusterModelProviderCredentialManagementRequestError ||
|
||||
error instanceof ClusterRunManagementTransportRequestError ||
|
||||
error instanceof ClusterRunManagementRequestError ||
|
||||
error instanceof PluginPackageManagementRequestError ||
|
||||
error instanceof WorkerCredentialManagementRequestError
|
||||
) {
|
||||
@@ -554,7 +573,8 @@ function responseError(error: unknown): HttpRequestError {
|
||||
error instanceof WorkerCredentialManagementAuthorizationError ||
|
||||
error instanceof ClusterAutomationManagementAuthorizationError ||
|
||||
error instanceof ClusterApprovalManagementTransportAuthorizationError ||
|
||||
error instanceof ClusterModelProviderCredentialManagementAuthorizationError
|
||||
error instanceof ClusterModelProviderCredentialManagementAuthorizationError ||
|
||||
error instanceof ClusterRunManagementAuthorizationError
|
||||
) {
|
||||
return new HttpRequestError(403, 'forbidden');
|
||||
}
|
||||
@@ -563,7 +583,8 @@ function responseError(error: unknown): HttpRequestError {
|
||||
error instanceof WorkerCredentialManagementConflictError ||
|
||||
error instanceof ClusterAutomationManagementConflictError ||
|
||||
error instanceof ClusterApprovalManagementTransportConflictError ||
|
||||
error instanceof ClusterModelProviderCredentialManagementConflictError
|
||||
error instanceof ClusterModelProviderCredentialManagementConflictError ||
|
||||
error instanceof ClusterRunManagementConflictError
|
||||
) {
|
||||
return new HttpRequestError(409, 'conflict');
|
||||
}
|
||||
@@ -578,8 +599,12 @@ function responseError(error: unknown): HttpRequestError {
|
||||
) {
|
||||
return new HttpRequestError(429, 'quota_exceeded', error.retryAfterMs);
|
||||
}
|
||||
if (error instanceof ClusterRunManagementRateLimitedError) {
|
||||
return new HttpRequestError(429, 'rate_limited', error.retryAfterMs);
|
||||
}
|
||||
if (
|
||||
error instanceof ClusterApprovalManagementTransportTargetUnavailableError
|
||||
error instanceof ClusterApprovalManagementTransportTargetUnavailableError ||
|
||||
error instanceof ClusterRunManagementTargetUnavailableError
|
||||
) {
|
||||
return new HttpRequestError(404, 'not_found');
|
||||
}
|
||||
@@ -594,6 +619,8 @@ function responseError(error: unknown): HttpRequestError {
|
||||
error instanceof
|
||||
ClusterModelProviderCredentialManagementTransportUnavailableError ||
|
||||
error instanceof ClusterModelProviderCredentialManagementUnavailableError ||
|
||||
error instanceof ClusterRunManagementTransportUnavailableError ||
|
||||
error instanceof ClusterRunManagementUnavailableError ||
|
||||
error instanceof PluginPackageManagementUnavailableError ||
|
||||
error instanceof WorkerCredentialManagementUnavailableError
|
||||
) {
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
import {
|
||||
PostgresProjectPolicyRepository,
|
||||
PostgresRunManualRetryRepository,
|
||||
PostgresSecurityAuditRepository,
|
||||
} from '@qinglong/cluster-postgres/run-manager';
|
||||
import type { PostgresPool } from '@qinglong/runtime-core';
|
||||
import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy';
|
||||
import {
|
||||
InvalidRunManualRetryError,
|
||||
RunManualRetryFenceRejectedError,
|
||||
RunManualRetryNotFoundError,
|
||||
RunManualRetryRateLimitedError,
|
||||
RunManualRetryUnavailableError,
|
||||
type RunManualRetryResult,
|
||||
type RunManualRetrySourceStatus,
|
||||
} from '@qinglong/runtime-core/run-manual-retry';
|
||||
import {
|
||||
normalizeSecurityPrincipal,
|
||||
type SecurityPolicyFence,
|
||||
type SecurityPrincipal,
|
||||
} from '@qinglong/runtime-core/security';
|
||||
import { normalizeSecurityAuditRecord } from '@qinglong/runtime-core/security-audit';
|
||||
|
||||
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const UUID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
|
||||
export interface ClusterRunManagementRetryRequest {
|
||||
readonly projectId: string;
|
||||
readonly sourceRunId: string;
|
||||
readonly mutationId: string;
|
||||
readonly expectedRunVersion: number;
|
||||
readonly expectedRunStatus: RunManualRetrySourceStatus;
|
||||
readonly requestId: string;
|
||||
readonly auditEventId: string;
|
||||
readonly failureAuditEventId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
}
|
||||
|
||||
export interface ClusterRunManagementService {
|
||||
retry(
|
||||
request: Readonly<ClusterRunManagementRetryRequest>,
|
||||
): Promise<Readonly<RunManualRetryResult>>;
|
||||
}
|
||||
|
||||
export interface ClusterRunManagementOptions {
|
||||
readonly pool: PostgresPool;
|
||||
readonly now?: () => number;
|
||||
readonly randomUuid?: () => string;
|
||||
}
|
||||
|
||||
export class ClusterRunManagementConfigurationError extends TypeError {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_CONFIGURATION_INVALID';
|
||||
constructor() {
|
||||
super('Cluster Run management configuration is invalid');
|
||||
this.name = 'ClusterRunManagementConfigurationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementRequestError extends TypeError {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_REQUEST_INVALID';
|
||||
constructor() {
|
||||
super('Cluster Run management request is invalid');
|
||||
this.name = 'ClusterRunManagementRequestError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementAuthorizationError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_FORBIDDEN';
|
||||
constructor() {
|
||||
super('Cluster Run management is forbidden');
|
||||
this.name = 'ClusterRunManagementAuthorizationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementTargetUnavailableError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_TARGET_UNAVAILABLE';
|
||||
constructor() {
|
||||
super('Cluster Run management target is unavailable');
|
||||
this.name = 'ClusterRunManagementTargetUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementConflictError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_CONFLICT';
|
||||
constructor() {
|
||||
super('Cluster Run management conflicts with durable state');
|
||||
this.name = 'ClusterRunManagementConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementRateLimitedError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_RATE_LIMITED';
|
||||
constructor(readonly retryAfterMs: number) {
|
||||
super('Cluster Run management rate limit is exhausted');
|
||||
this.name = 'ClusterRunManagementRateLimitedError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementUnavailableError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_UNAVAILABLE';
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('Cluster Run management is unavailable', options);
|
||||
this.name = 'ClusterRunManagementUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function exactRequest(
|
||||
value: unknown,
|
||||
): asserts value is Readonly<ClusterRunManagementRetryRequest> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
Object.keys(value).sort().join('\0') !==
|
||||
[
|
||||
'auditEventId',
|
||||
'expectedRunStatus',
|
||||
'expectedRunVersion',
|
||||
'failureAuditEventId',
|
||||
'mutationId',
|
||||
'principal',
|
||||
'projectId',
|
||||
'requestId',
|
||||
'sourceRunId',
|
||||
]
|
||||
.sort()
|
||||
.join('\0')
|
||||
) {
|
||||
throw new ClusterRunManagementRequestError();
|
||||
}
|
||||
}
|
||||
|
||||
function validUuid(value: unknown): value is string {
|
||||
return typeof value === 'string' && UUID_PATTERN.test(value);
|
||||
}
|
||||
|
||||
function failureReason(error: unknown): string {
|
||||
if (error instanceof ClusterRunManagementAuthorizationError) {
|
||||
return 'authorization_rejected';
|
||||
}
|
||||
if (error instanceof RunManualRetryNotFoundError) return 'run_not_found';
|
||||
if (error instanceof RunManualRetryRateLimitedError) return 'rate_limited';
|
||||
if (error instanceof RunManualRetryFenceRejectedError) return error.reason;
|
||||
return 'management_unavailable';
|
||||
}
|
||||
|
||||
/** Strong OIDC Run management composition over one run-manager Pool. */
|
||||
export function createClusterRunManagementService(
|
||||
options: ClusterRunManagementOptions,
|
||||
): Readonly<ClusterRunManagementService> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some(
|
||||
(key) => key !== 'pool' && key !== 'now' && key !== 'randomUuid',
|
||||
) ||
|
||||
!options.pool ||
|
||||
typeof options.pool.query !== 'function' ||
|
||||
typeof options.pool.connect !== 'function' ||
|
||||
(options.now !== undefined && typeof options.now !== 'function') ||
|
||||
(options.randomUuid !== undefined && typeof options.randomUuid !== 'function')
|
||||
) {
|
||||
throw new ClusterRunManagementConfigurationError();
|
||||
}
|
||||
const now = options.now ?? Date.now;
|
||||
const createId = options.randomUuid ?? randomUUID;
|
||||
const policy = new ProjectPolicyEngine(
|
||||
new PostgresProjectPolicyRepository(options.pool),
|
||||
);
|
||||
const retries = new PostgresRunManualRetryRepository(options.pool);
|
||||
const audit = new PostgresSecurityAuditRepository(options.pool);
|
||||
|
||||
return Object.freeze({
|
||||
async retry(requestValue: Readonly<ClusterRunManagementRetryRequest>) {
|
||||
exactRequest(requestValue);
|
||||
const observedAtMs = now();
|
||||
let principal: Readonly<SecurityPrincipal>;
|
||||
if (
|
||||
!Number.isSafeInteger(observedAtMs) ||
|
||||
observedAtMs < 0 ||
|
||||
!IDENTIFIER_PATTERN.test(requestValue.projectId) ||
|
||||
!IDENTIFIER_PATTERN.test(requestValue.sourceRunId) ||
|
||||
!IDENTIFIER_PATTERN.test(requestValue.requestId) ||
|
||||
!validUuid(requestValue.mutationId) ||
|
||||
!validUuid(requestValue.auditEventId) ||
|
||||
!validUuid(requestValue.failureAuditEventId) ||
|
||||
requestValue.auditEventId === requestValue.failureAuditEventId
|
||||
) {
|
||||
throw new ClusterRunManagementRequestError();
|
||||
}
|
||||
try {
|
||||
principal = normalizeSecurityPrincipal(
|
||||
requestValue.principal,
|
||||
observedAtMs,
|
||||
);
|
||||
} catch {
|
||||
throw new ClusterRunManagementRequestError();
|
||||
}
|
||||
|
||||
let fence: Readonly<SecurityPolicyFence> | null = null;
|
||||
try {
|
||||
const decision = await policy.authorize(
|
||||
principal,
|
||||
requestValue.projectId,
|
||||
'run.retry',
|
||||
);
|
||||
fence = decision.fence;
|
||||
if (
|
||||
decision.effect !== 'allow' ||
|
||||
!fence ||
|
||||
fence.bindingVersion === null
|
||||
) {
|
||||
throw new ClusterRunManagementAuthorizationError();
|
||||
}
|
||||
return await retries.retryRun({
|
||||
projectId: requestValue.projectId,
|
||||
sourceRunId: requestValue.sourceRunId,
|
||||
mutationId: requestValue.mutationId,
|
||||
expectedRunVersion: requestValue.expectedRunVersion,
|
||||
expectedRunStatus: requestValue.expectedRunStatus,
|
||||
runId: createId(),
|
||||
attemptId: createId(),
|
||||
createdEventId: createId(),
|
||||
queuedEventId: createId(),
|
||||
auditEventId: requestValue.auditEventId,
|
||||
requestId: requestValue.requestId,
|
||||
principal,
|
||||
policyFence: fence,
|
||||
});
|
||||
} catch (error) {
|
||||
try {
|
||||
await audit.record(
|
||||
normalizeSecurityAuditRecord({
|
||||
eventId: requestValue.failureAuditEventId,
|
||||
requestId: requestValue.requestId,
|
||||
operationId: 'run.retry',
|
||||
projectId: requestValue.projectId,
|
||||
subject: principal.subject,
|
||||
authenticationId: principal.authenticationId,
|
||||
outcome: 'denied',
|
||||
reasons: [failureReason(error)],
|
||||
fence,
|
||||
occurredAtMs: observedAtMs,
|
||||
}),
|
||||
);
|
||||
} catch (auditError) {
|
||||
throw new ClusterRunManagementUnavailableError({ cause: auditError });
|
||||
}
|
||||
if (error instanceof ClusterRunManagementAuthorizationError) throw error;
|
||||
if (error instanceof InvalidRunManualRetryError) {
|
||||
throw new ClusterRunManagementRequestError();
|
||||
}
|
||||
if (error instanceof RunManualRetryNotFoundError) {
|
||||
throw new ClusterRunManagementTargetUnavailableError();
|
||||
}
|
||||
if (error instanceof RunManualRetryFenceRejectedError) {
|
||||
throw new ClusterRunManagementConflictError();
|
||||
}
|
||||
if (error instanceof RunManualRetryRateLimitedError) {
|
||||
throw new ClusterRunManagementRateLimitedError(error.retryAfterMs);
|
||||
}
|
||||
if (error instanceof RunManualRetryUnavailableError) {
|
||||
throw new ClusterRunManagementUnavailableError({ cause: error });
|
||||
}
|
||||
throw new ClusterRunManagementUnavailableError({ cause: error });
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import {
|
||||
startClusterRunManagementProcess,
|
||||
type ClusterRunManagementProcessRuntime,
|
||||
} from './runManagementProcess';
|
||||
|
||||
const USAGE = 'Usage: ql3-run-manage';
|
||||
|
||||
function fact(error: unknown): Readonly<Record<string, unknown>> {
|
||||
const candidate = error as { readonly name?: unknown; readonly code?: unknown };
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-run-management',
|
||||
event: 'management_failed',
|
||||
name: typeof candidate?.name === 'string' ? candidate.name : 'Error',
|
||||
...(typeof candidate?.code === 'string' ? { code: candidate.code } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
function emit(value: Readonly<Record<string, unknown>>): void {
|
||||
process.stdout.write(`${JSON.stringify(value)}\n`);
|
||||
}
|
||||
|
||||
async function run(argv: readonly string[]): Promise<void> {
|
||||
if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
|
||||
process.stdout.write(`${USAGE}\n`);
|
||||
return;
|
||||
}
|
||||
if (argv.length !== 0) {
|
||||
process.stderr.write(`${JSON.stringify({ code: 'QL3_RUN_MANAGEMENT_CLI_USAGE_INVALID', message: USAGE })}\n`);
|
||||
process.exitCode = 64;
|
||||
return;
|
||||
}
|
||||
let runtime: Readonly<ClusterRunManagementProcessRuntime>;
|
||||
try {
|
||||
runtime = await startClusterRunManagementProcess({
|
||||
environment: process.env,
|
||||
onError: () => emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_unavailable' }),
|
||||
});
|
||||
} catch (error) {
|
||||
process.stderr.write(`${JSON.stringify(fact(error))}\n`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
if (runtime.status === 'disabled') {
|
||||
emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_disabled' });
|
||||
return;
|
||||
}
|
||||
emit({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-run-management',
|
||||
event: 'management_started',
|
||||
address: runtime.address,
|
||||
identityGeneration: runtime.identity.generation,
|
||||
databaseContractVersion: runtime.database.contractVersion,
|
||||
databaseMigrationCount: runtime.database.migrationIds.length,
|
||||
});
|
||||
let stopping: Promise<void> | undefined;
|
||||
const stop = (): Promise<void> => {
|
||||
stopping ??= runtime.close().then(() => emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_stopped' }));
|
||||
return stopping;
|
||||
};
|
||||
for (const signal of ['SIGINT', 'SIGTERM'] as const) {
|
||||
process.once(signal, () => {
|
||||
void stop().then(
|
||||
() => { process.exitCode = 0; },
|
||||
(error) => { process.stderr.write(`${JSON.stringify(fact(error))}\n`); process.exitCode = 1; },
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void run(process.argv.slice(2));
|
||||
@@ -0,0 +1,106 @@
|
||||
import {
|
||||
RUN_MANUAL_RETRY_SCHEMA,
|
||||
normalizeRunManualRetryResult,
|
||||
} from '@qinglong/runtime-core/run-manual-retry';
|
||||
import {
|
||||
ClusterPluginPackageManagementClientRequestError,
|
||||
executeClusterAuthenticatedManagementClient,
|
||||
type ClusterAuthenticatedManagementClientResult,
|
||||
type ClusterPluginPackageManagementClientConnectionOptions,
|
||||
type ClusterPluginPackageManagementClientPaths,
|
||||
} from '../management-support/pluginPackageManagementClient';
|
||||
import {
|
||||
normalizeClusterRunManagementCommand,
|
||||
type ClusterRunManagementCommand,
|
||||
type ClusterRunManagementTransportResult,
|
||||
} from './runManagementTransport';
|
||||
|
||||
const MANAGEMENT_PATH = '/api/v3/runs/management';
|
||||
|
||||
export type ClusterRunManagementClientPaths =
|
||||
ClusterPluginPackageManagementClientPaths;
|
||||
export type ClusterRunManagementClientConnectionOptions =
|
||||
ClusterPluginPackageManagementClientConnectionOptions;
|
||||
export type ClusterRunManagementClientResult =
|
||||
ClusterAuthenticatedManagementClientResult<ClusterRunManagementTransportResult>;
|
||||
|
||||
function invalid(): never {
|
||||
throw new ClusterPluginPackageManagementClientRequestError();
|
||||
}
|
||||
|
||||
function exact(value: unknown, keys: readonly string[]): Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) invalid();
|
||||
const actual = Object.keys(value as object).sort();
|
||||
const expected = [...keys].sort();
|
||||
if (
|
||||
actual.length !== expected.length ||
|
||||
actual.some((key, index) => key !== expected[index])
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
export function validateClusterRunManagementClientResult(
|
||||
value: unknown,
|
||||
command: Readonly<ClusterRunManagementCommand>,
|
||||
): Readonly<ClusterRunManagementTransportResult> {
|
||||
const envelope = exact(value, ['schemaVersion', 'operation', 'retry']);
|
||||
if (envelope.schemaVersion !== 1 || envelope.operation !== 'run.retry') invalid();
|
||||
const retry = exact(envelope.retry, [
|
||||
'schema',
|
||||
'status',
|
||||
'projectId',
|
||||
'sourceRunId',
|
||||
'sourceRunStatus',
|
||||
'sourceRunVersion',
|
||||
'runId',
|
||||
'retryOfRunId',
|
||||
'taskId',
|
||||
'taskRevision',
|
||||
'attemptId',
|
||||
'runStatus',
|
||||
'runVersion',
|
||||
'eventSequence',
|
||||
'executorType',
|
||||
'executionRevisionDigest',
|
||||
'createdAtMs',
|
||||
]);
|
||||
if (retry.schema !== RUN_MANUAL_RETRY_SCHEMA) invalid();
|
||||
try {
|
||||
const { schema: _schema, ...result } = retry;
|
||||
const normalized = normalizeRunManualRetryResult(result as never);
|
||||
if (
|
||||
normalized.projectId !== command.request.projectId ||
|
||||
normalized.sourceRunId !== command.request.sourceRunId ||
|
||||
normalized.sourceRunVersion !== command.request.body.expectedRunVersion ||
|
||||
normalized.sourceRunStatus !== command.request.body.expectedRunStatus ||
|
||||
normalized.executorType !== 'remote_worker'
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
} catch {
|
||||
invalid();
|
||||
}
|
||||
return Object.freeze(
|
||||
envelope as unknown as ClusterRunManagementTransportResult,
|
||||
);
|
||||
}
|
||||
|
||||
const PROTOCOL = Object.freeze({
|
||||
managementPath: MANAGEMENT_PATH,
|
||||
clientCertificate: 'required' as const,
|
||||
normalizeCommand: normalizeClusterRunManagementCommand,
|
||||
validateResult: validateClusterRunManagementClientResult,
|
||||
});
|
||||
|
||||
export function executeClusterRunManagementClient(
|
||||
paths: ClusterRunManagementClientPaths,
|
||||
connectionOptions?: ClusterRunManagementClientConnectionOptions,
|
||||
): Promise<Readonly<ClusterRunManagementClientResult>> {
|
||||
return executeClusterAuthenticatedManagementClient(
|
||||
paths,
|
||||
PROTOCOL,
|
||||
connectionOptions,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { ClusterPluginPackageManagementClientRemoteError } from '../management-support/pluginPackageManagementClient';
|
||||
import { executeClusterRunManagementClient } from './runManagementClient';
|
||||
|
||||
const USAGE =
|
||||
'Usage: ql3-run-client --config=/absolute/client.json --command=/absolute/command.json --assertion=/absolute/assertion.jwt';
|
||||
|
||||
function argumentsFrom(argv: readonly string[]): Readonly<{
|
||||
configFile: string;
|
||||
commandFile: string;
|
||||
assertionFile: string;
|
||||
}> | null {
|
||||
if (argv.length !== 3) return null;
|
||||
const values = new Map<string, string>();
|
||||
for (const argument of argv) {
|
||||
const match = /^--(config|command|assertion)=(\/.+)$/.exec(argument);
|
||||
if (!match || values.has(match[1]!)) return null;
|
||||
values.set(match[1]!, match[2]!);
|
||||
}
|
||||
if (!values.has('config') || !values.has('command') || !values.has('assertion')) return null;
|
||||
return Object.freeze({
|
||||
configFile: values.get('config')!,
|
||||
commandFile: values.get('command')!,
|
||||
assertionFile: values.get('assertion')!,
|
||||
});
|
||||
}
|
||||
|
||||
function failureFact(error: unknown): Readonly<Record<string, unknown>> {
|
||||
const candidate = error as { readonly code?: unknown };
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-run-management-client',
|
||||
event: 'command_failed',
|
||||
code: typeof candidate?.code === 'string' ? candidate.code : 'QL3_RUN_MANAGEMENT_CLIENT_FAILED',
|
||||
...(error instanceof ClusterPluginPackageManagementClientRemoteError
|
||||
? {
|
||||
statusCode: error.statusCode,
|
||||
responseCode: error.responseCode,
|
||||
requestId: error.requestId,
|
||||
...(error.retryAfterSeconds === null ? {} : { retryAfterSeconds: error.retryAfterSeconds }),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
async function run(argv: readonly string[]): Promise<void> {
|
||||
if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
|
||||
process.stdout.write(`${USAGE}\n`);
|
||||
return;
|
||||
}
|
||||
const paths = argumentsFrom(argv);
|
||||
if (!paths) {
|
||||
process.stderr.write(`${JSON.stringify({ schemaVersion: 1, component: 'qinglong3-run-management-client', event: 'usage_invalid', code: 'QL3_RUN_MANAGEMENT_CLIENT_USAGE_INVALID' })}\n`);
|
||||
process.exitCode = 64;
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const result = await executeClusterRunManagementClient(paths);
|
||||
process.stdout.write(`${JSON.stringify({ schemaVersion: 1, component: 'qinglong3-run-management-client', event: 'command_completed', requestId: result.requestId, result: result.result })}\n`);
|
||||
} catch (error) {
|
||||
process.stderr.write(`${JSON.stringify(failureFact(error))}\n`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
void run(process.argv.slice(2));
|
||||
@@ -0,0 +1,24 @@
|
||||
import {
|
||||
CLUSTER_RUN_MANAGEMENT_PATH,
|
||||
startClusterPluginPackageManagementHttp,
|
||||
type ClusterPluginPackageManagementHttpApplication,
|
||||
type StartClusterPluginPackageManagementHttpOptions,
|
||||
} from '../management-support/pluginPackageManagementHttp';
|
||||
|
||||
export type ClusterRunManagementHttpApplication =
|
||||
ClusterPluginPackageManagementHttpApplication;
|
||||
|
||||
export type StartClusterRunManagementHttpOptions = Omit<
|
||||
StartClusterPluginPackageManagementHttpOptions,
|
||||
'managementPath'
|
||||
>;
|
||||
|
||||
/** Starts the shared bounded OIDC/mTLS HTTPS adapter on the Run-only path. */
|
||||
export function startClusterRunManagementHttp(
|
||||
options: StartClusterRunManagementHttpOptions,
|
||||
): Promise<Readonly<ClusterRunManagementHttpApplication>> {
|
||||
return startClusterPluginPackageManagementHttp({
|
||||
...options,
|
||||
managementPath: CLUSTER_RUN_MANAGEMENT_PATH,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,467 @@
|
||||
import type {
|
||||
OpenPostgresDatabase,
|
||||
PostgresDatabaseResource,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
PostgresRunManagementIdentityKeysetLedgerRepository,
|
||||
assertPostgresRunManagerSchemaReady,
|
||||
createPostgresDatabaseOpener,
|
||||
isPostgresTlsDnsServername,
|
||||
loadPostgresCertificateAuthorityFile,
|
||||
loadPostgresConnectionEnvironment,
|
||||
type PostgresConnectionOptions,
|
||||
type PostgresPoolOptions,
|
||||
type PostgresSchemaReadinessReport,
|
||||
} from '@qinglong/cluster-postgres/run-manager';
|
||||
import {
|
||||
absoluteManagementEnvironmentFile,
|
||||
booleanManagementEnvironmentValue,
|
||||
boundedManagementEnvironmentValue,
|
||||
integerManagementEnvironmentValue,
|
||||
readManagementTlsFile,
|
||||
} from '../management-support/managementProcessSupport';
|
||||
import {
|
||||
createClusterRunIdentityKeysetFile,
|
||||
type ClusterPluginPackageIdentityKeysetFile,
|
||||
type ClusterPluginPackageIdentityKeysetSnapshot,
|
||||
} from '../management-support/pluginPackageIdentityKeyset';
|
||||
import { validateClusterManagementClientTrust } from '../worker-credential/management-server/workerCredentialManagementMutualTls';
|
||||
import { createClusterRunManagementService } from './runManagement';
|
||||
import {
|
||||
startClusterRunManagementHttp,
|
||||
type ClusterRunManagementHttpApplication,
|
||||
type StartClusterRunManagementHttpOptions,
|
||||
} from './runManagementHttp';
|
||||
import { createClusterRunManagementTransport } from './runManagementTransport';
|
||||
|
||||
const SAFE_HOST = /^[A-Za-z0-9][A-Za-z0-9.:-]{0,254}$/;
|
||||
const SAFE_APPLICATION_NAME = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,62}$/;
|
||||
|
||||
export type ClusterRunManagementProcessEnvironment = Readonly<
|
||||
Record<string, string | undefined>
|
||||
>;
|
||||
|
||||
export type ClusterRunManagementProcessConfig =
|
||||
| Readonly<{ enabled: false }>
|
||||
| Readonly<{
|
||||
enabled: true;
|
||||
profile: 'cluster-admin';
|
||||
host: string;
|
||||
port: number;
|
||||
certificateFile: string;
|
||||
privateKeyFile: string;
|
||||
clientCertificateAuthorityFile: string;
|
||||
clientCertificateRevocationListFile: string;
|
||||
identityKeysetFile: string;
|
||||
http: Readonly<{
|
||||
maxBodyBytes: number;
|
||||
maxConnections: number;
|
||||
maxConcurrentRequests: number;
|
||||
requestTimeoutMs: number;
|
||||
drainTimeoutMs: number;
|
||||
rateWindowMs: number;
|
||||
peerRequestLimit: number;
|
||||
globalRequestLimit: number;
|
||||
maxRateLimitPeers: number;
|
||||
}>;
|
||||
database: Readonly<{
|
||||
connection: PostgresConnectionOptions;
|
||||
pool: PostgresPoolOptions;
|
||||
}>;
|
||||
}>;
|
||||
|
||||
export type ClusterRunManagementProcessRuntime =
|
||||
| Readonly<{ status: 'disabled'; close(): Promise<void> }>
|
||||
| Readonly<{
|
||||
status: 'active';
|
||||
address: Readonly<{ host: string; port: number }>;
|
||||
database: PostgresSchemaReadinessReport;
|
||||
identity: ClusterPluginPackageIdentityKeysetSnapshot;
|
||||
availabilityStatus(): 'ready' | 'unavailable' | 'stopped';
|
||||
close(): Promise<void>;
|
||||
}>;
|
||||
|
||||
export interface StartClusterRunManagementProcessOptions {
|
||||
readonly environment: ClusterRunManagementProcessEnvironment;
|
||||
readonly openDatabase?: OpenPostgresDatabase;
|
||||
readonly identities?: ClusterPluginPackageIdentityKeysetFile;
|
||||
readonly assertReady?: (
|
||||
pool: PostgresDatabaseResource['pool'],
|
||||
) => Promise<PostgresSchemaReadinessReport>;
|
||||
readonly startHttp?: (
|
||||
options: StartClusterRunManagementHttpOptions,
|
||||
) => Promise<Readonly<ClusterRunManagementHttpApplication>>;
|
||||
readonly now?: () => number;
|
||||
readonly randomUuid?: () => string;
|
||||
readonly onError?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export class ClusterRunManagementProcessConfigError extends TypeError {
|
||||
readonly code = 'QL3_RUN_MANAGEMENT_PROCESS_CONFIG_INVALID';
|
||||
constructor(message: string) {
|
||||
super(`Run management process configuration is invalid: ${message}`);
|
||||
this.name = 'ClusterRunManagementProcessConfigError';
|
||||
}
|
||||
}
|
||||
|
||||
function failure(message: string): ClusterRunManagementProcessConfigError {
|
||||
return new ClusterRunManagementProcessConfigError(message);
|
||||
}
|
||||
|
||||
function bounded(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
name: string,
|
||||
maximumLength: number,
|
||||
required = false,
|
||||
): string | undefined {
|
||||
return boundedManagementEnvironmentValue(
|
||||
environment,
|
||||
name,
|
||||
maximumLength,
|
||||
failure,
|
||||
required,
|
||||
);
|
||||
}
|
||||
|
||||
function bool(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
name: string,
|
||||
): boolean {
|
||||
return booleanManagementEnvironmentValue(environment, name, failure);
|
||||
}
|
||||
|
||||
function integer(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
name: string,
|
||||
fallback: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): number {
|
||||
return integerManagementEnvironmentValue(
|
||||
environment,
|
||||
name,
|
||||
fallback,
|
||||
minimum,
|
||||
maximum,
|
||||
failure,
|
||||
);
|
||||
}
|
||||
|
||||
function absolute(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
name: string,
|
||||
): string {
|
||||
return absoluteManagementEnvironmentFile(environment, name, failure);
|
||||
}
|
||||
|
||||
function loadDatabase(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
): Readonly<{
|
||||
connection: PostgresConnectionOptions;
|
||||
pool: PostgresPoolOptions;
|
||||
}> {
|
||||
let connection: PostgresConnectionOptions;
|
||||
try {
|
||||
connection = loadPostgresConnectionEnvironment(environment, {
|
||||
connectionString: 'QL3_POSTGRES_RUN_MANAGER_URL',
|
||||
host: 'QL3_POSTGRES_RUN_MANAGER_HOST',
|
||||
port: 'QL3_POSTGRES_RUN_MANAGER_PORT',
|
||||
database: 'QL3_POSTGRES_RUN_MANAGER_DATABASE',
|
||||
user: 'QL3_POSTGRES_RUN_MANAGER_USER',
|
||||
password: 'QL3_POSTGRES_RUN_MANAGER_PASSWORD',
|
||||
});
|
||||
} catch (error) {
|
||||
throw failure(
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: 'PostgreSQL run manager connection is invalid',
|
||||
);
|
||||
}
|
||||
const mode = environment.QL3_POSTGRES_RUN_MANAGER_TLS_MODE ?? 'verify-full';
|
||||
if (mode !== 'verify-full' && mode !== 'disable') {
|
||||
throw failure('QL3_POSTGRES_RUN_MANAGER_TLS_MODE must be verify-full or disable');
|
||||
}
|
||||
if (
|
||||
mode === 'disable' &&
|
||||
!bool(environment, 'QL3_POSTGRES_RUN_MANAGER_ALLOW_INSECURE')
|
||||
) {
|
||||
throw failure(
|
||||
'disabling run manager PostgreSQL TLS requires QL3_POSTGRES_RUN_MANAGER_ALLOW_INSECURE=true',
|
||||
);
|
||||
}
|
||||
const servername = bounded(
|
||||
environment,
|
||||
'QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME',
|
||||
253,
|
||||
);
|
||||
if (mode === 'verify-full' && !isPostgresTlsDnsServername(servername)) {
|
||||
throw failure(
|
||||
'QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME must be an explicit DNS name',
|
||||
);
|
||||
}
|
||||
const caFile = bounded(
|
||||
environment,
|
||||
'QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE',
|
||||
4_096,
|
||||
);
|
||||
if (mode === 'disable' && caFile !== undefined) {
|
||||
throw failure(
|
||||
'QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE cannot be used when TLS is disabled',
|
||||
);
|
||||
}
|
||||
let ca: string | undefined;
|
||||
if (caFile !== undefined) {
|
||||
try {
|
||||
ca = loadPostgresCertificateAuthorityFile(caFile);
|
||||
} catch {
|
||||
throw failure('QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE is invalid');
|
||||
}
|
||||
}
|
||||
const applicationName =
|
||||
bounded(environment, 'QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME', 63) ??
|
||||
'qinglong3-run-manager';
|
||||
if (!SAFE_APPLICATION_NAME.test(applicationName)) {
|
||||
throw failure('QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
connection: Object.freeze({
|
||||
...connection,
|
||||
tls:
|
||||
mode === 'disable'
|
||||
? Object.freeze({ mode: 'disable' as const })
|
||||
: Object.freeze({
|
||||
mode: 'verify-full' as const,
|
||||
servername: servername!,
|
||||
...(ca === undefined ? {} : { ca }),
|
||||
}),
|
||||
}),
|
||||
pool: Object.freeze({
|
||||
applicationName,
|
||||
maxConnections: integer(
|
||||
environment,
|
||||
'QL3_POSTGRES_RUN_MANAGER_POOL_MAX',
|
||||
2,
|
||||
1,
|
||||
4,
|
||||
),
|
||||
idleTimeoutMs: integer(
|
||||
environment,
|
||||
'QL3_POSTGRES_RUN_MANAGER_IDLE_TIMEOUT_MS',
|
||||
10_000,
|
||||
1_000,
|
||||
60_000,
|
||||
),
|
||||
connectionTimeoutMs: integer(
|
||||
environment,
|
||||
'QL3_POSTGRES_RUN_MANAGER_CONNECTION_TIMEOUT_MS',
|
||||
5_000,
|
||||
100,
|
||||
60_000,
|
||||
),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export function loadClusterRunManagementProcessConfig(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
): Readonly<ClusterRunManagementProcessConfig> {
|
||||
if (!environment || typeof environment !== 'object') {
|
||||
throw failure('environment is invalid');
|
||||
}
|
||||
if (!bool(environment, 'QL3_RUN_MANAGEMENT_ENABLED')) {
|
||||
return Object.freeze({ enabled: false as const });
|
||||
}
|
||||
if (environment.QL3_PROFILE !== 'cluster-admin') {
|
||||
throw failure('QL3_PROFILE must be cluster-admin when Run management is enabled');
|
||||
}
|
||||
const host = bounded(environment, 'QL3_RUN_MANAGEMENT_HOST', 255) ?? '0.0.0.0';
|
||||
if (!SAFE_HOST.test(host)) throw failure('QL3_RUN_MANAGEMENT_HOST is invalid');
|
||||
const http = Object.freeze({
|
||||
maxBodyBytes: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_BODY_BYTES', 32 * 1024, 1_024, 256 * 1024),
|
||||
maxConnections: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_CONNECTIONS', 32, 1, 512),
|
||||
maxConcurrentRequests: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_CONCURRENT_REQUESTS', 16, 1, 256),
|
||||
requestTimeoutMs: integer(environment, 'QL3_RUN_MANAGEMENT_REQUEST_TIMEOUT_MS', 10_000, 1_000, 60_000),
|
||||
drainTimeoutMs: integer(environment, 'QL3_RUN_MANAGEMENT_DRAIN_TIMEOUT_MS', 5_000, 100, 60_000),
|
||||
rateWindowMs: integer(environment, 'QL3_RUN_MANAGEMENT_RATE_WINDOW_MS', 60_000, 1_000, 5 * 60_000),
|
||||
peerRequestLimit: integer(environment, 'QL3_RUN_MANAGEMENT_PEER_REQUEST_LIMIT', 30, 1, 10_000),
|
||||
globalRequestLimit: integer(environment, 'QL3_RUN_MANAGEMENT_GLOBAL_REQUEST_LIMIT', 300, 1, 100_000),
|
||||
maxRateLimitPeers: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_RATE_LIMIT_PEERS', 1_024, 1, 16_384),
|
||||
});
|
||||
if (http.globalRequestLimit < http.peerRequestLimit) {
|
||||
throw failure('global request limit cannot be below peer request limit');
|
||||
}
|
||||
return Object.freeze({
|
||||
enabled: true as const,
|
||||
profile: 'cluster-admin' as const,
|
||||
host,
|
||||
port: integer(environment, 'QL3_RUN_MANAGEMENT_PORT', 8_448, 1, 65_535),
|
||||
certificateFile: absolute(environment, 'QL3_RUN_MANAGEMENT_TLS_CERT_FILE'),
|
||||
privateKeyFile: absolute(environment, 'QL3_RUN_MANAGEMENT_TLS_KEY_FILE'),
|
||||
clientCertificateAuthorityFile: absolute(environment, 'QL3_RUN_MANAGEMENT_CLIENT_CA_FILE'),
|
||||
clientCertificateRevocationListFile: absolute(environment, 'QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE'),
|
||||
identityKeysetFile: absolute(environment, 'QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE'),
|
||||
http,
|
||||
database: loadDatabase(environment),
|
||||
});
|
||||
}
|
||||
|
||||
export async function startClusterRunManagementProcess(
|
||||
options: StartClusterRunManagementProcessOptions,
|
||||
): Promise<Readonly<ClusterRunManagementProcessRuntime>> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some(
|
||||
(key) =>
|
||||
![
|
||||
'environment',
|
||||
'openDatabase',
|
||||
'identities',
|
||||
'assertReady',
|
||||
'startHttp',
|
||||
'now',
|
||||
'randomUuid',
|
||||
'onError',
|
||||
].includes(key),
|
||||
) ||
|
||||
!options.environment ||
|
||||
typeof options.environment !== 'object' ||
|
||||
(options.openDatabase !== undefined && typeof options.openDatabase !== 'function') ||
|
||||
(options.identities !== undefined &&
|
||||
(typeof options.identities.reload !== 'function' ||
|
||||
typeof options.identities.bind !== 'function')) ||
|
||||
(options.assertReady !== undefined && typeof options.assertReady !== 'function') ||
|
||||
(options.startHttp !== undefined && typeof options.startHttp !== 'function') ||
|
||||
(options.now !== undefined && typeof options.now !== 'function') ||
|
||||
(options.randomUuid !== undefined && typeof options.randomUuid !== 'function') ||
|
||||
(options.onError !== undefined && typeof options.onError !== 'function')
|
||||
) {
|
||||
throw failure('options are invalid');
|
||||
}
|
||||
const config = loadClusterRunManagementProcessConfig(options.environment);
|
||||
if (!config.enabled) {
|
||||
return Object.freeze({ status: 'disabled' as const, close: () => Promise.resolve() });
|
||||
}
|
||||
const now = options.now ?? Date.now;
|
||||
let http: Readonly<ClusterRunManagementHttpApplication> | undefined;
|
||||
let database: PostgresDatabaseResource | undefined;
|
||||
let unavailableError: unknown;
|
||||
let closePromise: Promise<void> | undefined;
|
||||
const report = (error: unknown): void => {
|
||||
try {
|
||||
options.onError?.(error);
|
||||
} catch {
|
||||
// Diagnostics never own availability or cleanup.
|
||||
}
|
||||
};
|
||||
const openDatabase =
|
||||
options.openDatabase ??
|
||||
createPostgresDatabaseOpener({
|
||||
role: 'run-manager',
|
||||
connection: config.database.connection,
|
||||
pool: config.database.pool,
|
||||
onPoolError(error) {
|
||||
const first = unavailableError === undefined;
|
||||
unavailableError ??= error;
|
||||
http?.withdraw(error);
|
||||
if (first) report(error);
|
||||
},
|
||||
});
|
||||
try {
|
||||
database = await openDatabase();
|
||||
const evidence = await (options.assertReady ?? assertPostgresRunManagerSchemaReady)(database.pool);
|
||||
if (unavailableError !== undefined) throw unavailableError;
|
||||
const identities =
|
||||
options.identities ??
|
||||
createClusterRunIdentityKeysetFile({
|
||||
filePath: config.identityKeysetFile,
|
||||
now,
|
||||
ledger: new PostgresRunManagementIdentityKeysetLedgerRepository(
|
||||
database.pool,
|
||||
'run-management',
|
||||
),
|
||||
});
|
||||
const identity = await identities.reload();
|
||||
const service = createClusterRunManagementService({
|
||||
pool: database.pool,
|
||||
now,
|
||||
...(options.randomUuid === undefined ? {} : { randomUuid: options.randomUuid }),
|
||||
});
|
||||
const transport = createClusterRunManagementTransport({ service, now });
|
||||
const privateKey = readManagementTlsFile(config.privateKeyFile, true, failure);
|
||||
try {
|
||||
const certificate = readManagementTlsFile(config.certificateFile, false, failure);
|
||||
const clientCertificateAuthority = readManagementTlsFile(
|
||||
config.clientCertificateAuthorityFile,
|
||||
false,
|
||||
failure,
|
||||
);
|
||||
const clientCertificateRevocationList = readManagementTlsFile(
|
||||
config.clientCertificateRevocationListFile,
|
||||
false,
|
||||
failure,
|
||||
);
|
||||
validateClusterManagementClientTrust(
|
||||
clientCertificateAuthority,
|
||||
clientCertificateRevocationList,
|
||||
now(),
|
||||
failure,
|
||||
);
|
||||
http = await (options.startHttp ?? startClusterRunManagementHttp)({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
tls: {
|
||||
privateKey,
|
||||
certificate,
|
||||
clientCertificateAuthority,
|
||||
clientCertificateRevocationList,
|
||||
},
|
||||
transport,
|
||||
identities,
|
||||
limits: config.http,
|
||||
now,
|
||||
onError: report,
|
||||
});
|
||||
} finally {
|
||||
privateKey.fill(0);
|
||||
}
|
||||
if (unavailableError !== undefined) http.withdraw(unavailableError);
|
||||
return Object.freeze({
|
||||
status: 'active' as const,
|
||||
address: http.address,
|
||||
database: evidence,
|
||||
identity,
|
||||
availabilityStatus: () => http?.availabilityStatus() ?? 'stopped',
|
||||
close(): Promise<void> {
|
||||
closePromise ??= (async () => {
|
||||
let primaryError: unknown;
|
||||
try {
|
||||
await http?.close();
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
}
|
||||
try {
|
||||
await database?.close();
|
||||
} catch (error) {
|
||||
primaryError ??= error;
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
})();
|
||||
return closePromise;
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
try {
|
||||
await http?.close();
|
||||
} catch {
|
||||
// Preserve startup failure.
|
||||
}
|
||||
try {
|
||||
await database?.close();
|
||||
} catch {
|
||||
// Preserve startup failure.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
import {
|
||||
createRunManualRetryResponseBody,
|
||||
parseRunManualRetryRequestBody,
|
||||
type RunManualRetryResponseBody,
|
||||
} from '@qinglong/runtime-core/run-manual-retry';
|
||||
import {
|
||||
normalizeSecurityPrincipal,
|
||||
type SecurityPrincipal,
|
||||
} from '@qinglong/runtime-core/security';
|
||||
import type { ClusterRunManagementService } from './runManagement';
|
||||
|
||||
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const UUID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
const STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']);
|
||||
|
||||
export type ClusterRunManagementCommand = Readonly<{
|
||||
schemaVersion: 1;
|
||||
operation: 'run.retry';
|
||||
request: Readonly<{
|
||||
projectId: string;
|
||||
sourceRunId: string;
|
||||
requestId: string;
|
||||
auditEventId: string;
|
||||
failureAuditEventId: string;
|
||||
body: Readonly<{
|
||||
schema: 'qinglong/run-manual-retry@v1';
|
||||
mutationId: string;
|
||||
expectedRunVersion: number;
|
||||
expectedRunStatus: 'failed' | 'cancelled' | 'timed_out';
|
||||
}>;
|
||||
}>;
|
||||
}>;
|
||||
|
||||
export type ClusterRunManagementTransportResult = Readonly<{
|
||||
schemaVersion: 1;
|
||||
operation: 'run.retry';
|
||||
retry: Readonly<RunManualRetryResponseBody>;
|
||||
}>;
|
||||
|
||||
export interface ClusterRunManagementAuthentication {
|
||||
authenticate(): Promise<Readonly<SecurityPrincipal> | null>;
|
||||
}
|
||||
|
||||
export interface ClusterRunManagementTransport {
|
||||
execute(
|
||||
command: unknown,
|
||||
authentication: ClusterRunManagementAuthentication,
|
||||
): Promise<Readonly<ClusterRunManagementTransportResult>>;
|
||||
}
|
||||
|
||||
export class ClusterRunManagementTransportConfigurationError extends TypeError {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_CONFIGURATION_INVALID';
|
||||
constructor() {
|
||||
super('Cluster Run management transport configuration is invalid');
|
||||
this.name = 'ClusterRunManagementTransportConfigurationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementTransportRequestError extends TypeError {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_REQUEST_INVALID';
|
||||
constructor() {
|
||||
super('Cluster Run management transport request is invalid');
|
||||
this.name = 'ClusterRunManagementTransportRequestError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementTransportAuthenticationError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_AUTHENTICATION_REQUIRED';
|
||||
constructor() {
|
||||
super('Cluster Run management transport requires a strong User principal');
|
||||
this.name = 'ClusterRunManagementTransportAuthenticationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementTransportUnavailableError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_UNAVAILABLE';
|
||||
constructor() {
|
||||
super('Cluster Run management transport is unavailable');
|
||||
this.name = 'ClusterRunManagementTransportUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function invalid(): never {
|
||||
throw new ClusterRunManagementTransportRequestError();
|
||||
}
|
||||
|
||||
function exact(value: unknown, keys: readonly string[]): Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) invalid();
|
||||
const actual = Object.keys(value as object).sort();
|
||||
const expected = [...keys].sort();
|
||||
if (
|
||||
actual.length !== expected.length ||
|
||||
actual.some((key, index) => key !== expected[index])
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function identifier(value: unknown): string {
|
||||
if (typeof value !== 'string' || !IDENTIFIER_PATTERN.test(value)) invalid();
|
||||
return value;
|
||||
}
|
||||
|
||||
function uuid(value: unknown): string {
|
||||
if (typeof value !== 'string' || !UUID_PATTERN.test(value)) invalid();
|
||||
return value;
|
||||
}
|
||||
|
||||
export function normalizeClusterRunManagementCommand(
|
||||
value: unknown,
|
||||
): Readonly<ClusterRunManagementCommand> {
|
||||
const envelope = exact(value, ['schemaVersion', 'operation', 'request']);
|
||||
if (envelope.schemaVersion !== 1 || envelope.operation !== 'run.retry') invalid();
|
||||
const request = exact(envelope.request, [
|
||||
'projectId',
|
||||
'sourceRunId',
|
||||
'requestId',
|
||||
'auditEventId',
|
||||
'failureAuditEventId',
|
||||
'body',
|
||||
]);
|
||||
let body: ReturnType<typeof parseRunManualRetryRequestBody>;
|
||||
try {
|
||||
body = parseRunManualRetryRequestBody(request.body);
|
||||
} catch {
|
||||
invalid();
|
||||
}
|
||||
const auditEventId = uuid(request.auditEventId);
|
||||
const failureAuditEventId = uuid(request.failureAuditEventId);
|
||||
if (auditEventId === failureAuditEventId) invalid();
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
request: Object.freeze({
|
||||
projectId: identifier(request.projectId),
|
||||
sourceRunId: identifier(request.sourceRunId),
|
||||
requestId: identifier(request.requestId),
|
||||
auditEventId,
|
||||
failureAuditEventId,
|
||||
body,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export function createClusterRunManagementTransport(options: Readonly<{
|
||||
service: ClusterRunManagementService;
|
||||
now?: () => number;
|
||||
}>): Readonly<ClusterRunManagementTransport> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some((key) => key !== 'service' && key !== 'now') ||
|
||||
!options.service ||
|
||||
typeof options.service.retry !== 'function' ||
|
||||
(options.now !== undefined && typeof options.now !== 'function')
|
||||
) {
|
||||
throw new ClusterRunManagementTransportConfigurationError();
|
||||
}
|
||||
const now = options.now ?? Date.now;
|
||||
return Object.freeze({
|
||||
async execute(
|
||||
commandValue: unknown,
|
||||
authentication: ClusterRunManagementAuthentication,
|
||||
) {
|
||||
const command = normalizeClusterRunManagementCommand(commandValue);
|
||||
if (
|
||||
!authentication ||
|
||||
typeof authentication !== 'object' ||
|
||||
Array.isArray(authentication) ||
|
||||
Object.keys(authentication).length !== 1 ||
|
||||
typeof authentication.authenticate !== 'function'
|
||||
) {
|
||||
throw new ClusterRunManagementTransportConfigurationError();
|
||||
}
|
||||
let candidate: Readonly<SecurityPrincipal> | null;
|
||||
try {
|
||||
candidate = await authentication.authenticate();
|
||||
} catch {
|
||||
throw new ClusterRunManagementTransportUnavailableError();
|
||||
}
|
||||
let principal: Readonly<SecurityPrincipal>;
|
||||
try {
|
||||
principal = normalizeSecurityPrincipal(candidate as SecurityPrincipal, now());
|
||||
} catch {
|
||||
throw new ClusterRunManagementTransportAuthenticationError();
|
||||
}
|
||||
if (
|
||||
principal.subject.type !== 'user' ||
|
||||
!STRONG_ASSURANCES.has(principal.assurance)
|
||||
) {
|
||||
throw new ClusterRunManagementTransportAuthenticationError();
|
||||
}
|
||||
const result = await options.service.retry({
|
||||
projectId: command.request.projectId,
|
||||
sourceRunId: command.request.sourceRunId,
|
||||
mutationId: command.request.body.mutationId,
|
||||
expectedRunVersion: command.request.body.expectedRunVersion,
|
||||
expectedRunStatus: command.request.body.expectedRunStatus,
|
||||
requestId: command.request.requestId,
|
||||
auditEventId: command.request.auditEventId,
|
||||
failureAuditEventId: command.request.failureAuditEventId,
|
||||
principal,
|
||||
});
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
retry: createRunManualRetryResponseBody(result),
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -163,6 +163,7 @@ function database(serverVersionNum = '160014') {
|
||||
'enforce_plugin_package_stage_provenance',
|
||||
'lock_active_plugin_package_project',
|
||||
'lock_approval_policy_fence',
|
||||
'lock_run_management_policy_fence',
|
||||
'plugin_package_lifecycle_blocking_runs',
|
||||
'plugin_package_automation_start_allowed',
|
||||
'plugin_package_run_start_allowed',
|
||||
|
||||
@@ -12,6 +12,7 @@ const {
|
||||
createClusterAutomationIdentityKeysetFile,
|
||||
createClusterApprovalIdentityKeysetFile,
|
||||
createClusterModelProviderCredentialIdentityKeysetFile,
|
||||
createClusterRunIdentityKeysetFile,
|
||||
} = require('@qinglong/cluster-admin/plugin-package-identity-keyset');
|
||||
|
||||
const NOW_MS = 1_700_000_000_000;
|
||||
@@ -216,6 +217,38 @@ function providerCredentialAssertion(key, overrides = {}) {
|
||||
).toString('base64url')}`;
|
||||
}
|
||||
|
||||
function runAssertion(key, overrides = {}) {
|
||||
const header = Buffer.from(
|
||||
JSON.stringify({
|
||||
alg: 'EdDSA',
|
||||
kid: key.kid,
|
||||
typ: 'ql3-run-management+jwt',
|
||||
}),
|
||||
).toString('base64url');
|
||||
const now = Math.floor(NOW_MS / 1000);
|
||||
const payload = Buffer.from(
|
||||
JSON.stringify({
|
||||
acr: 'urn:ql3:mfa',
|
||||
amr: ['pwd', 'otp'],
|
||||
aud: 'qinglong3-run-management',
|
||||
auth_time: now - 10,
|
||||
exp: now + 120,
|
||||
iat: now,
|
||||
iss: ISSUER,
|
||||
jti: `run-assertion-${key.kid}`,
|
||||
ql3_purpose: 'run-management',
|
||||
sub: 'run-operator-1',
|
||||
...overrides,
|
||||
}),
|
||||
).toString('base64url');
|
||||
const signed = `${header}.${payload}`;
|
||||
return `${signed}.${sign(
|
||||
null,
|
||||
Buffer.from(signed, 'ascii'),
|
||||
key.privateKey,
|
||||
).toString('base64url')}`;
|
||||
}
|
||||
|
||||
async function atomicWrite(filePath, document) {
|
||||
const nextPath = `${filePath}.next`;
|
||||
await writeFile(nextPath, `${JSON.stringify(document)}\n`, { mode: 0o644 });
|
||||
@@ -374,6 +407,34 @@ test('loads a provider credential keyset isolated by type, purpose and audience'
|
||||
});
|
||||
});
|
||||
|
||||
test('loads a Run keyset isolated from every other management purpose', async () => {
|
||||
await fixture(async ({ filePath }) => {
|
||||
const key = reviewedKey('run-identity-key-1');
|
||||
await atomicWrite(filePath, {
|
||||
...keyset(1, [key]),
|
||||
audience: 'qinglong3-run-management',
|
||||
});
|
||||
const provider = createClusterRunIdentityKeysetFile({
|
||||
filePath,
|
||||
now: () => NOW_MS,
|
||||
});
|
||||
const principal = await provider.bind(runAssertion(key)).authenticate();
|
||||
assert.deepEqual(principal.subject, {
|
||||
type: 'user',
|
||||
id: 'run-operator-1',
|
||||
});
|
||||
await assert.rejects(provider.bind(approvalAssertion(key)).authenticate(), {
|
||||
code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID',
|
||||
});
|
||||
await assert.rejects(
|
||||
provider
|
||||
.bind(runAssertion(key, { ql3_purpose: 'approval-management' }))
|
||||
.authenticate(),
|
||||
{ code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID' },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('supports overlap rotation then immediately revokes the previous key', async () => {
|
||||
await fixture(async ({ filePath }) => {
|
||||
const first = reviewedKey('issuer-key-1');
|
||||
|
||||
@@ -197,6 +197,7 @@ function database(serverVersionNum = '160014') {
|
||||
'plugin_package_tool_start_allowed',
|
||||
'plugin_package_workflow_admission_snapshot',
|
||||
'plugin_package_workflow_task_attempt_snapshot',
|
||||
'lock_run_management_policy_fence',
|
||||
].includes(functionName),
|
||||
isOwner: false,
|
||||
})),
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ClusterRunManagementAuthorizationError,
|
||||
createClusterRunManagementService,
|
||||
} = require('@qinglong/cluster-admin/run-management');
|
||||
|
||||
const NOW = 1_000_000;
|
||||
const SOURCE_DIGEST = 'a'.repeat(64);
|
||||
const EXECUTION_DIGEST = 'b'.repeat(64);
|
||||
const TASK_REVISION = `qltd:v1:7:${SOURCE_DIGEST}`;
|
||||
const GENERATED = [
|
||||
'019f9500-0000-4000-8000-000000000010',
|
||||
'019f9500-0000-4000-8000-000000000011',
|
||||
'019f9500-0000-4000-8000-000000000012',
|
||||
'019f9500-0000-4000-8000-000000000013',
|
||||
];
|
||||
|
||||
function request() {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
mutationId: '019f9500-0000-4000-8000-000000000001',
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
requestId: 'request-1',
|
||||
auditEventId: '019f9500-0000-4000-8000-000000000002',
|
||||
failureAuditEventId: '019f9500-0000-4000-8000-000000000003',
|
||||
principal: {
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
authenticationId: 'oidc:run-management-1',
|
||||
authenticatedAtMs: 999_000,
|
||||
expiresAtMs: 1_100_000,
|
||||
assurance: 'multi_factor',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function policyRow(role = 'operator') {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
projectName: 'Project 1',
|
||||
projectSlug: 'project-1',
|
||||
projectStatus: 'active',
|
||||
projectVersion: 2,
|
||||
projectCreatedAtMs: '1',
|
||||
projectUpdatedAtMs: '2',
|
||||
bindingProjectId: 'project-1',
|
||||
bindingSubjectType: 'user',
|
||||
bindingSubjectId: 'operator-1',
|
||||
bindingVersion: 3,
|
||||
bindingState: 'active',
|
||||
bindingRole: role,
|
||||
bindingMutationId: 'binding-3',
|
||||
bindingChangedByType: 'user',
|
||||
bindingChangedById: 'owner-1',
|
||||
bindingCreatedAtMs: '3',
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(role = 'operator') {
|
||||
const calls = [];
|
||||
const pool = {
|
||||
async query(sql, params = []) {
|
||||
const text = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ scope: 'pool', sql: text, params });
|
||||
if (text.includes('LEFT JOIN LATERAL')) return { rows: [policyRow(role)] };
|
||||
if (text.startsWith('INSERT INTO "ql3"."security_audit_events"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`unexpected pool query: ${text}`);
|
||||
},
|
||||
async connect() {
|
||||
return {
|
||||
async query(sql, params = []) {
|
||||
const text = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ scope: 'client', sql: text, params });
|
||||
if (
|
||||
text === 'BEGIN ISOLATION LEVEL SERIALIZABLE' ||
|
||||
text === 'COMMIT' ||
|
||||
text === 'ROLLBACK' ||
|
||||
text.startsWith('SELECT set_config')
|
||||
) return { rows: [], rowCount: 0 };
|
||||
if (text.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: NOW }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('lock_run_management_policy_fence')) {
|
||||
return { rows: [{ matches: true }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('idempotency_key = $2')) return { rows: [] };
|
||||
if (text.includes('WHERE run.id = $1')) {
|
||||
return {
|
||||
rows: [{
|
||||
projectId: 'project-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: TASK_REVISION,
|
||||
taskName: 'Task 1',
|
||||
taskSnapshotRef: TASK_REVISION,
|
||||
parentRunId: null,
|
||||
triggerType: 'task_start',
|
||||
executionOwner: 'runtime',
|
||||
inputRef: null,
|
||||
priority: 1,
|
||||
runStatus: 'failed',
|
||||
runVersion: 7,
|
||||
attemptExecutorType: 'remote_worker',
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."task_definitions"')) {
|
||||
return { rows: [{ enabled: true }] };
|
||||
}
|
||||
if (text.includes('task_execution_revisions')) {
|
||||
return { rows: [{ sourceContentDigest: SOURCE_DIGEST, contentDigest: EXECUTION_DIGEST }] };
|
||||
}
|
||||
if (text.startsWith('SELECT') && text.includes("trigger_type = 'run_manual_retry'")) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text.startsWith('INSERT INTO')) return { rows: [], rowCount: 1 };
|
||||
throw new Error(`unexpected client query: ${text}`);
|
||||
},
|
||||
release() {},
|
||||
};
|
||||
},
|
||||
};
|
||||
let index = 0;
|
||||
return {
|
||||
calls,
|
||||
service: createClusterRunManagementService({
|
||||
pool,
|
||||
now: () => NOW,
|
||||
randomUuid: () => GENERATED[index++],
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
test('authorizes run.retry and keeps all generated aggregate identities server-side', async () => {
|
||||
const { calls, service } = fixture();
|
||||
const result = await service.retry(request());
|
||||
assert.equal(result.status, 'accepted');
|
||||
assert.equal(result.runId, GENERATED[0]);
|
||||
assert.equal(result.attemptId, GENERATED[1]);
|
||||
const runInsert = calls.find(({ sql }) => sql.startsWith('INSERT INTO "ql3"."runs"'));
|
||||
assert.equal(runInsert.params[0], GENERATED[0]);
|
||||
assert.equal(runInsert.params.includes(GENERATED[2]), false);
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql.includes('lock_run_management_policy_fence')),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('denied policy writes only the caller-supplied failure audit', async () => {
|
||||
const { calls, service } = fixture('viewer');
|
||||
await assert.rejects(service.retry(request()), ClusterRunManagementAuthorizationError);
|
||||
const audits = calls.filter(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."security_audit_events"'),
|
||||
);
|
||||
assert.equal(audits.length, 1);
|
||||
assert.equal(audits[0].params[0], request().failureAuditEventId);
|
||||
assert.equal(calls.some(({ scope }) => scope === 'client'), false);
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
validateClusterRunManagementClientResult,
|
||||
} = require('@qinglong/cluster-admin/run-management-client');
|
||||
const {
|
||||
normalizeClusterRunManagementCommand,
|
||||
} = require('@qinglong/cluster-admin/run-management-transport');
|
||||
const {
|
||||
ClusterPluginPackageManagementClientRequestError,
|
||||
} = require('@qinglong/cluster-admin/plugin-package-management-client');
|
||||
|
||||
const command = normalizeClusterRunManagementCommand({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
requestId: 'request-1',
|
||||
auditEventId: '019f9400-0000-4000-8000-000000000001',
|
||||
failureAuditEventId: '019f9400-0000-4000-8000-000000000002',
|
||||
body: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
mutationId: '019f9400-0000-4000-8000-000000000003',
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
function response(overrides = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
retry: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
status: 'accepted',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
sourceRunStatus: 'failed',
|
||||
sourceRunVersion: 7,
|
||||
runId: '019f9400-0000-4000-8000-000000000010',
|
||||
retryOfRunId: 'source-run-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: `qltd:v1:1:${'a'.repeat(64)}`,
|
||||
attemptId: '019f9400-0000-4000-8000-000000000011',
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'remote_worker',
|
||||
executionRevisionDigest: 'b'.repeat(64),
|
||||
createdAtMs: 1_000_000,
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('validates one low-sensitive retry response against the request fence', () => {
|
||||
assert.deepEqual(validateClusterRunManagementClientResult(response(), command), response());
|
||||
});
|
||||
|
||||
test('rejects response target, execution placement and shape drift', () => {
|
||||
for (const candidate of [
|
||||
response({ projectId: 'project-2' }),
|
||||
response({ executorType: 'local_process' }),
|
||||
response({ sourceRunVersion: 8 }),
|
||||
{ ...response(), principal: { type: 'user', id: 'operator-1' } },
|
||||
]) {
|
||||
assert.throws(
|
||||
() => validateClusterRunManagementClientResult(candidate, command),
|
||||
ClusterPluginPackageManagementClientRequestError,
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,108 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { readFileSync } = require('node:fs');
|
||||
const { request: httpsRequest } = require('node:https');
|
||||
const { resolve } = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ClusterRunManagementRateLimitedError,
|
||||
} = require('@qinglong/cluster-admin/run-management');
|
||||
const {
|
||||
startClusterRunManagementHttp,
|
||||
} = require('@qinglong/cluster-admin/run-management-http');
|
||||
|
||||
const SERVER_KEY = resolve(__dirname, '../../ql3-cluster-control/test/fixtures/mtls/server-key.pem');
|
||||
const SERVER_CERT = resolve(__dirname, '../../ql3-cluster-control/test/fixtures/mtls/server-cert.pem');
|
||||
const PATH = '/api/v3/runs/management';
|
||||
|
||||
function post(port, path = PATH) {
|
||||
const body = Buffer.from(JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
requestId: 'request-1',
|
||||
auditEventId: '019f9600-0000-4000-8000-000000000001',
|
||||
failureAuditEventId: '019f9600-0000-4000-8000-000000000002',
|
||||
body: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
mutationId: '019f9600-0000-4000-8000-000000000003',
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
},
|
||||
},
|
||||
}));
|
||||
return new Promise((resolvePromise, reject) => {
|
||||
const outgoing = httpsRequest({
|
||||
host: '127.0.0.1',
|
||||
port,
|
||||
path,
|
||||
method: 'POST',
|
||||
rejectUnauthorized: false,
|
||||
agent: false,
|
||||
headers: {
|
||||
authorization: 'Bearer assertion',
|
||||
'content-type': 'application/json',
|
||||
'content-length': String(body.length),
|
||||
},
|
||||
}, (incoming) => {
|
||||
const chunks = [];
|
||||
incoming.on('data', (chunk) => chunks.push(chunk));
|
||||
incoming.once('end', () => {
|
||||
const bytes = Buffer.concat(chunks);
|
||||
resolvePromise({
|
||||
statusCode: incoming.statusCode,
|
||||
headers: incoming.headers,
|
||||
body: bytes.length ? JSON.parse(bytes.toString('utf8')) : null,
|
||||
});
|
||||
});
|
||||
});
|
||||
outgoing.once('error', reject);
|
||||
outgoing.end(body);
|
||||
});
|
||||
}
|
||||
|
||||
test('serves only the Run path and maps durable quota to bounded HTTP facts', async () => {
|
||||
const application = await startClusterRunManagementHttp({
|
||||
host: '127.0.0.1',
|
||||
port: 0,
|
||||
tls: {
|
||||
privateKey: Buffer.from(readFileSync(SERVER_KEY)),
|
||||
certificate: Buffer.from(readFileSync(SERVER_CERT)),
|
||||
},
|
||||
identities: {
|
||||
async reload() { throw new Error('not used'); },
|
||||
bind() {
|
||||
return { authenticate: async () => ({
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
authenticationId: 'oidc:run-management-1',
|
||||
authenticatedAtMs: 900,
|
||||
expiresAtMs: 10_000,
|
||||
assurance: 'hardware',
|
||||
}) };
|
||||
},
|
||||
},
|
||||
transport: {
|
||||
async execute(_command, authentication) {
|
||||
await authentication.authenticate();
|
||||
throw new ClusterRunManagementRateLimitedError(1_500);
|
||||
},
|
||||
},
|
||||
now: () => 1_000,
|
||||
});
|
||||
try {
|
||||
const limited = await post(application.address.port);
|
||||
assert.equal(limited.statusCode, 429);
|
||||
assert.equal(limited.headers['retry-after'], '2');
|
||||
assert.equal(limited.body.error.code, 'rate_limited');
|
||||
assert.match(limited.body.requestId, /^[0-9a-f-]{36}$/);
|
||||
const absent = await post(application.address.port, '/api/v3/approvals/management');
|
||||
assert.equal(absent.statusCode, 404);
|
||||
assert.equal(absent.body.error.code, 'not_found');
|
||||
} finally {
|
||||
await application.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ClusterRunManagementProcessConfigError,
|
||||
loadClusterRunManagementProcessConfig,
|
||||
startClusterRunManagementProcess,
|
||||
} = require('@qinglong/cluster-admin/run-management-process');
|
||||
|
||||
function enabled(overrides = {}) {
|
||||
return {
|
||||
QL3_RUN_MANAGEMENT_ENABLED: 'true',
|
||||
QL3_PROFILE: 'cluster-admin',
|
||||
QL3_RUN_MANAGEMENT_TLS_CERT_FILE: '/run/ql3/run/tls.crt',
|
||||
QL3_RUN_MANAGEMENT_TLS_KEY_FILE: '/run/ql3/run/tls.key',
|
||||
QL3_RUN_MANAGEMENT_CLIENT_CA_FILE: '/run/ql3/run/client-ca.crt',
|
||||
QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE: '/run/ql3/run/client.crl',
|
||||
QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE: '/run/ql3/run/identity.json',
|
||||
QL3_POSTGRES_RUN_MANAGER_HOST: 'postgres.qinglong3-system.svc',
|
||||
QL3_POSTGRES_RUN_MANAGER_DATABASE: 'qinglong3',
|
||||
QL3_POSTGRES_RUN_MANAGER_USER: 'ql3_run_manager',
|
||||
QL3_POSTGRES_RUN_MANAGER_PASSWORD: 'secret',
|
||||
QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME: 'postgres.qinglong3-system.svc',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('disabled Run manager acquires no PostgreSQL or file authority', async () => {
|
||||
let opened = false;
|
||||
const runtime = await startClusterRunManagementProcess({
|
||||
environment: {},
|
||||
openDatabase: async () => {
|
||||
opened = true;
|
||||
throw new Error('must not open');
|
||||
},
|
||||
});
|
||||
assert.equal(runtime.status, 'disabled');
|
||||
assert.equal(opened, false);
|
||||
await runtime.close();
|
||||
});
|
||||
|
||||
test('loads a bounded opt-in Run-only process configuration', () => {
|
||||
const config = loadClusterRunManagementProcessConfig(enabled());
|
||||
assert.equal(config.enabled, true);
|
||||
assert.equal(config.port, 8448);
|
||||
assert.equal(config.http.maxConcurrentRequests, 16);
|
||||
assert.equal(config.database.pool.maxConnections, 2);
|
||||
assert.equal(config.database.connection.user, 'ql3_run_manager');
|
||||
assert.deepEqual(config.database.connection.tls, {
|
||||
mode: 'verify-full',
|
||||
servername: 'postgres.qinglong3-system.svc',
|
||||
});
|
||||
});
|
||||
|
||||
test('rejects profile drift and implicit insecure PostgreSQL', () => {
|
||||
assert.throws(
|
||||
() => loadClusterRunManagementProcessConfig(enabled({ QL3_PROFILE: 'cluster-control' })),
|
||||
ClusterRunManagementProcessConfigError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
loadClusterRunManagementProcessConfig(
|
||||
enabled({
|
||||
QL3_POSTGRES_RUN_MANAGER_TLS_MODE: 'disable',
|
||||
QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME: undefined,
|
||||
}),
|
||||
),
|
||||
ClusterRunManagementProcessConfigError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,141 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ClusterRunManagementTransportAuthenticationError,
|
||||
ClusterRunManagementTransportRequestError,
|
||||
createClusterRunManagementTransport,
|
||||
normalizeClusterRunManagementCommand,
|
||||
} = require('@qinglong/cluster-admin/run-management-transport');
|
||||
|
||||
const NOW = 1_000_000;
|
||||
|
||||
function principal(overrides = {}) {
|
||||
return {
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
authenticationId: 'oidc:run-management-1',
|
||||
authenticatedAtMs: 999_000,
|
||||
expiresAtMs: 1_100_000,
|
||||
assurance: 'multi_factor',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function command(overrides = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
requestId: 'request-1',
|
||||
auditEventId: '019f9300-0000-4000-8000-000000000001',
|
||||
failureAuditEventId: '019f9300-0000-4000-8000-000000000002',
|
||||
body: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
mutationId: '019f9300-0000-4000-8000-000000000003',
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
},
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function retryResult() {
|
||||
return {
|
||||
status: 'accepted',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
sourceRunStatus: 'failed',
|
||||
sourceRunVersion: 7,
|
||||
runId: '019f9300-0000-4000-8000-000000000010',
|
||||
retryOfRunId: 'source-run-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: `qltd:v1:1:${'a'.repeat(64)}`,
|
||||
attemptId: '019f9300-0000-4000-8000-000000000011',
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'remote_worker',
|
||||
executionRevisionDigest: 'b'.repeat(64),
|
||||
createdAtMs: NOW,
|
||||
};
|
||||
}
|
||||
|
||||
test('routes one exact strong User retry and emits the shared response', async () => {
|
||||
const calls = [];
|
||||
const transport = createClusterRunManagementTransport({
|
||||
now: () => NOW,
|
||||
service: {
|
||||
async retry(request) {
|
||||
calls.push(request);
|
||||
return retryResult();
|
||||
},
|
||||
},
|
||||
});
|
||||
const result = await transport.execute(command(), {
|
||||
authenticate: async () => principal(),
|
||||
});
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(calls[0].mutationId, command().request.body.mutationId);
|
||||
assert.equal(calls[0].principal.assurance, 'multi_factor');
|
||||
assert.deepEqual(result, {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
retry: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
...retryResult(),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test('rejects weak or non-User identity before service authority', async () => {
|
||||
let called = false;
|
||||
const transport = createClusterRunManagementTransport({
|
||||
now: () => NOW,
|
||||
service: {
|
||||
async retry() {
|
||||
called = true;
|
||||
return retryResult();
|
||||
},
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
transport.execute(command(), {
|
||||
authenticate: async () => principal({ assurance: 'single_factor' }),
|
||||
}),
|
||||
ClusterRunManagementTransportAuthenticationError,
|
||||
);
|
||||
await assert.rejects(
|
||||
transport.execute(command(), {
|
||||
authenticate: async () =>
|
||||
principal({ subject: { type: 'agent', id: 'agent-1' } }),
|
||||
}),
|
||||
ClusterRunManagementTransportAuthenticationError,
|
||||
);
|
||||
assert.equal(called, false);
|
||||
});
|
||||
|
||||
test('rejects widened commands and ambiguous audit identity', () => {
|
||||
assert.throws(
|
||||
() => normalizeClusterRunManagementCommand({ ...command(), principal: principal() }),
|
||||
ClusterRunManagementTransportRequestError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeClusterRunManagementCommand(
|
||||
command({ failureAuditEventId: command().request.auditEventId }),
|
||||
),
|
||||
ClusterRunManagementTransportRequestError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeClusterRunManagementCommand(
|
||||
command({ body: { ...command().request.body, expectedRunStatus: 'lost' } }),
|
||||
),
|
||||
ClusterRunManagementTransportRequestError,
|
||||
);
|
||||
});
|
||||
@@ -334,6 +334,7 @@ function databaseResource(events, options = {}) {
|
||||
'plugin_package_workflow_task_attempt_snapshot',
|
||||
'plugin_package_run_start_allowed',
|
||||
'plugin_package_tool_start_allowed',
|
||||
'lock_run_management_policy_fence',
|
||||
].includes(functionName),
|
||||
isOwner: false,
|
||||
})),
|
||||
|
||||
@@ -243,6 +243,7 @@ function databaseResource(events, overrides = {}) {
|
||||
'plugin_package_workflow_task_attempt_snapshot',
|
||||
'plugin_package_run_start_allowed',
|
||||
'plugin_package_tool_start_allowed',
|
||||
'lock_run_management_policy_fence',
|
||||
].includes(functionName),
|
||||
isOwner: false,
|
||||
})),
|
||||
|
||||
@@ -75,6 +75,11 @@
|
||||
"require": "./dist/run-management/runManualRetryRepository.js",
|
||||
"default": "./dist/run-management/runManualRetryRepository.js"
|
||||
},
|
||||
"./run-manager": {
|
||||
"types": "./dist/entrypoints/runManager.d.ts",
|
||||
"require": "./dist/entrypoints/runManager.js",
|
||||
"default": "./dist/entrypoints/runManager.js"
|
||||
},
|
||||
"./approval-manager": {
|
||||
"types": "./dist/approval-management/index.d.ts",
|
||||
"require": "./dist/approval-management/index.js",
|
||||
|
||||
@@ -19,6 +19,7 @@ const DEFAULT_AI_CREDENTIAL_TESTER_APPLICATION_NAME =
|
||||
const DEFAULT_AUTOMATION_MANAGER_APPLICATION_NAME =
|
||||
'qinglong-automation-manager';
|
||||
const DEFAULT_APPROVAL_MANAGER_APPLICATION_NAME = 'qinglong-approval-manager';
|
||||
const DEFAULT_RUN_MANAGER_APPLICATION_NAME = 'qinglong-run-manager';
|
||||
const DEFAULT_PACKAGE_MANAGER_APPLICATION_NAME = 'qinglong-package-manager';
|
||||
const DEFAULT_PACKAGE_EXECUTOR_APPLICATION_NAME = 'qinglong-package-executor';
|
||||
const DEFAULT_WORKER_CREDENTIAL_MANAGER_APPLICATION_NAME =
|
||||
@@ -89,6 +90,7 @@ export type PostgresDatabaseRole =
|
||||
| 'admin'
|
||||
| 'automation-manager'
|
||||
| 'approval-manager'
|
||||
| 'run-manager'
|
||||
| 'package-manager'
|
||||
| 'package-executor'
|
||||
| 'worker-credential-manager'
|
||||
@@ -255,6 +257,7 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig {
|
||||
'admin',
|
||||
'automation-manager',
|
||||
'approval-manager',
|
||||
'run-manager',
|
||||
'package-manager',
|
||||
'package-executor',
|
||||
'worker-credential-manager',
|
||||
@@ -296,6 +299,7 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig {
|
||||
const isAdmin = options.role === 'admin';
|
||||
const isAutomationManager = options.role === 'automation-manager';
|
||||
const isApprovalManager = options.role === 'approval-manager';
|
||||
const isRunManager = options.role === 'run-manager';
|
||||
const isPackageManager = options.role === 'package-manager';
|
||||
const isPackageExecutor = options.role === 'package-executor';
|
||||
const isWorkerCredentialManager =
|
||||
@@ -309,6 +313,7 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig {
|
||||
isAdmin ||
|
||||
isAutomationManager ||
|
||||
isApprovalManager ||
|
||||
isRunManager ||
|
||||
isPackageManager ||
|
||||
isPackageExecutor ||
|
||||
isWorkerCredentialManager ||
|
||||
@@ -370,6 +375,8 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig {
|
||||
? DEFAULT_AUTOMATION_MANAGER_APPLICATION_NAME
|
||||
: isApprovalManager
|
||||
? DEFAULT_APPROVAL_MANAGER_APPLICATION_NAME
|
||||
: isRunManager
|
||||
? DEFAULT_RUN_MANAGER_APPLICATION_NAME
|
||||
: isPackageManager
|
||||
? DEFAULT_PACKAGE_MANAGER_APPLICATION_NAME
|
||||
: isPackageExecutor
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
export { PostgresRunManualRetryRepository } from '../run-management/runManualRetryRepository';
|
||||
export { PostgresProjectPolicyRepository } from '../security/projectPolicyRepository';
|
||||
export { PostgresSecurityAuditRepository } from '../security/securityAuditRepository';
|
||||
export {
|
||||
PostgresPluginPackageIdentityKeysetLedgerRepository as PostgresRunManagementIdentityKeysetLedgerRepository,
|
||||
type ClusterManagementIdentityAuthority,
|
||||
} from '../management/pluginPackageIdentityKeysetLedgerRepository';
|
||||
export {
|
||||
PgPoolBinding,
|
||||
createPostgresDatabaseOpener,
|
||||
isPostgresTlsDnsServername,
|
||||
type OpenPostgresDatabaseOptions,
|
||||
type PostgresConnectionOptions,
|
||||
type PostgresDatabaseRole,
|
||||
type PostgresPoolOptions,
|
||||
} from '../connection/pool';
|
||||
export {
|
||||
PostgresConnectionEnvironmentError,
|
||||
loadPostgresConnectionEnvironment,
|
||||
type PostgresConnectionEnvironment,
|
||||
type PostgresConnectionEnvironmentKeys,
|
||||
} from '../connection/connectionEnvironment';
|
||||
export {
|
||||
loadPostgresCertificateAuthorityFile,
|
||||
type PostgresCertificateAuthorityFileInspection,
|
||||
} from '../connection/certificateAuthority';
|
||||
export {
|
||||
PostgresSchemaReadinessError,
|
||||
assertPostgresRunManagerSchemaReady,
|
||||
type PostgresSchemaReadinessReport,
|
||||
} from '../schema/schemaReadiness';
|
||||
+4
-2
@@ -5,7 +5,8 @@ export type ClusterManagementIdentityAuthority =
|
||||
| 'plugin-package-management'
|
||||
| 'worker-credential-management'
|
||||
| 'automation-management'
|
||||
| 'approval-management';
|
||||
| 'approval-management'
|
||||
| 'run-management';
|
||||
const DIGEST_PATTERN = /^[A-Za-z0-9_-]{43}$/;
|
||||
const KEY_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/;
|
||||
@@ -189,7 +190,8 @@ export class PostgresPluginPackageIdentityKeysetLedgerRepository
|
||||
authority !== 'plugin-package-management' &&
|
||||
authority !== 'worker-credential-management' &&
|
||||
authority !== 'automation-management' &&
|
||||
authority !== 'approval-management'
|
||||
authority !== 'approval-management' &&
|
||||
authority !== 'run-management'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'PostgreSQL management identity keyset authority is invalid',
|
||||
|
||||
@@ -283,5 +283,10 @@ export const postgresqlMainMigrationManifest: MigrationStreamManifest =
|
||||
checksum:
|
||||
'c775c65ec03ae3a1606f899064d2d38fa63fd136ce52cbd1b1172c3a51e6bf30',
|
||||
}),
|
||||
Object.freeze({
|
||||
id: 'pg-0056-run-management-boundary',
|
||||
checksum:
|
||||
'7aa2b2ade67cdfa6839d4af02209906646a68adfd6c12c4dddeb854021da72b8',
|
||||
}),
|
||||
]),
|
||||
});
|
||||
|
||||
@@ -58,6 +58,7 @@ import { pg0052AutomationManagementIdentityKeysetLedgerMigration } from './pg-00
|
||||
import { pg0053PluginPackageWorkflowRunListIndexMigration } from './pg-0053-plugin-package-workflow-run-list-index';
|
||||
import { pg0054ApprovalManagementBoundaryMigration } from './pg-0054-approval-management-boundary';
|
||||
import { pg0055RunAttemptLogRetentionMigration } from './pg-0055-run-attempt-log-retention';
|
||||
import { pg0056RunManagementBoundaryMigration } from '../run-management/pg-0056-run-management-boundary';
|
||||
|
||||
export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMigrationContext> =
|
||||
Object.freeze({
|
||||
@@ -121,5 +122,6 @@ export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMi
|
||||
pg0053PluginPackageWorkflowRunListIndexMigration,
|
||||
pg0054ApprovalManagementBoundaryMigration,
|
||||
pg0055RunAttemptLogRetentionMigration,
|
||||
pg0056RunManagementBoundaryMigration,
|
||||
]),
|
||||
});
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import { CAPABILITIES_V54 } from '../migrations/pg-0055-run-attempt-log-retention';
|
||||
import { definePostgresSqlMigration } from '../migrations/sqlMigration';
|
||||
|
||||
export const CAPABILITIES_V55 = CAPABILITIES_V54.replace(
|
||||
'"run_core":1,',
|
||||
'"run_management_boundary":1,"run_core":1,',
|
||||
);
|
||||
|
||||
export const pg0056RunManagementBoundaryMigration =
|
||||
definePostgresSqlMigration({
|
||||
id: 'pg-0056-run-management-boundary',
|
||||
statements: [
|
||||
`
|
||||
DO $ql3$
|
||||
DECLARE
|
||||
role_invalid boolean;
|
||||
BEGIN
|
||||
SELECT roles.rolname IS NULL
|
||||
OR roles.rolcanlogin IS NOT TRUE
|
||||
OR roles.rolsuper IS NOT FALSE
|
||||
OR roles.rolcreatedb IS NOT FALSE
|
||||
OR roles.rolcreaterole IS NOT FALSE
|
||||
OR roles.rolreplication IS NOT FALSE
|
||||
OR roles.rolbypassrls IS NOT FALSE
|
||||
INTO role_invalid
|
||||
FROM (SELECT 1) AS expected
|
||||
LEFT JOIN pg_catalog.pg_roles AS roles
|
||||
ON roles.rolname = 'ql3_run_manager';
|
||||
|
||||
IF role_invalid IS NOT FALSE THEN
|
||||
RAISE EXCEPTION
|
||||
'required QingLong run manager role is missing or privileged'
|
||||
USING ERRCODE = 'insufficient_privilege';
|
||||
END IF;
|
||||
END
|
||||
$ql3$
|
||||
`.trim(),
|
||||
`
|
||||
DO $ql3$
|
||||
BEGIN
|
||||
EXECUTE format(
|
||||
'GRANT CONNECT ON DATABASE %I TO ql3_run_manager',
|
||||
current_database()
|
||||
);
|
||||
END
|
||||
$ql3$
|
||||
`.trim(),
|
||||
`GRANT USAGE ON SCHEMA "ql3" TO ql3_run_manager`,
|
||||
`REVOKE ALL ON ALL TABLES IN SCHEMA "ql3" FROM ql3_run_manager`,
|
||||
`REVOKE ALL ON ALL FUNCTIONS IN SCHEMA "ql3" FROM ql3_run_manager`,
|
||||
`
|
||||
CREATE FUNCTION "ql3"."lock_run_management_policy_fence"(
|
||||
varchar,
|
||||
varchar,
|
||||
varchar,
|
||||
integer,
|
||||
integer
|
||||
)
|
||||
RETURNS boolean
|
||||
LANGUAGE plpgsql
|
||||
VOLATILE
|
||||
SECURITY DEFINER
|
||||
SET search_path = pg_catalog, ql3
|
||||
AS $ql3$
|
||||
DECLARE
|
||||
project_status varchar;
|
||||
project_version integer;
|
||||
binding_state varchar;
|
||||
binding_role varchar;
|
||||
binding_version integer;
|
||||
BEGIN
|
||||
SELECT project.status, project.version
|
||||
INTO project_status, project_version
|
||||
FROM "ql3"."projects" AS project
|
||||
WHERE project.id = $1
|
||||
FOR UPDATE;
|
||||
|
||||
SELECT binding.state, binding.role, binding.version
|
||||
INTO binding_state, binding_role, binding_version
|
||||
FROM "ql3"."project_role_bindings" AS binding
|
||||
WHERE binding.project_id = $1
|
||||
AND binding.subject_type = $2
|
||||
AND binding.subject_id = $3
|
||||
ORDER BY binding.version DESC
|
||||
LIMIT 1;
|
||||
|
||||
RETURN project_status = 'active'
|
||||
AND project_version = $4
|
||||
AND binding_state = 'active'
|
||||
AND binding_role IN ('owner', 'admin', 'operator')
|
||||
AND binding_version = $5;
|
||||
END
|
||||
$ql3$
|
||||
`.trim(),
|
||||
`REVOKE ALL ON FUNCTION "ql3"."lock_run_management_policy_fence"(varchar, varchar, varchar, integer, integer) FROM PUBLIC`,
|
||||
`GRANT EXECUTE ON FUNCTION "ql3"."lock_run_management_policy_fence"(varchar, varchar, varchar, integer, integer) TO ql3_runtime, ql3_run_manager`,
|
||||
`GRANT SELECT ON "ql3"."schema_migrations", "ql3"."schema_capabilities", "ql3"."projects", "ql3"."project_role_bindings", "ql3"."task_definitions", "ql3"."task_definition_revisions", "ql3"."task_execution_revisions" TO ql3_run_manager`,
|
||||
`GRANT SELECT, INSERT ON "ql3"."runs", "ql3"."run_attempts", "ql3"."run_events", "ql3"."security_audit_events" TO ql3_run_manager`,
|
||||
`GRANT SELECT, INSERT, UPDATE ON "ql3"."plugin_package_identity_keyset_ledger" TO ql3_run_manager`,
|
||||
`ALTER TABLE "ql3"."plugin_package_identity_keyset_ledger" DROP CONSTRAINT ql3_plugin_package_identity_keyset_authority_check`,
|
||||
`ALTER TABLE "ql3"."plugin_package_identity_keyset_ledger" ADD CONSTRAINT ql3_plugin_package_identity_keyset_authority_check CHECK (authority IN ('plugin-package-management', 'worker-credential-management', 'automation-management', 'approval-management', 'run-management'))`,
|
||||
`
|
||||
DO $ql3$
|
||||
BEGIN
|
||||
UPDATE "ql3"."schema_capabilities"
|
||||
SET contract_version = 55,
|
||||
migration_id = 'pg-0056-run-management-boundary',
|
||||
capabilities = '${CAPABILITIES_V55}'::jsonb,
|
||||
updated_at_ms = floor(extract(epoch FROM transaction_timestamp()) * 1000)::bigint
|
||||
WHERE contract_name = 'control-core'
|
||||
AND contract_version = 54
|
||||
AND migration_id = 'pg-0055-run-attempt-log-retention'
|
||||
AND capabilities = '${CAPABILITIES_V54}'::jsonb;
|
||||
IF NOT FOUND THEN
|
||||
RAISE EXCEPTION 'control-core capability is not at version 54'
|
||||
USING ERRCODE = 'check_violation';
|
||||
END IF;
|
||||
END
|
||||
$ql3$
|
||||
`.trim(),
|
||||
],
|
||||
});
|
||||
@@ -9,7 +9,6 @@ import {
|
||||
RunManualRetryUnavailableError,
|
||||
normalizeRunManualRetryCommand,
|
||||
normalizeRunManualRetryResult,
|
||||
type RunManualRetryAllowedRole,
|
||||
type RunManualRetryCommand,
|
||||
type RunManualRetryRepository,
|
||||
type RunManualRetryResult,
|
||||
@@ -33,11 +32,6 @@ type Row = Record<string, unknown>;
|
||||
export const CLUSTER_RUN_MANUAL_RETRY_RATE_WINDOW_MS = 60_000;
|
||||
export const CLUSTER_RUN_MANUAL_RETRY_RATE_LIMIT = 64;
|
||||
|
||||
const ALLOWED_ROLES = new Set<RunManualRetryAllowedRole>([
|
||||
'owner',
|
||||
'admin',
|
||||
'operator',
|
||||
]);
|
||||
const CLUSTER_STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']);
|
||||
const TASK_REVISION_PATTERN = /^qltd:v1:([1-9]\d*):([0-9a-f]{64})$/;
|
||||
|
||||
@@ -128,45 +122,23 @@ async function confirmAuthorization(
|
||||
client: PostgresClient,
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
): Promise<void> {
|
||||
const project = await client.query<Row>(
|
||||
const result = await client.query<Row>(
|
||||
`
|
||||
SELECT status AS "projectStatus", version AS "projectVersion"
|
||||
FROM "ql3"."projects" WHERE id = $1 FOR UPDATE
|
||||
`,
|
||||
[command.projectId],
|
||||
);
|
||||
if (project.rows.length === 0) throw new RunManualRetryNotFoundError();
|
||||
if (project.rows.length !== 1) throw unavailable();
|
||||
// Authorized management mutations take the same Project lock. Keeping this
|
||||
// append-only RoleBinding read lock-free avoids granting UPDATE authority to
|
||||
// the runtime role merely to use PostgreSQL row-lock syntax.
|
||||
const binding = await client.query<Row>(
|
||||
`
|
||||
SELECT version AS "bindingVersion", state AS "bindingState",
|
||||
role AS "bindingRole"
|
||||
FROM "ql3"."project_role_bindings"
|
||||
WHERE project_id = $1 AND subject_type = $2 AND subject_id = $3
|
||||
ORDER BY version DESC LIMIT 1
|
||||
SELECT "ql3"."lock_run_management_policy_fence"(
|
||||
$1::varchar, $2::varchar, $3::varchar, $4::integer, $5::integer
|
||||
) AS "matches"
|
||||
`,
|
||||
[
|
||||
command.projectId,
|
||||
command.principal.subject.type,
|
||||
command.principal.subject.id,
|
||||
command.policyFence.projectVersion,
|
||||
command.policyFence.bindingVersion,
|
||||
],
|
||||
);
|
||||
const currentProject = project.rows[0]!;
|
||||
const currentBinding = binding.rows[0];
|
||||
if (
|
||||
text(currentProject, 'projectStatus') !== 'active' ||
|
||||
integer(currentProject, 'projectVersion') !==
|
||||
command.policyFence.projectVersion ||
|
||||
!currentBinding ||
|
||||
integer(currentBinding, 'bindingVersion') !==
|
||||
command.policyFence.bindingVersion ||
|
||||
text(currentBinding, 'bindingState') !== 'active' ||
|
||||
!ALLOWED_ROLES.has(
|
||||
text(currentBinding, 'bindingRole') as RunManualRetryAllowedRole,
|
||||
)
|
||||
result.rows.length !== 1 ||
|
||||
!postgresRequiredBoolean(result.rows[0]!.matches, unavailable)
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('authorization_changed');
|
||||
}
|
||||
@@ -185,8 +157,6 @@ async function findReplay(
|
||||
run.execution_origin AS "executionOrigin",
|
||||
run.execution_owner AS "executionOwner",
|
||||
run.triggered_by AS "triggeredBy", run.request_id AS "requestId",
|
||||
run.status AS "runStatus", run.version AS "runVersion",
|
||||
run.event_sequence AS "eventSequence",
|
||||
run.created_at_ms AS "createdAtMs",
|
||||
attempt.id AS "attemptId", attempt.executor_type AS "executorType",
|
||||
created.actor_type AS "createdActorType",
|
||||
@@ -205,7 +175,6 @@ async function findReplay(
|
||||
ON queued.run_id = run.id AND queued.sequence = 2
|
||||
AND queued.type = 'run.queued'
|
||||
WHERE run.project_id = $1 AND run.idempotency_key = $2
|
||||
FOR UPDATE OF run
|
||||
`,
|
||||
[command.projectId, `ql3:run-manual-retry:v1:${command.mutationId}`],
|
||||
);
|
||||
@@ -227,9 +196,6 @@ function replayResult(
|
||||
text(row, 'executionOwner') !== 'runtime' ||
|
||||
text(row, 'triggeredBy') !== command.principal.subject.id ||
|
||||
text(row, 'requestId') !== command.mutationId ||
|
||||
text(row, 'runStatus') !== 'queued' ||
|
||||
integer(row, 'runVersion') !== 2 ||
|
||||
integer(row, 'eventSequence') !== 2 ||
|
||||
text(row, 'executorType') !== 'remote_worker' ||
|
||||
text(row, 'createdActorType') !== command.principal.subject.type ||
|
||||
text(row, 'createdActorId') !== command.principal.subject.id ||
|
||||
@@ -295,7 +261,6 @@ async function findSource(
|
||||
LIMIT 1
|
||||
) AS attempt ON true
|
||||
WHERE run.id = $1
|
||||
FOR UPDATE OF run
|
||||
`,
|
||||
[command.sourceRunId],
|
||||
);
|
||||
|
||||
@@ -15,13 +15,14 @@ export interface PostgresSchemaContractFunction {
|
||||
export interface PostgresSchemaContract {
|
||||
readonly schema: 'ql3';
|
||||
readonly contractName: 'control-core';
|
||||
readonly contractVersion: 54;
|
||||
readonly migrationId: 'pg-0055-run-attempt-log-retention';
|
||||
readonly contractVersion: 55;
|
||||
readonly migrationId: 'pg-0056-run-management-boundary';
|
||||
readonly minimumServerMajor: 16;
|
||||
readonly maximumServerMajor: 18;
|
||||
readonly capabilities: Readonly<{
|
||||
run_core: 1;
|
||||
run_attempt_log_retention: 1;
|
||||
run_management_boundary: 1;
|
||||
run_dispatch_lease: 1;
|
||||
run_retry_policy: 1;
|
||||
project_policy: 1;
|
||||
@@ -101,8 +102,8 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
Object.freeze({
|
||||
schema: 'ql3',
|
||||
contractName: 'control-core',
|
||||
contractVersion: 54,
|
||||
migrationId: 'pg-0055-run-attempt-log-retention',
|
||||
contractVersion: 55,
|
||||
migrationId: 'pg-0056-run-management-boundary',
|
||||
minimumServerMajor: 16,
|
||||
maximumServerMajor: 18,
|
||||
capabilities: Object.freeze({
|
||||
@@ -143,6 +144,7 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
project_tool_definition_snapshot: 1,
|
||||
run_core: 1,
|
||||
run_attempt_log_retention: 1,
|
||||
run_management_boundary: 1,
|
||||
run_dispatch_lease: 1,
|
||||
run_retry_policy: 1,
|
||||
security_audit: 1,
|
||||
@@ -2323,6 +2325,15 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
volatility: 'volatile',
|
||||
configuration: Object.freeze(['search_path=pg_catalog, ql3']),
|
||||
}),
|
||||
Object.freeze({
|
||||
name: 'lock_run_management_policy_fence',
|
||||
identityArguments:
|
||||
'character varying, character varying, character varying, integer, integer',
|
||||
owner: 'ql3_migration',
|
||||
securityDefiner: true,
|
||||
volatility: 'volatile',
|
||||
configuration: Object.freeze(['search_path=pg_catalog, ql3']),
|
||||
}),
|
||||
Object.freeze({
|
||||
name: 'commit_plugin_package_task_reconciliation',
|
||||
identityArguments:
|
||||
|
||||
@@ -19,6 +19,7 @@ export const POSTGRES_SCHEMA_READINESS_ERROR_CODES = [
|
||||
'admin_role_invalid',
|
||||
'automation_manager_role_invalid',
|
||||
'approval_manager_role_invalid',
|
||||
'run_manager_role_invalid',
|
||||
'package_manager_role_invalid',
|
||||
'package_executor_role_invalid',
|
||||
'worker_credential_manager_role_invalid',
|
||||
@@ -1361,6 +1362,37 @@ const REQUIRED_APPROVAL_MANAGER_PRIVILEGES: RequiredPrivileges = Object.freeze(
|
||||
),
|
||||
);
|
||||
|
||||
const REQUIRED_RUN_MANAGER_PRIVILEGES: RequiredPrivileges = Object.freeze(
|
||||
Object.fromEntries(
|
||||
postgresqlControlSchemaContract.tables.map(({ name }) => [
|
||||
name,
|
||||
Object.freeze(
|
||||
name === 'schema_migrations' ||
|
||||
name === 'schema_capabilities' ||
|
||||
name === 'projects' ||
|
||||
name === 'project_role_bindings' ||
|
||||
name === 'task_definitions' ||
|
||||
name === 'task_definition_revisions' ||
|
||||
name === 'task_execution_revisions'
|
||||
? { ...NO_TABLE_PRIVILEGES, select: true }
|
||||
: name === 'runs' ||
|
||||
name === 'run_attempts' ||
|
||||
name === 'run_events' ||
|
||||
name === 'security_audit_events'
|
||||
? { ...NO_TABLE_PRIVILEGES, select: true, insert: true }
|
||||
: name === 'plugin_package_identity_keyset_ledger'
|
||||
? {
|
||||
...NO_TABLE_PRIVILEGES,
|
||||
select: true,
|
||||
insert: true,
|
||||
update: true,
|
||||
}
|
||||
: NO_TABLE_PRIVILEGES,
|
||||
),
|
||||
]),
|
||||
),
|
||||
);
|
||||
|
||||
const REQUIRED_WORKER_CREDENTIAL_MANAGER_PRIVILEGES: RequiredPrivileges =
|
||||
Object.freeze(
|
||||
Object.fromEntries(
|
||||
@@ -1448,6 +1480,7 @@ const REQUIRED_RUNTIME_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
|
||||
enforce_plugin_package_stage_provenance: false,
|
||||
lock_active_plugin_package_project: false,
|
||||
lock_approval_policy_fence: false,
|
||||
lock_run_management_policy_fence: true,
|
||||
plugin_package_automation_start_allowed: true,
|
||||
plugin_package_workflow_admission_snapshot: true,
|
||||
plugin_package_workflow_task_attempt_snapshot: true,
|
||||
@@ -1464,6 +1497,7 @@ const REQUIRED_PACKAGE_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
|
||||
enforce_plugin_package_stage_provenance: false,
|
||||
lock_active_plugin_package_project: false,
|
||||
lock_approval_policy_fence: true,
|
||||
lock_run_management_policy_fence: false,
|
||||
plugin_package_automation_start_allowed: false,
|
||||
plugin_package_workflow_admission_snapshot: false,
|
||||
plugin_package_workflow_task_attempt_snapshot: false,
|
||||
@@ -1480,6 +1514,7 @@ const REQUIRED_PACKAGE_EXECUTOR_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges
|
||||
enforce_plugin_package_stage_provenance: false,
|
||||
lock_active_plugin_package_project: true,
|
||||
lock_approval_policy_fence: true,
|
||||
lock_run_management_policy_fence: false,
|
||||
plugin_package_automation_start_allowed: false,
|
||||
plugin_package_workflow_admission_snapshot: false,
|
||||
plugin_package_workflow_task_attempt_snapshot: false,
|
||||
@@ -1500,6 +1535,12 @@ const REQUIRED_APPROVAL_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges
|
||||
lock_approval_policy_fence: true,
|
||||
});
|
||||
|
||||
const REQUIRED_RUN_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
|
||||
Object.freeze({
|
||||
...NO_FUNCTION_PRIVILEGES,
|
||||
lock_run_management_policy_fence: true,
|
||||
});
|
||||
|
||||
function safeInteger(value: unknown): number | null {
|
||||
if (typeof value === 'number' && Number.isSafeInteger(value)) return value;
|
||||
if (typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value)) {
|
||||
@@ -1864,6 +1905,7 @@ async function assertRole(
|
||||
| 'admin_role_invalid'
|
||||
| 'automation_manager_role_invalid'
|
||||
| 'approval_manager_role_invalid'
|
||||
| 'run_manager_role_invalid'
|
||||
| 'package_manager_role_invalid'
|
||||
| 'package_executor_role_invalid'
|
||||
| 'worker_credential_manager_role_invalid'
|
||||
@@ -2120,6 +2162,30 @@ export async function assertPostgresApprovalManagerSchemaReady(
|
||||
});
|
||||
}
|
||||
|
||||
export async function assertPostgresRunManagerSchemaReady(
|
||||
queryable: PostgresMigrationQueryable,
|
||||
contract: PostgresSchemaContract = postgresqlControlSchemaContract,
|
||||
): Promise<PostgresSchemaReadinessReport> {
|
||||
const server = await readServer(queryable, contract);
|
||||
const migrationIds = await assertHistory(queryable);
|
||||
await assertCapability(queryable, contract);
|
||||
await assertSchemaContract(queryable, contract);
|
||||
await assertRole(
|
||||
queryable,
|
||||
contract,
|
||||
REQUIRED_RUN_MANAGER_PRIVILEGES,
|
||||
REQUIRED_RUN_MANAGER_FUNCTION_PRIVILEGES,
|
||||
'run_manager_role_invalid',
|
||||
);
|
||||
return Object.freeze({
|
||||
ready: true,
|
||||
...server,
|
||||
contractName: contract.contractName,
|
||||
contractVersion: contract.contractVersion,
|
||||
migrationIds,
|
||||
});
|
||||
}
|
||||
|
||||
export async function assertPostgresPackageManagerSchemaReady(
|
||||
queryable: PostgresMigrationQueryable,
|
||||
contract: PostgresSchemaContract = postgresqlControlSchemaContract,
|
||||
|
||||
+9
-1
@@ -108,7 +108,7 @@ test('serializes first observation, exact replay and append-only rotation', asyn
|
||||
assert.equal(value.releases(), 3);
|
||||
});
|
||||
|
||||
test('isolates Plugin, Worker, automation and Approval generations by authority key', async () => {
|
||||
test('isolates Plugin, Worker, automation, Approval and Run generations by authority key', async () => {
|
||||
const value = fixture('worker-credential-management');
|
||||
await value.repository.observe(
|
||||
snapshot(1, { audience: 'qinglong3-worker-credential-management' }),
|
||||
@@ -131,6 +131,14 @@ test('isolates Plugin, Worker, automation and Approval generations by authority
|
||||
approval.queries.find(({ text }) => text.startsWith('INSERT')).values[0],
|
||||
'approval-management',
|
||||
);
|
||||
const run = fixture('run-management');
|
||||
await run.repository.observe(
|
||||
snapshot(1, { audience: 'qinglong3-run-management' }),
|
||||
);
|
||||
assert.equal(
|
||||
run.queries.find(({ text }) => text.startsWith('INSERT')).values[0],
|
||||
'run-management',
|
||||
);
|
||||
assert.throws(
|
||||
() => fixture('worker-credential-executor'),
|
||||
TypeError,
|
||||
|
||||
@@ -104,6 +104,7 @@ test('enforces role-specific bounded pool sizes', () => {
|
||||
'ai-credential-tester',
|
||||
'automation-manager',
|
||||
'approval-manager',
|
||||
'run-manager',
|
||||
'worker-credential-manager',
|
||||
'worker-credential-executor',
|
||||
]) {
|
||||
|
||||
@@ -104,6 +104,7 @@ test('defines the immutable PostgreSQL capability and Run core stream', async ()
|
||||
'pg-0053-plugin-package-workflow-run-list-index',
|
||||
'pg-0054-approval-management-boundary',
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
],
|
||||
);
|
||||
for (const migration of postgresqlMainMigrationStream.migrations) {
|
||||
@@ -514,6 +515,11 @@ test('freezes every published PostgreSQL migration checksum', () => {
|
||||
checksum:
|
||||
'c775c65ec03ae3a1606f899064d2d38fa63fd136ce52cbd1b1172c3a51e6bf30',
|
||||
},
|
||||
{
|
||||
id: 'pg-0056-run-management-boundary',
|
||||
checksum:
|
||||
'7aa2b2ade67cdfa6839d4af02209906646a68adfd6c12c4dddeb854021da72b8',
|
||||
},
|
||||
];
|
||||
assert.deepEqual(
|
||||
postgresqlMainMigrationStream.migrations.map(({ id, checksum }) => ({
|
||||
@@ -1904,3 +1910,30 @@ test('advances capability v54 with durable Cluster log retention authority', asy
|
||||
/migration_id = 'pg-0054-approval-management-boundary'/,
|
||||
);
|
||||
});
|
||||
|
||||
test('advances capability v55 with isolated strong Run management authority', async () => {
|
||||
const migration = migrationById('pg-0056-run-management-boundary');
|
||||
const statements = [];
|
||||
await migration.up({
|
||||
async query(statement) {
|
||||
statements.push(statement);
|
||||
return { rows: [] };
|
||||
},
|
||||
});
|
||||
const sql = statements.join('\n');
|
||||
assert.match(sql, /ql3_run_manager/);
|
||||
assert.match(sql, /lock_run_management_policy_fence/);
|
||||
assert.match(
|
||||
sql,
|
||||
/GRANT SELECT, INSERT ON "ql3"\."runs", "ql3"\."run_attempts", "ql3"\."run_events", "ql3"\."security_audit_events" TO ql3_run_manager/,
|
||||
);
|
||||
assert.doesNotMatch(sql, /GRANT UPDATE ON "ql3"\."runs"/);
|
||||
assert.match(sql, /'run-management'/);
|
||||
assert.match(sql, /contract_version = 55/);
|
||||
assert.match(sql, /"run_management_boundary":1/);
|
||||
assert.match(sql, /contract_version = 54/);
|
||||
assert.match(
|
||||
sql,
|
||||
/migration_id = 'pg-0055-run-attempt-log-retention'/,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -4,6 +4,7 @@ const {
|
||||
PostgresSchemaReadinessError,
|
||||
assertPostgresAdminSchemaReady,
|
||||
assertPostgresApprovalManagerSchemaReady,
|
||||
assertPostgresRunManagerSchemaReady,
|
||||
assertPostgresAutomationManagerSchemaReady,
|
||||
assertPostgresPackageExecutorSchemaReady,
|
||||
assertPostgresPackageManagerSchemaReady,
|
||||
@@ -474,6 +475,37 @@ function approvalManagerPrivileges() {
|
||||
}));
|
||||
}
|
||||
|
||||
function runManagerPrivileges() {
|
||||
const readable = new Set([
|
||||
'schema_migrations',
|
||||
'schema_capabilities',
|
||||
'projects',
|
||||
'project_role_bindings',
|
||||
'task_definitions',
|
||||
'task_definition_revisions',
|
||||
'task_execution_revisions',
|
||||
'runs',
|
||||
'run_attempts',
|
||||
'run_events',
|
||||
'security_audit_events',
|
||||
'plugin_package_identity_keyset_ledger',
|
||||
]);
|
||||
return postgresqlControlSchemaContract.tables.map(({ name: tableName }) => ({
|
||||
tableName,
|
||||
selectAllowed: readable.has(tableName),
|
||||
insertAllowed: [
|
||||
'runs',
|
||||
'run_attempts',
|
||||
'run_events',
|
||||
'security_audit_events',
|
||||
'plugin_package_identity_keyset_ledger',
|
||||
].includes(tableName),
|
||||
updateAllowed: tableName === 'plugin_package_identity_keyset_ledger',
|
||||
deleteAllowed: false,
|
||||
isOwner: false,
|
||||
}));
|
||||
}
|
||||
|
||||
function workerCredentialPrivileges(kind) {
|
||||
const manager = kind === 'manager';
|
||||
const readable = new Set([
|
||||
@@ -634,6 +666,8 @@ function queryable(overrides = {}) {
|
||||
executeAllowed:
|
||||
overrides.functionMode === 'manager'
|
||||
? functionName === 'lock_approval_policy_fence'
|
||||
: overrides.functionMode === 'run-manager'
|
||||
? functionName === 'lock_run_management_policy_fence'
|
||||
: overrides.functionMode === 'executor'
|
||||
? [
|
||||
'commit_plugin_package_lifecycle',
|
||||
@@ -651,6 +685,7 @@ function queryable(overrides = {}) {
|
||||
'plugin_package_workflow_task_attempt_snapshot',
|
||||
'plugin_package_run_start_allowed',
|
||||
'plugin_package_tool_start_allowed',
|
||||
'lock_run_management_policy_fence',
|
||||
].includes(functionName),
|
||||
isOwner: false,
|
||||
})),
|
||||
@@ -696,7 +731,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
|
||||
serverMajor: 16,
|
||||
currentUser: 'ql3_runtime',
|
||||
contractName: 'control-core',
|
||||
contractVersion: 54,
|
||||
contractVersion: 55,
|
||||
migrationIds: [
|
||||
'pg-0001-schema-capability',
|
||||
'pg-0002-run-core',
|
||||
@@ -753,6 +788,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
|
||||
'pg-0053-plugin-package-workflow-run-list-index',
|
||||
'pg-0054-approval-management-boundary',
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
],
|
||||
});
|
||||
});
|
||||
@@ -783,10 +819,10 @@ test('accepts the exact schema and isolated least-privilege admin role', async (
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_admin');
|
||||
assert.equal(report.contractVersion, 54);
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -799,10 +835,10 @@ test('accepts the isolated least-privilege automation manager role', async () =>
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_automation_manager');
|
||||
assert.equal(report.contractVersion, 54);
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
);
|
||||
|
||||
const widened = automationManagerPrivileges();
|
||||
@@ -831,10 +867,10 @@ test('accepts the isolated least-privilege human Approval manager role', async (
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_approval_manager');
|
||||
assert.equal(report.contractVersion, 54);
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
);
|
||||
|
||||
const widened = approvalManagerPrivileges();
|
||||
@@ -856,6 +892,35 @@ test('accepts the isolated least-privilege human Approval manager role', async (
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts the isolated least-privilege Run manager role', async () => {
|
||||
const report = await assertPostgresRunManagerSchemaReady(
|
||||
queryable({
|
||||
currentUser: 'ql3_run_manager',
|
||||
privileges: runManagerPrivileges(),
|
||||
functionMode: 'run-manager',
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_run_manager');
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(report.migrationIds.at(-1), 'pg-0056-run-management-boundary');
|
||||
|
||||
const widened = runManagerPrivileges();
|
||||
widened.find(({ tableName }) => tableName === 'runs').updateAllowed = true;
|
||||
await assert.rejects(
|
||||
assertPostgresRunManagerSchemaReady(
|
||||
queryable({
|
||||
currentUser: 'ql3_run_manager',
|
||||
privileges: widened,
|
||||
functionMode: 'run-manager',
|
||||
}),
|
||||
),
|
||||
(error) =>
|
||||
error instanceof PostgresSchemaReadinessError &&
|
||||
error.code === 'run_manager_role_invalid' &&
|
||||
error.facts.includes('table-privileges:runs'),
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts isolated Package manager and executor roles', async () => {
|
||||
const manager = await assertPostgresPackageManagerSchemaReady(
|
||||
queryable({
|
||||
@@ -941,10 +1006,10 @@ test('accepts the exact schema and isolated Worker ingress role', async () => {
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_worker_ingress');
|
||||
assert.equal(report.contractVersion, 54);
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -138,21 +138,11 @@ function fixture(options = {}) {
|
||||
if (normalized.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: 1_000_000 }], rowCount: 1 };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."projects"')) {
|
||||
const rows = options.projectRows ?? [
|
||||
{ projectStatus: 'active', projectVersion: 2 },
|
||||
];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('project_role_bindings')) {
|
||||
const rows = options.bindingRows ?? [
|
||||
{
|
||||
bindingVersion: 3,
|
||||
bindingState: 'active',
|
||||
bindingRole: 'operator',
|
||||
},
|
||||
];
|
||||
return { rows, rowCount: rows.length };
|
||||
if (normalized.includes('lock_run_management_policy_fence')) {
|
||||
return {
|
||||
rows: [{ matches: options.authorizationMatches ?? true }],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('idempotency_key = $2')) {
|
||||
const rows = options.replayRows ?? [];
|
||||
@@ -245,7 +235,11 @@ test('atomically appends a linked queued Run, remote Attempt, events and allowed
|
||||
});
|
||||
|
||||
test('returns durable identities for an exact replay without appending again', async () => {
|
||||
const { calls, repository } = fixture({ replayRows: [replayRow()] });
|
||||
const { calls, repository } = fixture({
|
||||
replayRows: [
|
||||
replayRow({ runStatus: 'running', runVersion: 4, eventSequence: 4 }),
|
||||
],
|
||||
});
|
||||
const result = await repository.retryRun(
|
||||
command({
|
||||
runId: '019f9200-0000-4000-8000-000000000102',
|
||||
@@ -257,6 +251,7 @@ test('returns durable identities for an exact replay without appending again', a
|
||||
assert.equal(result.status, 'existing');
|
||||
assert.equal(result.runId, IDS.runId);
|
||||
assert.equal(result.attemptId, IDS.attemptId);
|
||||
assert.equal(result.runStatus, 'queued');
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql.startsWith('INSERT INTO')),
|
||||
false,
|
||||
@@ -283,15 +278,11 @@ test('rejects stale authentication and changed authorization inside the transact
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
stale.calls.some(({ sql }) => sql.includes('FROM "ql3"."projects"')),
|
||||
stale.calls.some(({ sql }) => sql.includes('lock_run_management_policy_fence')),
|
||||
false,
|
||||
);
|
||||
|
||||
const changed = fixture({
|
||||
bindingRows: [
|
||||
{ bindingVersion: 4, bindingState: 'active', bindingRole: 'operator' },
|
||||
],
|
||||
});
|
||||
const changed = fixture({ authorizationMatches: false });
|
||||
await assert.rejects(
|
||||
changed.repository.retryRun(command()),
|
||||
(error) =>
|
||||
|
||||
Reference in New Issue
Block a user