mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): add strong cluster run management
This commit is contained in:
whyour
@@ -75,6 +75,11 @@
|
||||
"require": "./dist/run-management/runManualRetryRepository.js",
|
||||
"default": "./dist/run-management/runManualRetryRepository.js"
|
||||
},
|
||||
"./run-manager": {
|
||||
"types": "./dist/entrypoints/runManager.d.ts",
|
||||
"require": "./dist/entrypoints/runManager.js",
|
||||
"default": "./dist/entrypoints/runManager.js"
|
||||
},
|
||||
"./approval-manager": {
|
||||
"types": "./dist/approval-management/index.d.ts",
|
||||
"require": "./dist/approval-management/index.js",
|
||||
|
||||
@@ -19,6 +19,7 @@ const DEFAULT_AI_CREDENTIAL_TESTER_APPLICATION_NAME =
|
||||
const DEFAULT_AUTOMATION_MANAGER_APPLICATION_NAME =
|
||||
'qinglong-automation-manager';
|
||||
const DEFAULT_APPROVAL_MANAGER_APPLICATION_NAME = 'qinglong-approval-manager';
|
||||
const DEFAULT_RUN_MANAGER_APPLICATION_NAME = 'qinglong-run-manager';
|
||||
const DEFAULT_PACKAGE_MANAGER_APPLICATION_NAME = 'qinglong-package-manager';
|
||||
const DEFAULT_PACKAGE_EXECUTOR_APPLICATION_NAME = 'qinglong-package-executor';
|
||||
const DEFAULT_WORKER_CREDENTIAL_MANAGER_APPLICATION_NAME =
|
||||
@@ -89,6 +90,7 @@ export type PostgresDatabaseRole =
|
||||
| 'admin'
|
||||
| 'automation-manager'
|
||||
| 'approval-manager'
|
||||
| 'run-manager'
|
||||
| 'package-manager'
|
||||
| 'package-executor'
|
||||
| 'worker-credential-manager'
|
||||
@@ -255,6 +257,7 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig {
|
||||
'admin',
|
||||
'automation-manager',
|
||||
'approval-manager',
|
||||
'run-manager',
|
||||
'package-manager',
|
||||
'package-executor',
|
||||
'worker-credential-manager',
|
||||
@@ -296,6 +299,7 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig {
|
||||
const isAdmin = options.role === 'admin';
|
||||
const isAutomationManager = options.role === 'automation-manager';
|
||||
const isApprovalManager = options.role === 'approval-manager';
|
||||
const isRunManager = options.role === 'run-manager';
|
||||
const isPackageManager = options.role === 'package-manager';
|
||||
const isPackageExecutor = options.role === 'package-executor';
|
||||
const isWorkerCredentialManager =
|
||||
@@ -309,6 +313,7 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig {
|
||||
isAdmin ||
|
||||
isAutomationManager ||
|
||||
isApprovalManager ||
|
||||
isRunManager ||
|
||||
isPackageManager ||
|
||||
isPackageExecutor ||
|
||||
isWorkerCredentialManager ||
|
||||
@@ -370,6 +375,8 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig {
|
||||
? DEFAULT_AUTOMATION_MANAGER_APPLICATION_NAME
|
||||
: isApprovalManager
|
||||
? DEFAULT_APPROVAL_MANAGER_APPLICATION_NAME
|
||||
: isRunManager
|
||||
? DEFAULT_RUN_MANAGER_APPLICATION_NAME
|
||||
: isPackageManager
|
||||
? DEFAULT_PACKAGE_MANAGER_APPLICATION_NAME
|
||||
: isPackageExecutor
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
export { PostgresRunManualRetryRepository } from '../run-management/runManualRetryRepository';
|
||||
export { PostgresProjectPolicyRepository } from '../security/projectPolicyRepository';
|
||||
export { PostgresSecurityAuditRepository } from '../security/securityAuditRepository';
|
||||
export {
|
||||
PostgresPluginPackageIdentityKeysetLedgerRepository as PostgresRunManagementIdentityKeysetLedgerRepository,
|
||||
type ClusterManagementIdentityAuthority,
|
||||
} from '../management/pluginPackageIdentityKeysetLedgerRepository';
|
||||
export {
|
||||
PgPoolBinding,
|
||||
createPostgresDatabaseOpener,
|
||||
isPostgresTlsDnsServername,
|
||||
type OpenPostgresDatabaseOptions,
|
||||
type PostgresConnectionOptions,
|
||||
type PostgresDatabaseRole,
|
||||
type PostgresPoolOptions,
|
||||
} from '../connection/pool';
|
||||
export {
|
||||
PostgresConnectionEnvironmentError,
|
||||
loadPostgresConnectionEnvironment,
|
||||
type PostgresConnectionEnvironment,
|
||||
type PostgresConnectionEnvironmentKeys,
|
||||
} from '../connection/connectionEnvironment';
|
||||
export {
|
||||
loadPostgresCertificateAuthorityFile,
|
||||
type PostgresCertificateAuthorityFileInspection,
|
||||
} from '../connection/certificateAuthority';
|
||||
export {
|
||||
PostgresSchemaReadinessError,
|
||||
assertPostgresRunManagerSchemaReady,
|
||||
type PostgresSchemaReadinessReport,
|
||||
} from '../schema/schemaReadiness';
|
||||
+4
-2
@@ -5,7 +5,8 @@ export type ClusterManagementIdentityAuthority =
|
||||
| 'plugin-package-management'
|
||||
| 'worker-credential-management'
|
||||
| 'automation-management'
|
||||
| 'approval-management';
|
||||
| 'approval-management'
|
||||
| 'run-management';
|
||||
const DIGEST_PATTERN = /^[A-Za-z0-9_-]{43}$/;
|
||||
const KEY_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/;
|
||||
@@ -189,7 +190,8 @@ export class PostgresPluginPackageIdentityKeysetLedgerRepository
|
||||
authority !== 'plugin-package-management' &&
|
||||
authority !== 'worker-credential-management' &&
|
||||
authority !== 'automation-management' &&
|
||||
authority !== 'approval-management'
|
||||
authority !== 'approval-management' &&
|
||||
authority !== 'run-management'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'PostgreSQL management identity keyset authority is invalid',
|
||||
|
||||
@@ -283,5 +283,10 @@ export const postgresqlMainMigrationManifest: MigrationStreamManifest =
|
||||
checksum:
|
||||
'c775c65ec03ae3a1606f899064d2d38fa63fd136ce52cbd1b1172c3a51e6bf30',
|
||||
}),
|
||||
Object.freeze({
|
||||
id: 'pg-0056-run-management-boundary',
|
||||
checksum:
|
||||
'7aa2b2ade67cdfa6839d4af02209906646a68adfd6c12c4dddeb854021da72b8',
|
||||
}),
|
||||
]),
|
||||
});
|
||||
|
||||
@@ -58,6 +58,7 @@ import { pg0052AutomationManagementIdentityKeysetLedgerMigration } from './pg-00
|
||||
import { pg0053PluginPackageWorkflowRunListIndexMigration } from './pg-0053-plugin-package-workflow-run-list-index';
|
||||
import { pg0054ApprovalManagementBoundaryMigration } from './pg-0054-approval-management-boundary';
|
||||
import { pg0055RunAttemptLogRetentionMigration } from './pg-0055-run-attempt-log-retention';
|
||||
import { pg0056RunManagementBoundaryMigration } from '../run-management/pg-0056-run-management-boundary';
|
||||
|
||||
export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMigrationContext> =
|
||||
Object.freeze({
|
||||
@@ -121,5 +122,6 @@ export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMi
|
||||
pg0053PluginPackageWorkflowRunListIndexMigration,
|
||||
pg0054ApprovalManagementBoundaryMigration,
|
||||
pg0055RunAttemptLogRetentionMigration,
|
||||
pg0056RunManagementBoundaryMigration,
|
||||
]),
|
||||
});
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import { CAPABILITIES_V54 } from '../migrations/pg-0055-run-attempt-log-retention';
|
||||
import { definePostgresSqlMigration } from '../migrations/sqlMigration';
|
||||
|
||||
export const CAPABILITIES_V55 = CAPABILITIES_V54.replace(
|
||||
'"run_core":1,',
|
||||
'"run_management_boundary":1,"run_core":1,',
|
||||
);
|
||||
|
||||
export const pg0056RunManagementBoundaryMigration =
|
||||
definePostgresSqlMigration({
|
||||
id: 'pg-0056-run-management-boundary',
|
||||
statements: [
|
||||
`
|
||||
DO $ql3$
|
||||
DECLARE
|
||||
role_invalid boolean;
|
||||
BEGIN
|
||||
SELECT roles.rolname IS NULL
|
||||
OR roles.rolcanlogin IS NOT TRUE
|
||||
OR roles.rolsuper IS NOT FALSE
|
||||
OR roles.rolcreatedb IS NOT FALSE
|
||||
OR roles.rolcreaterole IS NOT FALSE
|
||||
OR roles.rolreplication IS NOT FALSE
|
||||
OR roles.rolbypassrls IS NOT FALSE
|
||||
INTO role_invalid
|
||||
FROM (SELECT 1) AS expected
|
||||
LEFT JOIN pg_catalog.pg_roles AS roles
|
||||
ON roles.rolname = 'ql3_run_manager';
|
||||
|
||||
IF role_invalid IS NOT FALSE THEN
|
||||
RAISE EXCEPTION
|
||||
'required QingLong run manager role is missing or privileged'
|
||||
USING ERRCODE = 'insufficient_privilege';
|
||||
END IF;
|
||||
END
|
||||
$ql3$
|
||||
`.trim(),
|
||||
`
|
||||
DO $ql3$
|
||||
BEGIN
|
||||
EXECUTE format(
|
||||
'GRANT CONNECT ON DATABASE %I TO ql3_run_manager',
|
||||
current_database()
|
||||
);
|
||||
END
|
||||
$ql3$
|
||||
`.trim(),
|
||||
`GRANT USAGE ON SCHEMA "ql3" TO ql3_run_manager`,
|
||||
`REVOKE ALL ON ALL TABLES IN SCHEMA "ql3" FROM ql3_run_manager`,
|
||||
`REVOKE ALL ON ALL FUNCTIONS IN SCHEMA "ql3" FROM ql3_run_manager`,
|
||||
`
|
||||
CREATE FUNCTION "ql3"."lock_run_management_policy_fence"(
|
||||
varchar,
|
||||
varchar,
|
||||
varchar,
|
||||
integer,
|
||||
integer
|
||||
)
|
||||
RETURNS boolean
|
||||
LANGUAGE plpgsql
|
||||
VOLATILE
|
||||
SECURITY DEFINER
|
||||
SET search_path = pg_catalog, ql3
|
||||
AS $ql3$
|
||||
DECLARE
|
||||
project_status varchar;
|
||||
project_version integer;
|
||||
binding_state varchar;
|
||||
binding_role varchar;
|
||||
binding_version integer;
|
||||
BEGIN
|
||||
SELECT project.status, project.version
|
||||
INTO project_status, project_version
|
||||
FROM "ql3"."projects" AS project
|
||||
WHERE project.id = $1
|
||||
FOR UPDATE;
|
||||
|
||||
SELECT binding.state, binding.role, binding.version
|
||||
INTO binding_state, binding_role, binding_version
|
||||
FROM "ql3"."project_role_bindings" AS binding
|
||||
WHERE binding.project_id = $1
|
||||
AND binding.subject_type = $2
|
||||
AND binding.subject_id = $3
|
||||
ORDER BY binding.version DESC
|
||||
LIMIT 1;
|
||||
|
||||
RETURN project_status = 'active'
|
||||
AND project_version = $4
|
||||
AND binding_state = 'active'
|
||||
AND binding_role IN ('owner', 'admin', 'operator')
|
||||
AND binding_version = $5;
|
||||
END
|
||||
$ql3$
|
||||
`.trim(),
|
||||
`REVOKE ALL ON FUNCTION "ql3"."lock_run_management_policy_fence"(varchar, varchar, varchar, integer, integer) FROM PUBLIC`,
|
||||
`GRANT EXECUTE ON FUNCTION "ql3"."lock_run_management_policy_fence"(varchar, varchar, varchar, integer, integer) TO ql3_runtime, ql3_run_manager`,
|
||||
`GRANT SELECT ON "ql3"."schema_migrations", "ql3"."schema_capabilities", "ql3"."projects", "ql3"."project_role_bindings", "ql3"."task_definitions", "ql3"."task_definition_revisions", "ql3"."task_execution_revisions" TO ql3_run_manager`,
|
||||
`GRANT SELECT, INSERT ON "ql3"."runs", "ql3"."run_attempts", "ql3"."run_events", "ql3"."security_audit_events" TO ql3_run_manager`,
|
||||
`GRANT SELECT, INSERT, UPDATE ON "ql3"."plugin_package_identity_keyset_ledger" TO ql3_run_manager`,
|
||||
`ALTER TABLE "ql3"."plugin_package_identity_keyset_ledger" DROP CONSTRAINT ql3_plugin_package_identity_keyset_authority_check`,
|
||||
`ALTER TABLE "ql3"."plugin_package_identity_keyset_ledger" ADD CONSTRAINT ql3_plugin_package_identity_keyset_authority_check CHECK (authority IN ('plugin-package-management', 'worker-credential-management', 'automation-management', 'approval-management', 'run-management'))`,
|
||||
`
|
||||
DO $ql3$
|
||||
BEGIN
|
||||
UPDATE "ql3"."schema_capabilities"
|
||||
SET contract_version = 55,
|
||||
migration_id = 'pg-0056-run-management-boundary',
|
||||
capabilities = '${CAPABILITIES_V55}'::jsonb,
|
||||
updated_at_ms = floor(extract(epoch FROM transaction_timestamp()) * 1000)::bigint
|
||||
WHERE contract_name = 'control-core'
|
||||
AND contract_version = 54
|
||||
AND migration_id = 'pg-0055-run-attempt-log-retention'
|
||||
AND capabilities = '${CAPABILITIES_V54}'::jsonb;
|
||||
IF NOT FOUND THEN
|
||||
RAISE EXCEPTION 'control-core capability is not at version 54'
|
||||
USING ERRCODE = 'check_violation';
|
||||
END IF;
|
||||
END
|
||||
$ql3$
|
||||
`.trim(),
|
||||
],
|
||||
});
|
||||
@@ -9,7 +9,6 @@ import {
|
||||
RunManualRetryUnavailableError,
|
||||
normalizeRunManualRetryCommand,
|
||||
normalizeRunManualRetryResult,
|
||||
type RunManualRetryAllowedRole,
|
||||
type RunManualRetryCommand,
|
||||
type RunManualRetryRepository,
|
||||
type RunManualRetryResult,
|
||||
@@ -33,11 +32,6 @@ type Row = Record<string, unknown>;
|
||||
export const CLUSTER_RUN_MANUAL_RETRY_RATE_WINDOW_MS = 60_000;
|
||||
export const CLUSTER_RUN_MANUAL_RETRY_RATE_LIMIT = 64;
|
||||
|
||||
const ALLOWED_ROLES = new Set<RunManualRetryAllowedRole>([
|
||||
'owner',
|
||||
'admin',
|
||||
'operator',
|
||||
]);
|
||||
const CLUSTER_STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']);
|
||||
const TASK_REVISION_PATTERN = /^qltd:v1:([1-9]\d*):([0-9a-f]{64})$/;
|
||||
|
||||
@@ -128,45 +122,23 @@ async function confirmAuthorization(
|
||||
client: PostgresClient,
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
): Promise<void> {
|
||||
const project = await client.query<Row>(
|
||||
const result = await client.query<Row>(
|
||||
`
|
||||
SELECT status AS "projectStatus", version AS "projectVersion"
|
||||
FROM "ql3"."projects" WHERE id = $1 FOR UPDATE
|
||||
`,
|
||||
[command.projectId],
|
||||
);
|
||||
if (project.rows.length === 0) throw new RunManualRetryNotFoundError();
|
||||
if (project.rows.length !== 1) throw unavailable();
|
||||
// Authorized management mutations take the same Project lock. Keeping this
|
||||
// append-only RoleBinding read lock-free avoids granting UPDATE authority to
|
||||
// the runtime role merely to use PostgreSQL row-lock syntax.
|
||||
const binding = await client.query<Row>(
|
||||
`
|
||||
SELECT version AS "bindingVersion", state AS "bindingState",
|
||||
role AS "bindingRole"
|
||||
FROM "ql3"."project_role_bindings"
|
||||
WHERE project_id = $1 AND subject_type = $2 AND subject_id = $3
|
||||
ORDER BY version DESC LIMIT 1
|
||||
SELECT "ql3"."lock_run_management_policy_fence"(
|
||||
$1::varchar, $2::varchar, $3::varchar, $4::integer, $5::integer
|
||||
) AS "matches"
|
||||
`,
|
||||
[
|
||||
command.projectId,
|
||||
command.principal.subject.type,
|
||||
command.principal.subject.id,
|
||||
command.policyFence.projectVersion,
|
||||
command.policyFence.bindingVersion,
|
||||
],
|
||||
);
|
||||
const currentProject = project.rows[0]!;
|
||||
const currentBinding = binding.rows[0];
|
||||
if (
|
||||
text(currentProject, 'projectStatus') !== 'active' ||
|
||||
integer(currentProject, 'projectVersion') !==
|
||||
command.policyFence.projectVersion ||
|
||||
!currentBinding ||
|
||||
integer(currentBinding, 'bindingVersion') !==
|
||||
command.policyFence.bindingVersion ||
|
||||
text(currentBinding, 'bindingState') !== 'active' ||
|
||||
!ALLOWED_ROLES.has(
|
||||
text(currentBinding, 'bindingRole') as RunManualRetryAllowedRole,
|
||||
)
|
||||
result.rows.length !== 1 ||
|
||||
!postgresRequiredBoolean(result.rows[0]!.matches, unavailable)
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('authorization_changed');
|
||||
}
|
||||
@@ -185,8 +157,6 @@ async function findReplay(
|
||||
run.execution_origin AS "executionOrigin",
|
||||
run.execution_owner AS "executionOwner",
|
||||
run.triggered_by AS "triggeredBy", run.request_id AS "requestId",
|
||||
run.status AS "runStatus", run.version AS "runVersion",
|
||||
run.event_sequence AS "eventSequence",
|
||||
run.created_at_ms AS "createdAtMs",
|
||||
attempt.id AS "attemptId", attempt.executor_type AS "executorType",
|
||||
created.actor_type AS "createdActorType",
|
||||
@@ -205,7 +175,6 @@ async function findReplay(
|
||||
ON queued.run_id = run.id AND queued.sequence = 2
|
||||
AND queued.type = 'run.queued'
|
||||
WHERE run.project_id = $1 AND run.idempotency_key = $2
|
||||
FOR UPDATE OF run
|
||||
`,
|
||||
[command.projectId, `ql3:run-manual-retry:v1:${command.mutationId}`],
|
||||
);
|
||||
@@ -227,9 +196,6 @@ function replayResult(
|
||||
text(row, 'executionOwner') !== 'runtime' ||
|
||||
text(row, 'triggeredBy') !== command.principal.subject.id ||
|
||||
text(row, 'requestId') !== command.mutationId ||
|
||||
text(row, 'runStatus') !== 'queued' ||
|
||||
integer(row, 'runVersion') !== 2 ||
|
||||
integer(row, 'eventSequence') !== 2 ||
|
||||
text(row, 'executorType') !== 'remote_worker' ||
|
||||
text(row, 'createdActorType') !== command.principal.subject.type ||
|
||||
text(row, 'createdActorId') !== command.principal.subject.id ||
|
||||
@@ -295,7 +261,6 @@ async function findSource(
|
||||
LIMIT 1
|
||||
) AS attempt ON true
|
||||
WHERE run.id = $1
|
||||
FOR UPDATE OF run
|
||||
`,
|
||||
[command.sourceRunId],
|
||||
);
|
||||
|
||||
@@ -15,13 +15,14 @@ export interface PostgresSchemaContractFunction {
|
||||
export interface PostgresSchemaContract {
|
||||
readonly schema: 'ql3';
|
||||
readonly contractName: 'control-core';
|
||||
readonly contractVersion: 54;
|
||||
readonly migrationId: 'pg-0055-run-attempt-log-retention';
|
||||
readonly contractVersion: 55;
|
||||
readonly migrationId: 'pg-0056-run-management-boundary';
|
||||
readonly minimumServerMajor: 16;
|
||||
readonly maximumServerMajor: 18;
|
||||
readonly capabilities: Readonly<{
|
||||
run_core: 1;
|
||||
run_attempt_log_retention: 1;
|
||||
run_management_boundary: 1;
|
||||
run_dispatch_lease: 1;
|
||||
run_retry_policy: 1;
|
||||
project_policy: 1;
|
||||
@@ -101,8 +102,8 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
Object.freeze({
|
||||
schema: 'ql3',
|
||||
contractName: 'control-core',
|
||||
contractVersion: 54,
|
||||
migrationId: 'pg-0055-run-attempt-log-retention',
|
||||
contractVersion: 55,
|
||||
migrationId: 'pg-0056-run-management-boundary',
|
||||
minimumServerMajor: 16,
|
||||
maximumServerMajor: 18,
|
||||
capabilities: Object.freeze({
|
||||
@@ -143,6 +144,7 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
project_tool_definition_snapshot: 1,
|
||||
run_core: 1,
|
||||
run_attempt_log_retention: 1,
|
||||
run_management_boundary: 1,
|
||||
run_dispatch_lease: 1,
|
||||
run_retry_policy: 1,
|
||||
security_audit: 1,
|
||||
@@ -2323,6 +2325,15 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
volatility: 'volatile',
|
||||
configuration: Object.freeze(['search_path=pg_catalog, ql3']),
|
||||
}),
|
||||
Object.freeze({
|
||||
name: 'lock_run_management_policy_fence',
|
||||
identityArguments:
|
||||
'character varying, character varying, character varying, integer, integer',
|
||||
owner: 'ql3_migration',
|
||||
securityDefiner: true,
|
||||
volatility: 'volatile',
|
||||
configuration: Object.freeze(['search_path=pg_catalog, ql3']),
|
||||
}),
|
||||
Object.freeze({
|
||||
name: 'commit_plugin_package_task_reconciliation',
|
||||
identityArguments:
|
||||
|
||||
@@ -19,6 +19,7 @@ export const POSTGRES_SCHEMA_READINESS_ERROR_CODES = [
|
||||
'admin_role_invalid',
|
||||
'automation_manager_role_invalid',
|
||||
'approval_manager_role_invalid',
|
||||
'run_manager_role_invalid',
|
||||
'package_manager_role_invalid',
|
||||
'package_executor_role_invalid',
|
||||
'worker_credential_manager_role_invalid',
|
||||
@@ -1361,6 +1362,37 @@ const REQUIRED_APPROVAL_MANAGER_PRIVILEGES: RequiredPrivileges = Object.freeze(
|
||||
),
|
||||
);
|
||||
|
||||
const REQUIRED_RUN_MANAGER_PRIVILEGES: RequiredPrivileges = Object.freeze(
|
||||
Object.fromEntries(
|
||||
postgresqlControlSchemaContract.tables.map(({ name }) => [
|
||||
name,
|
||||
Object.freeze(
|
||||
name === 'schema_migrations' ||
|
||||
name === 'schema_capabilities' ||
|
||||
name === 'projects' ||
|
||||
name === 'project_role_bindings' ||
|
||||
name === 'task_definitions' ||
|
||||
name === 'task_definition_revisions' ||
|
||||
name === 'task_execution_revisions'
|
||||
? { ...NO_TABLE_PRIVILEGES, select: true }
|
||||
: name === 'runs' ||
|
||||
name === 'run_attempts' ||
|
||||
name === 'run_events' ||
|
||||
name === 'security_audit_events'
|
||||
? { ...NO_TABLE_PRIVILEGES, select: true, insert: true }
|
||||
: name === 'plugin_package_identity_keyset_ledger'
|
||||
? {
|
||||
...NO_TABLE_PRIVILEGES,
|
||||
select: true,
|
||||
insert: true,
|
||||
update: true,
|
||||
}
|
||||
: NO_TABLE_PRIVILEGES,
|
||||
),
|
||||
]),
|
||||
),
|
||||
);
|
||||
|
||||
const REQUIRED_WORKER_CREDENTIAL_MANAGER_PRIVILEGES: RequiredPrivileges =
|
||||
Object.freeze(
|
||||
Object.fromEntries(
|
||||
@@ -1448,6 +1480,7 @@ const REQUIRED_RUNTIME_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
|
||||
enforce_plugin_package_stage_provenance: false,
|
||||
lock_active_plugin_package_project: false,
|
||||
lock_approval_policy_fence: false,
|
||||
lock_run_management_policy_fence: true,
|
||||
plugin_package_automation_start_allowed: true,
|
||||
plugin_package_workflow_admission_snapshot: true,
|
||||
plugin_package_workflow_task_attempt_snapshot: true,
|
||||
@@ -1464,6 +1497,7 @@ const REQUIRED_PACKAGE_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
|
||||
enforce_plugin_package_stage_provenance: false,
|
||||
lock_active_plugin_package_project: false,
|
||||
lock_approval_policy_fence: true,
|
||||
lock_run_management_policy_fence: false,
|
||||
plugin_package_automation_start_allowed: false,
|
||||
plugin_package_workflow_admission_snapshot: false,
|
||||
plugin_package_workflow_task_attempt_snapshot: false,
|
||||
@@ -1480,6 +1514,7 @@ const REQUIRED_PACKAGE_EXECUTOR_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges
|
||||
enforce_plugin_package_stage_provenance: false,
|
||||
lock_active_plugin_package_project: true,
|
||||
lock_approval_policy_fence: true,
|
||||
lock_run_management_policy_fence: false,
|
||||
plugin_package_automation_start_allowed: false,
|
||||
plugin_package_workflow_admission_snapshot: false,
|
||||
plugin_package_workflow_task_attempt_snapshot: false,
|
||||
@@ -1500,6 +1535,12 @@ const REQUIRED_APPROVAL_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges
|
||||
lock_approval_policy_fence: true,
|
||||
});
|
||||
|
||||
const REQUIRED_RUN_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
|
||||
Object.freeze({
|
||||
...NO_FUNCTION_PRIVILEGES,
|
||||
lock_run_management_policy_fence: true,
|
||||
});
|
||||
|
||||
function safeInteger(value: unknown): number | null {
|
||||
if (typeof value === 'number' && Number.isSafeInteger(value)) return value;
|
||||
if (typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value)) {
|
||||
@@ -1864,6 +1905,7 @@ async function assertRole(
|
||||
| 'admin_role_invalid'
|
||||
| 'automation_manager_role_invalid'
|
||||
| 'approval_manager_role_invalid'
|
||||
| 'run_manager_role_invalid'
|
||||
| 'package_manager_role_invalid'
|
||||
| 'package_executor_role_invalid'
|
||||
| 'worker_credential_manager_role_invalid'
|
||||
@@ -2120,6 +2162,30 @@ export async function assertPostgresApprovalManagerSchemaReady(
|
||||
});
|
||||
}
|
||||
|
||||
export async function assertPostgresRunManagerSchemaReady(
|
||||
queryable: PostgresMigrationQueryable,
|
||||
contract: PostgresSchemaContract = postgresqlControlSchemaContract,
|
||||
): Promise<PostgresSchemaReadinessReport> {
|
||||
const server = await readServer(queryable, contract);
|
||||
const migrationIds = await assertHistory(queryable);
|
||||
await assertCapability(queryable, contract);
|
||||
await assertSchemaContract(queryable, contract);
|
||||
await assertRole(
|
||||
queryable,
|
||||
contract,
|
||||
REQUIRED_RUN_MANAGER_PRIVILEGES,
|
||||
REQUIRED_RUN_MANAGER_FUNCTION_PRIVILEGES,
|
||||
'run_manager_role_invalid',
|
||||
);
|
||||
return Object.freeze({
|
||||
ready: true,
|
||||
...server,
|
||||
contractName: contract.contractName,
|
||||
contractVersion: contract.contractVersion,
|
||||
migrationIds,
|
||||
});
|
||||
}
|
||||
|
||||
export async function assertPostgresPackageManagerSchemaReady(
|
||||
queryable: PostgresMigrationQueryable,
|
||||
contract: PostgresSchemaContract = postgresqlControlSchemaContract,
|
||||
|
||||
+9
-1
@@ -108,7 +108,7 @@ test('serializes first observation, exact replay and append-only rotation', asyn
|
||||
assert.equal(value.releases(), 3);
|
||||
});
|
||||
|
||||
test('isolates Plugin, Worker, automation and Approval generations by authority key', async () => {
|
||||
test('isolates Plugin, Worker, automation, Approval and Run generations by authority key', async () => {
|
||||
const value = fixture('worker-credential-management');
|
||||
await value.repository.observe(
|
||||
snapshot(1, { audience: 'qinglong3-worker-credential-management' }),
|
||||
@@ -131,6 +131,14 @@ test('isolates Plugin, Worker, automation and Approval generations by authority
|
||||
approval.queries.find(({ text }) => text.startsWith('INSERT')).values[0],
|
||||
'approval-management',
|
||||
);
|
||||
const run = fixture('run-management');
|
||||
await run.repository.observe(
|
||||
snapshot(1, { audience: 'qinglong3-run-management' }),
|
||||
);
|
||||
assert.equal(
|
||||
run.queries.find(({ text }) => text.startsWith('INSERT')).values[0],
|
||||
'run-management',
|
||||
);
|
||||
assert.throws(
|
||||
() => fixture('worker-credential-executor'),
|
||||
TypeError,
|
||||
|
||||
@@ -104,6 +104,7 @@ test('enforces role-specific bounded pool sizes', () => {
|
||||
'ai-credential-tester',
|
||||
'automation-manager',
|
||||
'approval-manager',
|
||||
'run-manager',
|
||||
'worker-credential-manager',
|
||||
'worker-credential-executor',
|
||||
]) {
|
||||
|
||||
@@ -104,6 +104,7 @@ test('defines the immutable PostgreSQL capability and Run core stream', async ()
|
||||
'pg-0053-plugin-package-workflow-run-list-index',
|
||||
'pg-0054-approval-management-boundary',
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
],
|
||||
);
|
||||
for (const migration of postgresqlMainMigrationStream.migrations) {
|
||||
@@ -514,6 +515,11 @@ test('freezes every published PostgreSQL migration checksum', () => {
|
||||
checksum:
|
||||
'c775c65ec03ae3a1606f899064d2d38fa63fd136ce52cbd1b1172c3a51e6bf30',
|
||||
},
|
||||
{
|
||||
id: 'pg-0056-run-management-boundary',
|
||||
checksum:
|
||||
'7aa2b2ade67cdfa6839d4af02209906646a68adfd6c12c4dddeb854021da72b8',
|
||||
},
|
||||
];
|
||||
assert.deepEqual(
|
||||
postgresqlMainMigrationStream.migrations.map(({ id, checksum }) => ({
|
||||
@@ -1904,3 +1910,30 @@ test('advances capability v54 with durable Cluster log retention authority', asy
|
||||
/migration_id = 'pg-0054-approval-management-boundary'/,
|
||||
);
|
||||
});
|
||||
|
||||
test('advances capability v55 with isolated strong Run management authority', async () => {
|
||||
const migration = migrationById('pg-0056-run-management-boundary');
|
||||
const statements = [];
|
||||
await migration.up({
|
||||
async query(statement) {
|
||||
statements.push(statement);
|
||||
return { rows: [] };
|
||||
},
|
||||
});
|
||||
const sql = statements.join('\n');
|
||||
assert.match(sql, /ql3_run_manager/);
|
||||
assert.match(sql, /lock_run_management_policy_fence/);
|
||||
assert.match(
|
||||
sql,
|
||||
/GRANT SELECT, INSERT ON "ql3"\."runs", "ql3"\."run_attempts", "ql3"\."run_events", "ql3"\."security_audit_events" TO ql3_run_manager/,
|
||||
);
|
||||
assert.doesNotMatch(sql, /GRANT UPDATE ON "ql3"\."runs"/);
|
||||
assert.match(sql, /'run-management'/);
|
||||
assert.match(sql, /contract_version = 55/);
|
||||
assert.match(sql, /"run_management_boundary":1/);
|
||||
assert.match(sql, /contract_version = 54/);
|
||||
assert.match(
|
||||
sql,
|
||||
/migration_id = 'pg-0055-run-attempt-log-retention'/,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -4,6 +4,7 @@ const {
|
||||
PostgresSchemaReadinessError,
|
||||
assertPostgresAdminSchemaReady,
|
||||
assertPostgresApprovalManagerSchemaReady,
|
||||
assertPostgresRunManagerSchemaReady,
|
||||
assertPostgresAutomationManagerSchemaReady,
|
||||
assertPostgresPackageExecutorSchemaReady,
|
||||
assertPostgresPackageManagerSchemaReady,
|
||||
@@ -474,6 +475,37 @@ function approvalManagerPrivileges() {
|
||||
}));
|
||||
}
|
||||
|
||||
function runManagerPrivileges() {
|
||||
const readable = new Set([
|
||||
'schema_migrations',
|
||||
'schema_capabilities',
|
||||
'projects',
|
||||
'project_role_bindings',
|
||||
'task_definitions',
|
||||
'task_definition_revisions',
|
||||
'task_execution_revisions',
|
||||
'runs',
|
||||
'run_attempts',
|
||||
'run_events',
|
||||
'security_audit_events',
|
||||
'plugin_package_identity_keyset_ledger',
|
||||
]);
|
||||
return postgresqlControlSchemaContract.tables.map(({ name: tableName }) => ({
|
||||
tableName,
|
||||
selectAllowed: readable.has(tableName),
|
||||
insertAllowed: [
|
||||
'runs',
|
||||
'run_attempts',
|
||||
'run_events',
|
||||
'security_audit_events',
|
||||
'plugin_package_identity_keyset_ledger',
|
||||
].includes(tableName),
|
||||
updateAllowed: tableName === 'plugin_package_identity_keyset_ledger',
|
||||
deleteAllowed: false,
|
||||
isOwner: false,
|
||||
}));
|
||||
}
|
||||
|
||||
function workerCredentialPrivileges(kind) {
|
||||
const manager = kind === 'manager';
|
||||
const readable = new Set([
|
||||
@@ -634,6 +666,8 @@ function queryable(overrides = {}) {
|
||||
executeAllowed:
|
||||
overrides.functionMode === 'manager'
|
||||
? functionName === 'lock_approval_policy_fence'
|
||||
: overrides.functionMode === 'run-manager'
|
||||
? functionName === 'lock_run_management_policy_fence'
|
||||
: overrides.functionMode === 'executor'
|
||||
? [
|
||||
'commit_plugin_package_lifecycle',
|
||||
@@ -651,6 +685,7 @@ function queryable(overrides = {}) {
|
||||
'plugin_package_workflow_task_attempt_snapshot',
|
||||
'plugin_package_run_start_allowed',
|
||||
'plugin_package_tool_start_allowed',
|
||||
'lock_run_management_policy_fence',
|
||||
].includes(functionName),
|
||||
isOwner: false,
|
||||
})),
|
||||
@@ -696,7 +731,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
|
||||
serverMajor: 16,
|
||||
currentUser: 'ql3_runtime',
|
||||
contractName: 'control-core',
|
||||
contractVersion: 54,
|
||||
contractVersion: 55,
|
||||
migrationIds: [
|
||||
'pg-0001-schema-capability',
|
||||
'pg-0002-run-core',
|
||||
@@ -753,6 +788,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
|
||||
'pg-0053-plugin-package-workflow-run-list-index',
|
||||
'pg-0054-approval-management-boundary',
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
],
|
||||
});
|
||||
});
|
||||
@@ -783,10 +819,10 @@ test('accepts the exact schema and isolated least-privilege admin role', async (
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_admin');
|
||||
assert.equal(report.contractVersion, 54);
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -799,10 +835,10 @@ test('accepts the isolated least-privilege automation manager role', async () =>
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_automation_manager');
|
||||
assert.equal(report.contractVersion, 54);
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
);
|
||||
|
||||
const widened = automationManagerPrivileges();
|
||||
@@ -831,10 +867,10 @@ test('accepts the isolated least-privilege human Approval manager role', async (
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_approval_manager');
|
||||
assert.equal(report.contractVersion, 54);
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
);
|
||||
|
||||
const widened = approvalManagerPrivileges();
|
||||
@@ -856,6 +892,35 @@ test('accepts the isolated least-privilege human Approval manager role', async (
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts the isolated least-privilege Run manager role', async () => {
|
||||
const report = await assertPostgresRunManagerSchemaReady(
|
||||
queryable({
|
||||
currentUser: 'ql3_run_manager',
|
||||
privileges: runManagerPrivileges(),
|
||||
functionMode: 'run-manager',
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_run_manager');
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(report.migrationIds.at(-1), 'pg-0056-run-management-boundary');
|
||||
|
||||
const widened = runManagerPrivileges();
|
||||
widened.find(({ tableName }) => tableName === 'runs').updateAllowed = true;
|
||||
await assert.rejects(
|
||||
assertPostgresRunManagerSchemaReady(
|
||||
queryable({
|
||||
currentUser: 'ql3_run_manager',
|
||||
privileges: widened,
|
||||
functionMode: 'run-manager',
|
||||
}),
|
||||
),
|
||||
(error) =>
|
||||
error instanceof PostgresSchemaReadinessError &&
|
||||
error.code === 'run_manager_role_invalid' &&
|
||||
error.facts.includes('table-privileges:runs'),
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts isolated Package manager and executor roles', async () => {
|
||||
const manager = await assertPostgresPackageManagerSchemaReady(
|
||||
queryable({
|
||||
@@ -941,10 +1006,10 @@ test('accepts the exact schema and isolated Worker ingress role', async () => {
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_worker_ingress');
|
||||
assert.equal(report.contractVersion, 54);
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -138,21 +138,11 @@ function fixture(options = {}) {
|
||||
if (normalized.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: 1_000_000 }], rowCount: 1 };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."projects"')) {
|
||||
const rows = options.projectRows ?? [
|
||||
{ projectStatus: 'active', projectVersion: 2 },
|
||||
];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('project_role_bindings')) {
|
||||
const rows = options.bindingRows ?? [
|
||||
{
|
||||
bindingVersion: 3,
|
||||
bindingState: 'active',
|
||||
bindingRole: 'operator',
|
||||
},
|
||||
];
|
||||
return { rows, rowCount: rows.length };
|
||||
if (normalized.includes('lock_run_management_policy_fence')) {
|
||||
return {
|
||||
rows: [{ matches: options.authorizationMatches ?? true }],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('idempotency_key = $2')) {
|
||||
const rows = options.replayRows ?? [];
|
||||
@@ -245,7 +235,11 @@ test('atomically appends a linked queued Run, remote Attempt, events and allowed
|
||||
});
|
||||
|
||||
test('returns durable identities for an exact replay without appending again', async () => {
|
||||
const { calls, repository } = fixture({ replayRows: [replayRow()] });
|
||||
const { calls, repository } = fixture({
|
||||
replayRows: [
|
||||
replayRow({ runStatus: 'running', runVersion: 4, eventSequence: 4 }),
|
||||
],
|
||||
});
|
||||
const result = await repository.retryRun(
|
||||
command({
|
||||
runId: '019f9200-0000-4000-8000-000000000102',
|
||||
@@ -257,6 +251,7 @@ test('returns durable identities for an exact replay without appending again', a
|
||||
assert.equal(result.status, 'existing');
|
||||
assert.equal(result.runId, IDS.runId);
|
||||
assert.equal(result.attemptId, IDS.attemptId);
|
||||
assert.equal(result.runStatus, 'queued');
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql.startsWith('INSERT INTO')),
|
||||
false,
|
||||
@@ -283,15 +278,11 @@ test('rejects stale authentication and changed authorization inside the transact
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
stale.calls.some(({ sql }) => sql.includes('FROM "ql3"."projects"')),
|
||||
stale.calls.some(({ sql }) => sql.includes('lock_run_management_policy_fence')),
|
||||
false,
|
||||
);
|
||||
|
||||
const changed = fixture({
|
||||
bindingRows: [
|
||||
{ bindingVersion: 4, bindingState: 'active', bindingRole: 'operator' },
|
||||
],
|
||||
});
|
||||
const changed = fixture({ authorizationMatches: false });
|
||||
await assert.rejects(
|
||||
changed.repository.retryRun(command()),
|
||||
(error) =>
|
||||
|
||||
Reference in New Issue
Block a user