mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): add strong cluster run stop
This commit is contained in:
@@ -1,4 +1,8 @@
|
||||
export { PostgresRunManualRetryRepository } from '../run-management/runManualRetryRepository';
|
||||
export {
|
||||
PostgresClusterRunCancellationRepository,
|
||||
type PostgresRunManagementCancellationCommand,
|
||||
} from '../run-recovery/clusterRunCancellationRepository';
|
||||
export { PostgresProjectPolicyRepository } from '../security/projectPolicyRepository';
|
||||
export { PostgresSecurityAuditRepository } from '../security/securityAuditRepository';
|
||||
export {
|
||||
|
||||
@@ -288,5 +288,10 @@ export const postgresqlMainMigrationManifest: MigrationStreamManifest =
|
||||
checksum:
|
||||
'7aa2b2ade67cdfa6839d4af02209906646a68adfd6c12c4dddeb854021da72b8',
|
||||
}),
|
||||
Object.freeze({
|
||||
id: 'pg-0057-run-management-stop-boundary',
|
||||
checksum:
|
||||
'ab2d0eee3d85a937e1e87243b1fd1e75181529122b64026303488404162e4ba7',
|
||||
}),
|
||||
]),
|
||||
});
|
||||
|
||||
@@ -59,6 +59,7 @@ import { pg0053PluginPackageWorkflowRunListIndexMigration } from './pg-0053-plug
|
||||
import { pg0054ApprovalManagementBoundaryMigration } from './pg-0054-approval-management-boundary';
|
||||
import { pg0055RunAttemptLogRetentionMigration } from './pg-0055-run-attempt-log-retention';
|
||||
import { pg0056RunManagementBoundaryMigration } from '../run-management/pg-0056-run-management-boundary';
|
||||
import { pg0057RunManagementStopBoundaryMigration } from '../run-management/pg-0057-run-management-stop-boundary';
|
||||
|
||||
export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMigrationContext> =
|
||||
Object.freeze({
|
||||
@@ -123,5 +124,6 @@ export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMi
|
||||
pg0054ApprovalManagementBoundaryMigration,
|
||||
pg0055RunAttemptLogRetentionMigration,
|
||||
pg0056RunManagementBoundaryMigration,
|
||||
pg0057RunManagementStopBoundaryMigration,
|
||||
]),
|
||||
});
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
import { CAPABILITIES_V55 } from './pg-0056-run-management-boundary';
|
||||
import { definePostgresSqlMigration } from '../migrations/sqlMigration';
|
||||
|
||||
export const CAPABILITIES_V56 = CAPABILITIES_V55.replace(
|
||||
'"run_management_boundary":1,',
|
||||
'"run_management_boundary":1,"run_management_stop":1,',
|
||||
);
|
||||
|
||||
export const pg0057RunManagementStopBoundaryMigration =
|
||||
definePostgresSqlMigration({
|
||||
id: 'pg-0057-run-management-stop-boundary',
|
||||
statements: [
|
||||
`REVOKE UPDATE ON "ql3"."runs" FROM ql3_run_manager`,
|
||||
`GRANT UPDATE (cancel_requested_at_ms, cancel_reason, version, event_sequence) ON "ql3"."runs" TO ql3_run_manager`,
|
||||
`
|
||||
DO $ql3$
|
||||
BEGIN
|
||||
UPDATE "ql3"."schema_capabilities"
|
||||
SET contract_version = 56,
|
||||
migration_id = 'pg-0057-run-management-stop-boundary',
|
||||
capabilities = '${CAPABILITIES_V56}'::jsonb,
|
||||
updated_at_ms = floor(extract(epoch FROM transaction_timestamp()) * 1000)::bigint
|
||||
WHERE contract_name = 'control-core'
|
||||
AND contract_version = 55
|
||||
AND migration_id = 'pg-0056-run-management-boundary'
|
||||
AND capabilities = '${CAPABILITIES_V55}'::jsonb;
|
||||
IF NOT FOUND THEN
|
||||
RAISE EXCEPTION 'control-core capability is not at version 55'
|
||||
USING ERRCODE = 'check_violation';
|
||||
END IF;
|
||||
END
|
||||
$ql3$
|
||||
`.trim(),
|
||||
],
|
||||
});
|
||||
+352
-109
@@ -7,20 +7,25 @@ import {
|
||||
InvalidClusterRunCancellationError,
|
||||
normalizeClusterRunCancellationCommand,
|
||||
normalizeClusterRunCancellationResult,
|
||||
type ClusterRunCancellationAllowedRole,
|
||||
type ClusterRunCancellationCommand,
|
||||
type ClusterRunCancellationRepository,
|
||||
type ClusterRunCancellationResult,
|
||||
} from '@qinglong/runtime-core/cluster-run-cancellation';
|
||||
import { RUN_STATUSES, type RunStatus } from '@qinglong/runtime-core';
|
||||
import {
|
||||
RUN_STATUSES,
|
||||
type RunStatus,
|
||||
type SecurityPolicyFence,
|
||||
type SecurityPrincipal,
|
||||
} from '@qinglong/runtime-core';
|
||||
import { normalizeSecurityPrincipal } from '@qinglong/runtime-core/security';
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
|
||||
const ALLOWED_ROLES = new Set<ClusterRunCancellationAllowedRole>([
|
||||
'owner',
|
||||
'admin',
|
||||
'operator',
|
||||
]);
|
||||
const STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']);
|
||||
const MAX_AUTHENTICATION_AGE_MS = 5 * 60 * 1_000;
|
||||
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const UUID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
const TERMINAL = new Set<RunStatus>([
|
||||
'succeeded',
|
||||
'failed',
|
||||
@@ -35,6 +40,23 @@ const CANCEL_REASONS = new Set([
|
||||
'timeout',
|
||||
]);
|
||||
|
||||
export interface PostgresRunManagementCancellationCommand {
|
||||
readonly projectId: string;
|
||||
readonly runId: string;
|
||||
readonly mutationId: string;
|
||||
readonly eventId: string;
|
||||
readonly requestId: string;
|
||||
readonly auditEventId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
readonly policyFence: Readonly<SecurityPolicyFence>;
|
||||
}
|
||||
|
||||
interface CancellationAudit {
|
||||
readonly requestId: string;
|
||||
readonly auditEventId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
}
|
||||
|
||||
function text(row: Row, key: string): string {
|
||||
const value = row[key];
|
||||
if (typeof value !== 'string' || value.length < 1) {
|
||||
@@ -45,14 +67,9 @@ function text(row: Row, key: string): string {
|
||||
|
||||
function integer(row: Row, key: string): number {
|
||||
const raw = row[key];
|
||||
const value = typeof raw === 'string' && /^(0|[1-9]\d*)$/.test(raw)
|
||||
? Number(raw)
|
||||
: raw;
|
||||
if (
|
||||
typeof value !== 'number' ||
|
||||
!Number.isSafeInteger(value) ||
|
||||
value < 0
|
||||
) {
|
||||
const value =
|
||||
typeof raw === 'string' && /^(0|[1-9]\d*)$/.test(raw) ? Number(raw) : raw;
|
||||
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
|
||||
throw new TypeError(`PostgreSQL Run cancellation ${key} is invalid`);
|
||||
}
|
||||
return value;
|
||||
@@ -70,6 +87,94 @@ function optionalText(row: Row, key: string): string | undefined {
|
||||
: text(row, key);
|
||||
}
|
||||
|
||||
function exact(
|
||||
value: unknown,
|
||||
keys: readonly string[],
|
||||
): Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new InvalidClusterRunCancellationError(
|
||||
'management command is invalid',
|
||||
);
|
||||
}
|
||||
const actual = Object.keys(value).sort();
|
||||
const expected = [...keys].sort();
|
||||
if (
|
||||
actual.length !== expected.length ||
|
||||
actual.some((key, index) => key !== expected[index])
|
||||
) {
|
||||
throw new InvalidClusterRunCancellationError(
|
||||
'management command shape is invalid',
|
||||
);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function managementIdentifier(value: unknown, name: string): string {
|
||||
if (typeof value !== 'string' || !IDENTIFIER_PATTERN.test(value)) {
|
||||
throw new InvalidClusterRunCancellationError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function managementUuid(value: unknown, name: string): string {
|
||||
if (typeof value !== 'string' || !UUID_PATTERN.test(value)) {
|
||||
throw new InvalidClusterRunCancellationError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function normalizeManagementCommand(
|
||||
value: Readonly<PostgresRunManagementCancellationCommand>,
|
||||
): Readonly<{
|
||||
command: Readonly<ClusterRunCancellationCommand>;
|
||||
audit: Readonly<CancellationAudit>;
|
||||
}> {
|
||||
const input = exact(value, [
|
||||
'projectId',
|
||||
'runId',
|
||||
'mutationId',
|
||||
'eventId',
|
||||
'requestId',
|
||||
'auditEventId',
|
||||
'principal',
|
||||
'policyFence',
|
||||
]);
|
||||
const principalInput = exact(input.principal, [
|
||||
'subject',
|
||||
'authenticationId',
|
||||
'authenticatedAtMs',
|
||||
'expiresAtMs',
|
||||
'assurance',
|
||||
]) as unknown as SecurityPrincipal;
|
||||
const projectId = managementIdentifier(input.projectId, 'projectId');
|
||||
const runId = managementIdentifier(input.runId, 'runId');
|
||||
const mutationId = managementUuid(input.mutationId, 'mutationId');
|
||||
const eventId = managementUuid(input.eventId, 'eventId');
|
||||
const requestId = managementIdentifier(input.requestId, 'requestId');
|
||||
const auditEventId = managementUuid(input.auditEventId, 'auditEventId');
|
||||
if (eventId === auditEventId) {
|
||||
throw new InvalidClusterRunCancellationError(
|
||||
'event and audit identity must differ',
|
||||
);
|
||||
}
|
||||
const command = normalizeClusterRunCancellationCommand({
|
||||
projectId,
|
||||
runId,
|
||||
mutationId,
|
||||
eventId,
|
||||
subject: principalInput.subject,
|
||||
policyFence: input.policyFence as SecurityPolicyFence,
|
||||
});
|
||||
return Object.freeze({
|
||||
command,
|
||||
audit: Object.freeze({
|
||||
requestId,
|
||||
auditEventId,
|
||||
principal: principalInput,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
function runStatus(row: Row): RunStatus {
|
||||
const value = text(row, 'runStatus') as RunStatus;
|
||||
if (!RUN_STATUSES.includes(value)) {
|
||||
@@ -131,6 +236,122 @@ async function databaseNow(client: PostgresClient): Promise<number> {
|
||||
return integer(result.rows[0]!, 'nowMs');
|
||||
}
|
||||
|
||||
function confirmStrongAuthentication(
|
||||
value: Readonly<SecurityPrincipal>,
|
||||
observedAtMs: number,
|
||||
): Readonly<SecurityPrincipal> {
|
||||
let principal: Readonly<SecurityPrincipal>;
|
||||
try {
|
||||
principal = normalizeSecurityPrincipal(value, observedAtMs);
|
||||
} catch {
|
||||
throw new ClusterRunCancellationFenceRejectedError('authorization_changed');
|
||||
}
|
||||
if (
|
||||
principal.subject.type !== 'user' ||
|
||||
!STRONG_ASSURANCES.has(principal.assurance) ||
|
||||
principal.authenticatedAtMs > observedAtMs ||
|
||||
principal.expiresAtMs <= observedAtMs ||
|
||||
observedAtMs - principal.authenticatedAtMs > MAX_AUTHENTICATION_AGE_MS
|
||||
) {
|
||||
throw new ClusterRunCancellationFenceRejectedError('authorization_changed');
|
||||
}
|
||||
return principal;
|
||||
}
|
||||
|
||||
async function confirmAuthorization(
|
||||
client: PostgresClient,
|
||||
command: Readonly<ClusterRunCancellationCommand>,
|
||||
): Promise<void> {
|
||||
const result = await client.query<Row>(
|
||||
`
|
||||
SELECT "ql3"."lock_run_management_policy_fence"(
|
||||
$1::varchar, $2::varchar, $3::varchar, $4::integer, $5::integer
|
||||
) AS "matches"
|
||||
`,
|
||||
[
|
||||
command.projectId,
|
||||
command.subject.type,
|
||||
command.subject.id,
|
||||
command.policyFence.projectVersion,
|
||||
command.policyFence.bindingVersion,
|
||||
],
|
||||
);
|
||||
if (result.rows.length !== 1 || result.rows[0]?.matches !== true) {
|
||||
throw new ClusterRunCancellationFenceRejectedError('authorization_changed');
|
||||
}
|
||||
}
|
||||
|
||||
async function recordAllowedAudit(
|
||||
client: PostgresClient,
|
||||
command: Readonly<ClusterRunCancellationCommand>,
|
||||
audit: Readonly<CancellationAudit>,
|
||||
observedAtMs: number,
|
||||
): Promise<void> {
|
||||
const inserted = await client.query<Row>(
|
||||
`
|
||||
INSERT INTO "ql3"."security_audit_events" (
|
||||
event_id, request_id, operation_id, project_id,
|
||||
subject_type, subject_id, authentication_id, outcome, reasons,
|
||||
project_version, binding_version, occurred_at_ms
|
||||
) VALUES (
|
||||
$1, $2, 'run.stop', $3, $4, $5, $6, 'allowed', $7::jsonb,
|
||||
$8, $9, $10
|
||||
)
|
||||
ON CONFLICT (event_id) DO NOTHING
|
||||
RETURNING event_id AS "eventId"
|
||||
`,
|
||||
[
|
||||
audit.auditEventId,
|
||||
audit.requestId,
|
||||
command.projectId,
|
||||
audit.principal.subject.type,
|
||||
audit.principal.subject.id,
|
||||
audit.principal.authenticationId,
|
||||
JSON.stringify(['role_grant', 'strong_authentication']),
|
||||
command.policyFence.projectVersion,
|
||||
command.policyFence.bindingVersion,
|
||||
observedAtMs,
|
||||
],
|
||||
);
|
||||
if (inserted.rows.length === 1) return;
|
||||
if (inserted.rows.length !== 0) {
|
||||
throw new TypeError('PostgreSQL Run cancellation audit is invalid');
|
||||
}
|
||||
const replay = await client.query<Row>(
|
||||
`
|
||||
SELECT request_id AS "requestId", operation_id AS "operationId",
|
||||
project_id AS "projectId", subject_type AS "subjectType",
|
||||
subject_id AS "subjectId", authentication_id AS "authenticationId",
|
||||
outcome, reasons, project_version AS "projectVersion",
|
||||
binding_version AS "bindingVersion"
|
||||
FROM "ql3"."security_audit_events"
|
||||
WHERE event_id = $1
|
||||
`,
|
||||
[audit.auditEventId],
|
||||
);
|
||||
const row = replay.rows[0];
|
||||
const reasons = row?.reasons;
|
||||
if (
|
||||
replay.rows.length !== 1 ||
|
||||
!row ||
|
||||
row.requestId !== audit.requestId ||
|
||||
row.operationId !== 'run.stop' ||
|
||||
row.projectId !== command.projectId ||
|
||||
row.subjectType !== audit.principal.subject.type ||
|
||||
row.subjectId !== audit.principal.subject.id ||
|
||||
row.authenticationId !== audit.principal.authenticationId ||
|
||||
row.outcome !== 'allowed' ||
|
||||
!Array.isArray(reasons) ||
|
||||
reasons.length !== 2 ||
|
||||
reasons[0] !== 'role_grant' ||
|
||||
reasons[1] !== 'strong_authentication' ||
|
||||
integer(row, 'projectVersion') !== command.policyFence.projectVersion ||
|
||||
integer(row, 'bindingVersion') !== command.policyFence.bindingVersion
|
||||
) {
|
||||
throw new TypeError('PostgreSQL Run cancellation audit replay drifted');
|
||||
}
|
||||
}
|
||||
|
||||
async function rollback(client: PostgresClient): Promise<void> {
|
||||
try {
|
||||
await client.query('ROLLBACK');
|
||||
@@ -140,7 +361,8 @@ async function rollback(client: PostgresClient): Promise<void> {
|
||||
}
|
||||
|
||||
export class PostgresClusterRunCancellationRepository
|
||||
implements ClusterRunCancellationRepository {
|
||||
implements ClusterRunCancellationRepository
|
||||
{
|
||||
constructor(private readonly pool: PostgresPool) {
|
||||
if (!pool || typeof pool.connect !== 'function') {
|
||||
throw new TypeError('PostgreSQL Run cancellation pool is invalid');
|
||||
@@ -151,67 +373,71 @@ export class PostgresClusterRunCancellationRepository
|
||||
value: Readonly<ClusterRunCancellationCommand>,
|
||||
): Promise<Readonly<ClusterRunCancellationResult>> {
|
||||
const command = normalizeClusterRunCancellationCommand(value);
|
||||
return this.requestCancellation(command);
|
||||
}
|
||||
|
||||
async requestUserCancellationAudited(
|
||||
value: Readonly<PostgresRunManagementCancellationCommand>,
|
||||
): Promise<Readonly<ClusterRunCancellationResult>> {
|
||||
const normalized = normalizeManagementCommand(value);
|
||||
return this.requestCancellation(normalized.command, normalized.audit);
|
||||
}
|
||||
|
||||
private requestCancellation(
|
||||
command: Readonly<ClusterRunCancellationCommand>,
|
||||
audit?: Readonly<CancellationAudit>,
|
||||
): Promise<Readonly<ClusterRunCancellationResult>> {
|
||||
return this.transaction(async (client) => {
|
||||
const project = await client.query<Row>(`
|
||||
SELECT status AS "projectStatus", version AS "projectVersion"
|
||||
FROM "ql3"."projects" WHERE id = $1 FOR UPDATE
|
||||
`, [command.projectId]);
|
||||
if (project.rows.length === 0) {
|
||||
throw new ClusterRunCancellationNotFoundError();
|
||||
}
|
||||
if (project.rows.length !== 1) {
|
||||
throw new TypeError('PostgreSQL Run cancellation Project is invalid');
|
||||
}
|
||||
const binding = await client.query<Row>(`
|
||||
SELECT version AS "bindingVersion", state AS "bindingState",
|
||||
role AS "bindingRole"
|
||||
FROM "ql3"."project_role_bindings"
|
||||
WHERE project_id = $1 AND subject_type = $2 AND subject_id = $3
|
||||
ORDER BY version DESC LIMIT 1
|
||||
`, [
|
||||
command.projectId,
|
||||
command.subject.type,
|
||||
command.subject.id,
|
||||
]);
|
||||
const currentProject = project.rows[0]!;
|
||||
const currentBinding = binding.rows[0];
|
||||
const observedAtMs = audit ? await databaseNow(client) : undefined;
|
||||
const confirmedAudit = audit
|
||||
? Object.freeze({
|
||||
...audit,
|
||||
principal: confirmStrongAuthentication(
|
||||
audit.principal,
|
||||
observedAtMs!,
|
||||
),
|
||||
})
|
||||
: undefined;
|
||||
if (
|
||||
text(currentProject, 'projectStatus') !== 'active' ||
|
||||
integer(currentProject, 'projectVersion') !==
|
||||
command.policyFence.projectVersion ||
|
||||
!currentBinding ||
|
||||
integer(currentBinding, 'bindingVersion') !==
|
||||
command.policyFence.bindingVersion ||
|
||||
text(currentBinding, 'bindingState') !== 'active' ||
|
||||
!ALLOWED_ROLES.has(
|
||||
text(currentBinding, 'bindingRole') as ClusterRunCancellationAllowedRole,
|
||||
)
|
||||
confirmedAudit &&
|
||||
(confirmedAudit.principal.subject.type !== command.subject.type ||
|
||||
confirmedAudit.principal.subject.id !== command.subject.id)
|
||||
) {
|
||||
throw new ClusterRunCancellationFenceRejectedError(
|
||||
'authorization_changed',
|
||||
);
|
||||
}
|
||||
await confirmAuthorization(client, command);
|
||||
|
||||
const run = await client.query<Row>(`
|
||||
const run = await client.query<Row>(
|
||||
`
|
||||
SELECT project_id AS "projectId", status AS "runStatus",
|
||||
version AS "runVersion", event_sequence AS "eventSequence",
|
||||
cancel_requested_at_ms AS "cancelRequestedAtMs",
|
||||
cancel_reason AS "cancelReason"
|
||||
FROM "ql3"."runs" WHERE id = $1 FOR UPDATE
|
||||
`, [command.runId]);
|
||||
if (run.rows.length === 0 || run.rows[0]?.projectId !== command.projectId) {
|
||||
`,
|
||||
[command.runId],
|
||||
);
|
||||
if (
|
||||
run.rows.length === 0 ||
|
||||
run.rows[0]?.projectId !== command.projectId
|
||||
) {
|
||||
throw new ClusterRunCancellationNotFoundError();
|
||||
}
|
||||
if (run.rows.length !== 1) {
|
||||
throw new TypeError('PostgreSQL Run cancellation Run is invalid');
|
||||
}
|
||||
if (command.workflowTarget) {
|
||||
const admission = await client.query<Row>(`
|
||||
const admission = await client.query<Row>(
|
||||
`
|
||||
SELECT project_id AS "projectId", package_name AS "packageName",
|
||||
workflow_id AS "workflowId"
|
||||
FROM "ql3"."plugin_package_workflow_admissions"
|
||||
WHERE run_id = $1
|
||||
`, [command.runId]);
|
||||
`,
|
||||
[command.runId],
|
||||
);
|
||||
const target = admission.rows[0];
|
||||
if (
|
||||
admission.rows.length !== 1 ||
|
||||
@@ -225,65 +451,82 @@ export class PostgresClusterRunCancellationRepository
|
||||
}
|
||||
const current = run.rows[0]!;
|
||||
const currentStatus = runStatus(current);
|
||||
let result: Readonly<ClusterRunCancellationResult>;
|
||||
if (TERMINAL.has(currentStatus)) {
|
||||
return cancellationResult('already_terminal', command, current);
|
||||
}
|
||||
if (optionalInteger(current, 'cancelRequestedAtMs') !== undefined) {
|
||||
return cancellationResult('already_requested', command, current);
|
||||
}
|
||||
if (optionalText(current, 'cancelReason') !== undefined) {
|
||||
result = cancellationResult('already_terminal', command, current);
|
||||
} else if (
|
||||
optionalInteger(current, 'cancelRequestedAtMs') !== undefined
|
||||
) {
|
||||
result = cancellationResult('already_requested', command, current);
|
||||
} else if (optionalText(current, 'cancelReason') !== undefined) {
|
||||
throw new TypeError('PostgreSQL Run cancellation intent is invalid');
|
||||
} else {
|
||||
const runVersion = integer(current, 'runVersion');
|
||||
const eventSequence = integer(current, 'eventSequence');
|
||||
if (runVersion >= 2_147_483_647 || eventSequence >= 2_147_483_647) {
|
||||
throw new TypeError('PostgreSQL Run cancellation counter overflowed');
|
||||
}
|
||||
const mutationObservedAtMs =
|
||||
observedAtMs ?? (await databaseNow(client));
|
||||
const updated = await client.query<Row>(
|
||||
`
|
||||
UPDATE "ql3"."runs"
|
||||
SET cancel_requested_at_ms = $2, cancel_reason = 'user',
|
||||
version = $3, event_sequence = $4
|
||||
WHERE id = $1 AND version = $5 AND cancel_requested_at_ms IS NULL
|
||||
RETURNING project_id AS "projectId", status AS "runStatus",
|
||||
version AS "runVersion", event_sequence AS "eventSequence",
|
||||
cancel_requested_at_ms AS "cancelRequestedAtMs",
|
||||
cancel_reason AS "cancelReason"
|
||||
`,
|
||||
[
|
||||
command.runId,
|
||||
mutationObservedAtMs,
|
||||
runVersion + 1,
|
||||
eventSequence + 1,
|
||||
runVersion,
|
||||
],
|
||||
);
|
||||
if (updated.rows.length !== 1) {
|
||||
throw new ClusterRunCancellationFenceRejectedError('state_mismatch');
|
||||
}
|
||||
await client.query(
|
||||
`
|
||||
INSERT INTO "ql3"."run_events" (
|
||||
id, run_id, sequence, type, dedupe_key, actor_type, actor_id,
|
||||
attempt_id, step_run_id, payload, created_at_ms
|
||||
) VALUES ($1, $2, $3, 'run.cancel_requested', $4, $5, $6,
|
||||
NULL, NULL, $7::jsonb, $8)
|
||||
`,
|
||||
[
|
||||
command.eventId,
|
||||
command.runId,
|
||||
eventSequence + 1,
|
||||
`user-cancel:${command.mutationId}`,
|
||||
command.subject.type,
|
||||
command.subject.id,
|
||||
JSON.stringify({
|
||||
reason: 'user',
|
||||
mutation_id: command.mutationId,
|
||||
policy_fence: {
|
||||
project_version: command.policyFence.projectVersion,
|
||||
binding_version: command.policyFence.bindingVersion,
|
||||
},
|
||||
}),
|
||||
mutationObservedAtMs,
|
||||
],
|
||||
);
|
||||
result = cancellationResult('accepted', command, updated.rows[0]!);
|
||||
}
|
||||
|
||||
const runVersion = integer(current, 'runVersion');
|
||||
const eventSequence = integer(current, 'eventSequence');
|
||||
if (runVersion >= 2_147_483_647 || eventSequence >= 2_147_483_647) {
|
||||
throw new TypeError('PostgreSQL Run cancellation counter overflowed');
|
||||
if (confirmedAudit) {
|
||||
await recordAllowedAudit(
|
||||
client,
|
||||
command,
|
||||
confirmedAudit,
|
||||
observedAtMs!,
|
||||
);
|
||||
}
|
||||
const observedAtMs = await databaseNow(client);
|
||||
const updated = await client.query<Row>(`
|
||||
UPDATE "ql3"."runs"
|
||||
SET cancel_requested_at_ms = $2, cancel_reason = 'user',
|
||||
version = $3, event_sequence = $4
|
||||
WHERE id = $1 AND version = $5 AND cancel_requested_at_ms IS NULL
|
||||
RETURNING project_id AS "projectId", status AS "runStatus",
|
||||
version AS "runVersion", event_sequence AS "eventSequence",
|
||||
cancel_requested_at_ms AS "cancelRequestedAtMs",
|
||||
cancel_reason AS "cancelReason"
|
||||
`, [
|
||||
command.runId,
|
||||
observedAtMs,
|
||||
runVersion + 1,
|
||||
eventSequence + 1,
|
||||
runVersion,
|
||||
]);
|
||||
if (updated.rows.length !== 1) {
|
||||
throw new ClusterRunCancellationFenceRejectedError('state_mismatch');
|
||||
}
|
||||
await client.query(`
|
||||
INSERT INTO "ql3"."run_events" (
|
||||
id, run_id, sequence, type, dedupe_key, actor_type, actor_id,
|
||||
attempt_id, step_run_id, payload, created_at_ms
|
||||
) VALUES ($1, $2, $3, 'run.cancel_requested', $4, $5, $6,
|
||||
NULL, NULL, $7::jsonb, $8)
|
||||
`, [
|
||||
command.eventId,
|
||||
command.runId,
|
||||
eventSequence + 1,
|
||||
`user-cancel:${command.mutationId}`,
|
||||
command.subject.type,
|
||||
command.subject.id,
|
||||
JSON.stringify({
|
||||
reason: 'user',
|
||||
mutation_id: command.mutationId,
|
||||
policy_fence: {
|
||||
project_version: command.policyFence.projectVersion,
|
||||
binding_version: command.policyFence.bindingVersion,
|
||||
},
|
||||
}),
|
||||
observedAtMs,
|
||||
]);
|
||||
return cancellationResult('accepted', command, updated.rows[0]!);
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -15,14 +15,15 @@ export interface PostgresSchemaContractFunction {
|
||||
export interface PostgresSchemaContract {
|
||||
readonly schema: 'ql3';
|
||||
readonly contractName: 'control-core';
|
||||
readonly contractVersion: 55;
|
||||
readonly migrationId: 'pg-0056-run-management-boundary';
|
||||
readonly contractVersion: 56;
|
||||
readonly migrationId: 'pg-0057-run-management-stop-boundary';
|
||||
readonly minimumServerMajor: 16;
|
||||
readonly maximumServerMajor: 18;
|
||||
readonly capabilities: Readonly<{
|
||||
run_core: 1;
|
||||
run_attempt_log_retention: 1;
|
||||
run_management_boundary: 1;
|
||||
run_management_stop: 1;
|
||||
run_dispatch_lease: 1;
|
||||
run_retry_policy: 1;
|
||||
project_policy: 1;
|
||||
@@ -102,8 +103,8 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
Object.freeze({
|
||||
schema: 'ql3',
|
||||
contractName: 'control-core',
|
||||
contractVersion: 55,
|
||||
migrationId: 'pg-0056-run-management-boundary',
|
||||
contractVersion: 56,
|
||||
migrationId: 'pg-0057-run-management-stop-boundary',
|
||||
minimumServerMajor: 16,
|
||||
maximumServerMajor: 18,
|
||||
capabilities: Object.freeze({
|
||||
@@ -145,6 +146,7 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
run_core: 1,
|
||||
run_attempt_log_retention: 1,
|
||||
run_management_boundary: 1,
|
||||
run_management_stop: 1,
|
||||
run_dispatch_lease: 1,
|
||||
run_retry_policy: 1,
|
||||
security_audit: 1,
|
||||
|
||||
@@ -119,6 +119,11 @@ interface FunctionPrivilegeRow extends Record<string, unknown> {
|
||||
isOwner: unknown;
|
||||
}
|
||||
|
||||
interface ColumnPrivilegeRow extends Record<string, unknown> {
|
||||
columnName: unknown;
|
||||
updateAllowed: unknown;
|
||||
}
|
||||
|
||||
const REQUIRED_RUNTIME_PRIVILEGES = Object.freeze({
|
||||
schema_migrations: Object.freeze({
|
||||
select: true,
|
||||
@@ -1315,13 +1320,13 @@ const REQUIRED_AUTOMATION_MANAGER_PRIVILEGES: RequiredPrivileges =
|
||||
update: true,
|
||||
}
|
||||
: name === 'security_audit_events' ||
|
||||
name === 'task_definition_revisions' ||
|
||||
name === 'task_execution_revisions' ||
|
||||
name === 'trigger_revisions'
|
||||
name === 'task_definition_revisions' ||
|
||||
name === 'task_execution_revisions' ||
|
||||
name === 'trigger_revisions'
|
||||
? { ...NO_TABLE_PRIVILEGES, select: true, insert: true }
|
||||
: name === 'task_definitions' ||
|
||||
name === 'triggers' ||
|
||||
name === 'trigger_schedules'
|
||||
name === 'triggers' ||
|
||||
name === 'trigger_schedules'
|
||||
? {
|
||||
...NO_TABLE_PRIVILEGES,
|
||||
select: true,
|
||||
@@ -1376,9 +1381,9 @@ const REQUIRED_RUN_MANAGER_PRIVILEGES: RequiredPrivileges = Object.freeze(
|
||||
name === 'task_execution_revisions'
|
||||
? { ...NO_TABLE_PRIVILEGES, select: true }
|
||||
: name === 'runs' ||
|
||||
name === 'run_attempts' ||
|
||||
name === 'run_events' ||
|
||||
name === 'security_audit_events'
|
||||
name === 'run_attempts' ||
|
||||
name === 'run_events' ||
|
||||
name === 'security_audit_events'
|
||||
? { ...NO_TABLE_PRIVILEGES, select: true, insert: true }
|
||||
: name === 'plugin_package_identity_keyset_ledger'
|
||||
? {
|
||||
@@ -1415,7 +1420,7 @@ const REQUIRED_WORKER_CREDENTIAL_MANAGER_PRIVILEGES: RequiredPrivileges =
|
||||
update: true,
|
||||
}
|
||||
: name === 'worker_credential_management_quota_buckets' ||
|
||||
name === 'plugin_package_identity_keyset_ledger'
|
||||
name === 'plugin_package_identity_keyset_ledger'
|
||||
? {
|
||||
...NO_TABLE_PRIVILEGES,
|
||||
select: true,
|
||||
@@ -2042,6 +2047,56 @@ ORDER BY requested.function_name
|
||||
}
|
||||
}
|
||||
|
||||
async function assertRunManagerColumnPrivileges(
|
||||
queryable: PostgresMigrationQueryable,
|
||||
contract: PostgresSchemaContract,
|
||||
): Promise<void> {
|
||||
const run = contract.tables.find(({ name }) => name === 'runs');
|
||||
if (!run) {
|
||||
throw new PostgresSchemaReadinessError('run_manager_role_invalid', [
|
||||
'missing-runs-contract',
|
||||
]);
|
||||
}
|
||||
const result = await queryable.query<ColumnPrivilegeRow>(
|
||||
`
|
||||
SELECT
|
||||
requested.column_name AS "columnName",
|
||||
has_column_privilege(
|
||||
current_user,
|
||||
format('%I.%I', $1::text, 'runs'),
|
||||
requested.column_name,
|
||||
'UPDATE'
|
||||
) AS "updateAllowed"
|
||||
FROM unnest($2::text[]) AS requested(column_name)
|
||||
ORDER BY requested.column_name
|
||||
`.trim(),
|
||||
[contract.schema, run.columns],
|
||||
);
|
||||
const allowed = new Set([
|
||||
'cancel_requested_at_ms',
|
||||
'cancel_reason',
|
||||
'version',
|
||||
'event_sequence',
|
||||
]);
|
||||
const actual = new Map(result.rows.map((row) => [row.columnName, row]));
|
||||
const findings: string[] = [];
|
||||
for (const columnName of run.columns) {
|
||||
const row = actual.get(columnName);
|
||||
if (!row || row.updateAllowed !== allowed.has(columnName)) {
|
||||
findings.push(`column-update-privilege:runs.${columnName}`);
|
||||
}
|
||||
}
|
||||
if (actual.size !== run.columns.length) {
|
||||
findings.push('column-privilege-row-count:runs');
|
||||
}
|
||||
if (findings.length > 0) {
|
||||
throw new PostgresSchemaReadinessError(
|
||||
'run_manager_role_invalid',
|
||||
sorted(findings),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function assertPostgresSchemaReady(
|
||||
queryable: PostgresMigrationQueryable,
|
||||
contract: PostgresSchemaContract = postgresqlControlSchemaContract,
|
||||
@@ -2177,6 +2232,7 @@ export async function assertPostgresRunManagerSchemaReady(
|
||||
REQUIRED_RUN_MANAGER_FUNCTION_PRIVILEGES,
|
||||
'run_manager_role_invalid',
|
||||
);
|
||||
await assertRunManagerColumnPrivileges(queryable, contract);
|
||||
return Object.freeze({
|
||||
ready: true,
|
||||
...server,
|
||||
|
||||
@@ -41,25 +41,38 @@ function fixture(options = {}) {
|
||||
const normalized = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ sql: normalized, params });
|
||||
if (
|
||||
normalized.startsWith('BEGIN') || normalized === 'COMMIT' ||
|
||||
normalized === 'ROLLBACK' || normalized.startsWith('SELECT set_config')
|
||||
) return { rows: [], rowCount: 0 };
|
||||
normalized.startsWith('BEGIN') ||
|
||||
normalized === 'COMMIT' ||
|
||||
normalized === 'ROLLBACK' ||
|
||||
normalized.startsWith('SELECT set_config')
|
||||
)
|
||||
return { rows: [], rowCount: 0 };
|
||||
if (normalized.includes('lock_run_management_policy_fence')) {
|
||||
return {
|
||||
rows: [{ matches: options.policyMatches ?? true }],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."projects"')) {
|
||||
return {
|
||||
rows: options.projectRows ?? [{
|
||||
projectStatus: 'active',
|
||||
projectVersion: 2,
|
||||
}],
|
||||
rows: options.projectRows ?? [
|
||||
{
|
||||
projectStatus: 'active',
|
||||
projectVersion: 2,
|
||||
},
|
||||
],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."project_role_bindings"')) {
|
||||
return {
|
||||
rows: options.bindingRows ?? [{
|
||||
bindingVersion: 3,
|
||||
bindingState: 'active',
|
||||
bindingRole: 'operator',
|
||||
}],
|
||||
rows: options.bindingRows ?? [
|
||||
{
|
||||
bindingVersion: 3,
|
||||
bindingState: 'active',
|
||||
bindingRole: 'operator',
|
||||
},
|
||||
],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
@@ -73,9 +86,7 @@ function fixture(options = {}) {
|
||||
};
|
||||
}
|
||||
if (
|
||||
normalized.includes(
|
||||
'FROM "ql3"."plugin_package_workflow_admissions"',
|
||||
)
|
||||
normalized.includes('FROM "ql3"."plugin_package_workflow_admissions"')
|
||||
) {
|
||||
const rows = options.workflowAdmissionRows ?? [
|
||||
{
|
||||
@@ -91,25 +102,40 @@ function fixture(options = {}) {
|
||||
}
|
||||
if (normalized.startsWith('UPDATE "ql3"."runs"')) {
|
||||
return {
|
||||
rows: options.updatedRows ?? [run({
|
||||
runVersion: 5,
|
||||
eventSequence: 7,
|
||||
cancelRequestedAtMs: options.nowMs ?? 1_000,
|
||||
cancelReason: 'user',
|
||||
})],
|
||||
rows: options.updatedRows ?? [
|
||||
run({
|
||||
runVersion: 5,
|
||||
eventSequence: 7,
|
||||
cancelRequestedAtMs: options.nowMs ?? 1_000,
|
||||
cancelReason: 'user',
|
||||
}),
|
||||
],
|
||||
rowCount: options.updatedRows?.length ?? 1,
|
||||
};
|
||||
}
|
||||
if (normalized.startsWith('INSERT INTO "ql3"."run_events"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (normalized.startsWith('INSERT INTO "ql3"."security_audit_events"')) {
|
||||
return {
|
||||
rows: options.auditInserted === false ? [] : [{ eventId: params[0] }],
|
||||
rowCount: options.auditInserted === false ? 0 : 1,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."security_audit_events"')) {
|
||||
return { rows: options.auditReplayRows ?? [], rowCount: 0 };
|
||||
}
|
||||
throw new Error(`Unexpected SQL: ${normalized}`);
|
||||
},
|
||||
release() { calls.push({ sql: 'RELEASE', params: [] }); },
|
||||
release() {
|
||||
calls.push({ sql: 'RELEASE', params: [] });
|
||||
},
|
||||
};
|
||||
return {
|
||||
repository: new PostgresClusterRunCancellationRepository({
|
||||
async connect() { return client; },
|
||||
async connect() {
|
||||
return client;
|
||||
},
|
||||
}),
|
||||
calls,
|
||||
};
|
||||
@@ -127,23 +153,26 @@ test('revalidates policy authority and commits one database-timed intent', async
|
||||
cancelRequestedAtMs: 1_000,
|
||||
cancelReason: 'user',
|
||||
});
|
||||
const projectIndex = calls.findIndex(({ sql }) =>
|
||||
sql.includes('FROM "ql3"."projects"'));
|
||||
const bindingIndex = calls.findIndex(({ sql }) =>
|
||||
sql.includes('FROM "ql3"."project_role_bindings"'));
|
||||
const policyIndex = calls.findIndex(({ sql }) =>
|
||||
sql.includes('lock_run_management_policy_fence'),
|
||||
);
|
||||
const runIndex = calls.findIndex(({ sql }) =>
|
||||
sql.includes('FROM "ql3"."runs"'));
|
||||
assert.ok(projectIndex < bindingIndex && bindingIndex < runIndex);
|
||||
const update = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"'));
|
||||
sql.includes('FROM "ql3"."runs"'),
|
||||
);
|
||||
assert.ok(policyIndex >= 0 && policyIndex < runIndex);
|
||||
const update = calls.find(({ sql }) => sql.startsWith('UPDATE "ql3"."runs"'));
|
||||
assert.deepEqual(update.params, ['run-1', 1_000, 5, 7, 4]);
|
||||
const event = calls.find(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"'));
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"'),
|
||||
);
|
||||
assert.equal(event.params[0], command().eventId);
|
||||
assert.equal(event.params[3], 'user-cancel:mutation-1');
|
||||
assert.equal(event.params[4], 'user');
|
||||
assert.equal(JSON.parse(event.params[6]).reason, 'user');
|
||||
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), true);
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql === 'COMMIT'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('returns existing intent and terminal state without adding an event', async () => {
|
||||
@@ -154,8 +183,12 @@ test('returns existing intent and terminal state without adding an event', async
|
||||
(await existing.repository.requestUserCancellation(command())).status,
|
||||
'already_requested',
|
||||
);
|
||||
assert.equal(existing.calls.some(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"')), false);
|
||||
assert.equal(
|
||||
existing.calls.some(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"'),
|
||||
),
|
||||
false,
|
||||
);
|
||||
|
||||
const terminal = fixture({
|
||||
runRows: [run({ runStatus: 'succeeded', runVersion: 5 })],
|
||||
@@ -164,20 +197,24 @@ test('returns existing intent and terminal state without adding an event', async
|
||||
(await terminal.repository.requestUserCancellation(command())).status,
|
||||
'already_terminal',
|
||||
);
|
||||
assert.equal(terminal.calls.some(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"')), false);
|
||||
assert.equal(
|
||||
terminal.calls.some(({ sql }) => sql.startsWith('UPDATE "ql3"."runs"')),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts cancellation for a lost Run that still owns retry authority', async () => {
|
||||
const { repository } = fixture({
|
||||
runRows: [run({ runStatus: 'lost' })],
|
||||
updatedRows: [run({
|
||||
runStatus: 'lost',
|
||||
runVersion: 5,
|
||||
eventSequence: 7,
|
||||
cancelRequestedAtMs: 1_000,
|
||||
cancelReason: 'user',
|
||||
})],
|
||||
updatedRows: [
|
||||
run({
|
||||
runStatus: 'lost',
|
||||
runVersion: 5,
|
||||
eventSequence: 7,
|
||||
cancelRequestedAtMs: 1_000,
|
||||
cancelReason: 'user',
|
||||
}),
|
||||
],
|
||||
});
|
||||
assert.equal(
|
||||
(await repository.requestUserCancellation(command())).status,
|
||||
@@ -187,11 +224,7 @@ test('accepts cancellation for a lost Run that still owns retry authority', asyn
|
||||
|
||||
test('rejects a revoked policy fence before locking the Run', async () => {
|
||||
const { repository, calls } = fixture({
|
||||
bindingRows: [{
|
||||
bindingVersion: 4,
|
||||
bindingState: 'revoked',
|
||||
bindingRole: null,
|
||||
}],
|
||||
policyMatches: false,
|
||||
});
|
||||
await assert.rejects(
|
||||
repository.requestUserCancellation(command()),
|
||||
@@ -199,8 +232,78 @@ test('rejects a revoked policy fence before locking the Run', async () => {
|
||||
error instanceof ClusterRunCancellationFenceRejectedError &&
|
||||
error.reason === 'authorization_changed',
|
||||
);
|
||||
assert.equal(calls.some(({ sql }) => sql.includes('FROM "ql3"."runs"')), false);
|
||||
assert.equal(calls.some(({ sql }) => sql === 'ROLLBACK'), true);
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql.includes('FROM "ql3"."runs"')),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql === 'ROLLBACK'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('atomically records strong management audit and exact audit replay', async () => {
|
||||
const { repository, calls } = fixture();
|
||||
const { subject: _subject, ...baseCommand } = command();
|
||||
const managed = {
|
||||
...baseCommand,
|
||||
mutationId: '019f0000-0000-4000-8000-000000000001',
|
||||
requestId: 'request-stop-1',
|
||||
auditEventId: '019f0000-0000-4000-8000-000000000002',
|
||||
principal: {
|
||||
subject: command().subject,
|
||||
authenticationId: 'oidc:run-management-1',
|
||||
authenticatedAtMs: 900,
|
||||
expiresAtMs: 2_000,
|
||||
assurance: 'hardware',
|
||||
},
|
||||
};
|
||||
const result = await repository.requestUserCancellationAudited(managed);
|
||||
assert.equal(result.status, 'accepted');
|
||||
const audit = calls.find(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."security_audit_events"'),
|
||||
);
|
||||
assert.equal(audit.params[0], managed.auditEventId);
|
||||
assert.equal(audit.params[1], managed.requestId);
|
||||
assert.equal(audit.params[5], managed.principal.authenticationId);
|
||||
assert.ok(
|
||||
calls.findIndex(({ sql }) => sql.startsWith('UPDATE "ql3"."runs"')) <
|
||||
calls.findIndex(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."security_audit_events"'),
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
calls.findIndex(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."security_audit_events"'),
|
||||
) < calls.findIndex(({ sql }) => sql === 'COMMIT'),
|
||||
);
|
||||
|
||||
const replay = fixture({
|
||||
runRows: [run({ cancelRequestedAtMs: 1_000, cancelReason: 'user' })],
|
||||
auditInserted: false,
|
||||
auditReplayRows: [
|
||||
{
|
||||
requestId: managed.requestId,
|
||||
operationId: 'run.stop',
|
||||
projectId: managed.projectId,
|
||||
subjectType: 'user',
|
||||
subjectId: 'user-1',
|
||||
authenticationId: managed.principal.authenticationId,
|
||||
outcome: 'allowed',
|
||||
reasons: ['role_grant', 'strong_authentication'],
|
||||
projectVersion: 2,
|
||||
bindingVersion: 3,
|
||||
},
|
||||
],
|
||||
});
|
||||
assert.equal(
|
||||
(await replay.repository.requestUserCancellationAudited(managed)).status,
|
||||
'already_requested',
|
||||
);
|
||||
assert.equal(
|
||||
replay.calls.some(({ sql }) => sql.startsWith('UPDATE "ql3"."runs"')),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('masks cross-Project and missing Runs', async () => {
|
||||
@@ -255,9 +358,7 @@ test('binds Workflow cancellation to the immutable admission target', async () =
|
||||
ClusterRunCancellationNotFoundError,
|
||||
);
|
||||
assert.equal(
|
||||
rejected.calls.some(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."runs"'),
|
||||
),
|
||||
rejected.calls.some(({ sql }) => sql.startsWith('UPDATE "ql3"."runs"')),
|
||||
false,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const { postgresqlControlSchemaContract } = require('../dist/schema/schemaContract');
|
||||
const {
|
||||
postgresqlControlSchemaContract,
|
||||
} = require('../dist/schema/schemaContract');
|
||||
const {
|
||||
postgresqlMainMigrationManifest,
|
||||
} = require('../dist/migration/migrationManifest');
|
||||
@@ -105,6 +107,7 @@ test('defines the immutable PostgreSQL capability and Run core stream', async ()
|
||||
'pg-0054-approval-management-boundary',
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
'pg-0057-run-management-stop-boundary',
|
||||
],
|
||||
);
|
||||
for (const migration of postgresqlMainMigrationStream.migrations) {
|
||||
@@ -520,6 +523,11 @@ test('freezes every published PostgreSQL migration checksum', () => {
|
||||
checksum:
|
||||
'7aa2b2ade67cdfa6839d4af02209906646a68adfd6c12c4dddeb854021da72b8',
|
||||
},
|
||||
{
|
||||
id: 'pg-0057-run-management-stop-boundary',
|
||||
checksum:
|
||||
'ab2d0eee3d85a937e1e87243b1fd1e75181529122b64026303488404162e4ba7',
|
||||
},
|
||||
];
|
||||
assert.deepEqual(
|
||||
postgresqlMainMigrationStream.migrations.map(({ id, checksum }) => ({
|
||||
@@ -1588,7 +1596,10 @@ test('advances capability v45 with generation-bound Workflow Task attempts', asy
|
||||
/CREATE FUNCTION "ql3"\."plugin_package_workflow_task_attempt_snapshot"/,
|
||||
);
|
||||
assert.match(sql, /SECURITY DEFINER/);
|
||||
assert.match(sql, /FOR KEY SHARE OF workflow, source, reconciliation, item, execution/);
|
||||
assert.match(
|
||||
sql,
|
||||
/FOR KEY SHARE OF workflow, source, reconciliation, item, execution/,
|
||||
);
|
||||
assert.match(
|
||||
sql,
|
||||
/GRANT SELECT, INSERT[\s\S]*plugin_package_workflow_task_attempt_admissions[\s\S]*TO ql3_runtime/,
|
||||
@@ -1611,16 +1622,11 @@ test('advances capability v45 with generation-bound Workflow Task attempts', asy
|
||||
sql,
|
||||
/migration_id\s*=\s*'pg-0045-plugin-package-workflow-admissions'/,
|
||||
);
|
||||
assert.match(
|
||||
sql,
|
||||
/"plugin_package_workflow_task_attempt_admission":1/,
|
||||
);
|
||||
assert.match(sql, /"plugin_package_workflow_task_attempt_admission":1/);
|
||||
});
|
||||
|
||||
test('advances capability v46 with split Worker credential management authorities', async () => {
|
||||
const migration = migrationById(
|
||||
'pg-0047-worker-credential-management-plans',
|
||||
);
|
||||
const migration = migrationById('pg-0047-worker-credential-management-plans');
|
||||
const statements = [];
|
||||
await migration.up({
|
||||
async query(statement) {
|
||||
@@ -1629,10 +1635,7 @@ test('advances capability v46 with split Worker credential management authoritie
|
||||
},
|
||||
});
|
||||
const sql = statements.join('\n');
|
||||
assert.match(
|
||||
sql,
|
||||
/CREATE TABLE "ql3"\."worker_credential_management_plans"/,
|
||||
);
|
||||
assert.match(sql, /CREATE TABLE "ql3"\."worker_credential_management_plans"/);
|
||||
assert.match(sql, /'ql3_worker_credential_manager'/);
|
||||
assert.match(sql, /'ql3_worker_credential_executor'/);
|
||||
assert.match(
|
||||
@@ -1676,10 +1679,7 @@ test('advances capability v47 without invalidating preapproved Worker credential
|
||||
},
|
||||
});
|
||||
const sql = statements.join('\n');
|
||||
assert.match(
|
||||
sql,
|
||||
/DROP CONSTRAINT ql3_worker_credentials_lifetime_check/,
|
||||
);
|
||||
assert.match(sql, /DROP CONSTRAINT ql3_worker_credentials_lifetime_check/);
|
||||
assert.match(
|
||||
sql,
|
||||
/expires_at_ms > GREATEST\(created_at_ms, not_before_at_ms\)/,
|
||||
@@ -1747,7 +1747,10 @@ test('advances capability v49 with durable Worker credential management boundari
|
||||
},
|
||||
});
|
||||
const sql = statements.join('\n');
|
||||
assert.match(sql, /CREATE TABLE "ql3"\."worker_credential_management_quota_buckets"/);
|
||||
assert.match(
|
||||
sql,
|
||||
/CREATE TABLE "ql3"\."worker_credential_management_quota_buckets"/,
|
||||
);
|
||||
assert.match(sql, /TO ql3_worker_credential_manager/);
|
||||
assert.doesNotMatch(
|
||||
sql,
|
||||
@@ -1820,10 +1823,7 @@ test('advances capability v51 with a restart-safe automation identity keyset led
|
||||
assert.match(sql, /contract_version = 51/);
|
||||
assert.match(sql, /"automation_management_identity_keyset_ledger":1/);
|
||||
assert.match(sql, /contract_version = 50/);
|
||||
assert.match(
|
||||
sql,
|
||||
/migration_id = 'pg-0051-automation-management-boundary'/,
|
||||
);
|
||||
assert.match(sql, /migration_id = 'pg-0051-automation-management-boundary'/);
|
||||
});
|
||||
|
||||
test('advances capability v52 with a bounded Workflow Run history index', async () => {
|
||||
@@ -1905,10 +1905,7 @@ test('advances capability v54 with durable Cluster log retention authority', asy
|
||||
assert.match(sql, /contract_version = 54/);
|
||||
assert.match(sql, /"run_attempt_log_retention":1/);
|
||||
assert.match(sql, /contract_version = 53/);
|
||||
assert.match(
|
||||
sql,
|
||||
/migration_id = 'pg-0054-approval-management-boundary'/,
|
||||
);
|
||||
assert.match(sql, /migration_id = 'pg-0054-approval-management-boundary'/);
|
||||
});
|
||||
|
||||
test('advances capability v55 with isolated strong Run management authority', async () => {
|
||||
@@ -1932,8 +1929,26 @@ test('advances capability v55 with isolated strong Run management authority', as
|
||||
assert.match(sql, /contract_version = 55/);
|
||||
assert.match(sql, /"run_management_boundary":1/);
|
||||
assert.match(sql, /contract_version = 54/);
|
||||
assert.match(sql, /migration_id = 'pg-0055-run-attempt-log-retention'/);
|
||||
});
|
||||
|
||||
test('advances capability v56 with column-scoped Run stop authority', async () => {
|
||||
const migration = migrationById('pg-0057-run-management-stop-boundary');
|
||||
const statements = [];
|
||||
await migration.up({
|
||||
async query(statement) {
|
||||
statements.push(statement);
|
||||
return { rows: [] };
|
||||
},
|
||||
});
|
||||
const sql = statements.join('\n');
|
||||
assert.match(
|
||||
sql,
|
||||
/migration_id = 'pg-0055-run-attempt-log-retention'/,
|
||||
/GRANT UPDATE \(cancel_requested_at_ms, cancel_reason, version, event_sequence\) ON "ql3"\."runs" TO ql3_run_manager/,
|
||||
);
|
||||
assert.doesNotMatch(sql, /GRANT UPDATE ON "ql3"\."runs" TO ql3_run_manager/);
|
||||
assert.match(sql, /contract_version = 56/);
|
||||
assert.match(sql, /"run_management_stop":1/);
|
||||
assert.match(sql, /contract_version = 55/);
|
||||
assert.match(sql, /migration_id = 'pg-0056-run-management-boundary'/);
|
||||
});
|
||||
|
||||
@@ -13,7 +13,9 @@ const {
|
||||
assertPostgresWorkerCredentialManagerSchemaReady,
|
||||
assertPostgresWorkerIngressSchemaReady,
|
||||
} = require('../dist/schema/schemaReadiness');
|
||||
const { postgresqlControlSchemaContract } = require('../dist/schema/schemaContract');
|
||||
const {
|
||||
postgresqlControlSchemaContract,
|
||||
} = require('../dist/schema/schemaContract');
|
||||
const { postgresqlMainMigrationStream } = require('../dist/migrations');
|
||||
|
||||
function validHistory() {
|
||||
@@ -86,12 +88,7 @@ function validPrivileges() {
|
||||
plugin_package_automation_publication_heads: [true, false, false, false],
|
||||
plugin_package_workflow_admissions: [true, true, false, false],
|
||||
plugin_package_workflow_admission_steps: [true, true, false, false],
|
||||
plugin_package_workflow_task_attempt_admissions: [
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
],
|
||||
plugin_package_workflow_task_attempt_admissions: [true, true, false, false],
|
||||
plugin_package_publisher_provenance: [false, false, false, false],
|
||||
plugin_package_publisher_revocation_receipts: [false, false, false, false],
|
||||
plugin_package_publisher_revocation_impacts: [false, false, false, false],
|
||||
@@ -523,11 +520,11 @@ function workerCredentialPrivileges(kind) {
|
||||
]
|
||||
: []),
|
||||
...(manager
|
||||
? []
|
||||
: [
|
||||
'approved_action_dispatches',
|
||||
'approved_action_executions',
|
||||
'worker_credentials',
|
||||
? []
|
||||
: [
|
||||
'approved_action_dispatches',
|
||||
'approved_action_executions',
|
||||
'worker_credentials',
|
||||
'worker_credential_mutations',
|
||||
'worker_credential_deliveries',
|
||||
'worker_credential_stage_discards',
|
||||
@@ -714,6 +711,26 @@ function queryable(overrides = {}) {
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('has_column_privilege')) {
|
||||
assert.match(text, /format\('%I\.%I', \$1::text, 'runs'\)/);
|
||||
const columns = contract.tables.find(
|
||||
({ name }) => name === 'runs',
|
||||
).columns;
|
||||
const allowed = new Set([
|
||||
'cancel_requested_at_ms',
|
||||
'cancel_reason',
|
||||
'version',
|
||||
'event_sequence',
|
||||
]);
|
||||
return {
|
||||
rows:
|
||||
overrides.runManagerColumnPrivileges ??
|
||||
columns.map((columnName) => ({
|
||||
columnName,
|
||||
updateAllowed: allowed.has(columnName),
|
||||
})),
|
||||
};
|
||||
}
|
||||
if (text.includes('has_table_privilege')) {
|
||||
return { rows: overrides.privileges ?? validPrivileges() };
|
||||
}
|
||||
@@ -731,7 +748,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
|
||||
serverMajor: 16,
|
||||
currentUser: 'ql3_runtime',
|
||||
contractName: 'control-core',
|
||||
contractVersion: 55,
|
||||
contractVersion: 56,
|
||||
migrationIds: [
|
||||
'pg-0001-schema-capability',
|
||||
'pg-0002-run-core',
|
||||
@@ -789,6 +806,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
|
||||
'pg-0054-approval-management-boundary',
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
'pg-0057-run-management-stop-boundary',
|
||||
],
|
||||
});
|
||||
});
|
||||
@@ -819,10 +837,10 @@ test('accepts the exact schema and isolated least-privilege admin role', async (
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_admin');
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(report.contractVersion, 56);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0056-run-management-boundary',
|
||||
'pg-0057-run-management-stop-boundary',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -835,10 +853,10 @@ test('accepts the isolated least-privilege automation manager role', async () =>
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_automation_manager');
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(report.contractVersion, 56);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0056-run-management-boundary',
|
||||
'pg-0057-run-management-stop-boundary',
|
||||
);
|
||||
|
||||
const widened = automationManagerPrivileges();
|
||||
@@ -867,10 +885,10 @@ test('accepts the isolated least-privilege human Approval manager role', async (
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_approval_manager');
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(report.contractVersion, 56);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0056-run-management-boundary',
|
||||
'pg-0057-run-management-stop-boundary',
|
||||
);
|
||||
|
||||
const widened = approvalManagerPrivileges();
|
||||
@@ -901,8 +919,11 @@ test('accepts the isolated least-privilege Run manager role', async () => {
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_run_manager');
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(report.migrationIds.at(-1), 'pg-0056-run-management-boundary');
|
||||
assert.equal(report.contractVersion, 56);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0057-run-management-stop-boundary',
|
||||
);
|
||||
|
||||
const widened = runManagerPrivileges();
|
||||
widened.find(({ tableName }) => tableName === 'runs').updateAllowed = true;
|
||||
@@ -919,6 +940,33 @@ test('accepts the isolated least-privilege Run manager role', async () => {
|
||||
error.code === 'run_manager_role_invalid' &&
|
||||
error.facts.includes('table-privileges:runs'),
|
||||
);
|
||||
|
||||
const widenedColumns = postgresqlControlSchemaContract.tables
|
||||
.find(({ name }) => name === 'runs')
|
||||
.columns.map((columnName) => ({
|
||||
columnName,
|
||||
updateAllowed: [
|
||||
'cancel_requested_at_ms',
|
||||
'cancel_reason',
|
||||
'version',
|
||||
'event_sequence',
|
||||
'status',
|
||||
].includes(columnName),
|
||||
}));
|
||||
await assert.rejects(
|
||||
assertPostgresRunManagerSchemaReady(
|
||||
queryable({
|
||||
currentUser: 'ql3_run_manager',
|
||||
privileges: runManagerPrivileges(),
|
||||
functionMode: 'run-manager',
|
||||
runManagerColumnPrivileges: widenedColumns,
|
||||
}),
|
||||
),
|
||||
(error) =>
|
||||
error instanceof PostgresSchemaReadinessError &&
|
||||
error.code === 'run_manager_role_invalid' &&
|
||||
error.facts.includes('column-update-privilege:runs.status'),
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts isolated Package manager and executor roles', async () => {
|
||||
@@ -1006,10 +1054,10 @@ test('accepts the exact schema and isolated Worker ingress role', async () => {
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_worker_ingress');
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(report.contractVersion, 56);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0056-run-management-boundary',
|
||||
'pg-0057-run-management-stop-boundary',
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user