mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,21 @@
|
||||
import {
|
||||
POSTGRESQL_MAIN_MIGRATION_STREAM_ID,
|
||||
type PostgresMigrationContext,
|
||||
} from '../adapters/postgresMigrationStreamStore';
|
||||
import type { MigrationStreamDefinition } from '../core/migrationStream';
|
||||
import { pg0001SchemaCapabilityMigration } from './pg-0001-schema-capability';
|
||||
import { pg0002RunCoreMigration } from './pg-0002-run-core';
|
||||
import { pg0003RunRetryPolicyMigration } from './pg-0003-run-retry-policy';
|
||||
|
||||
export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMigrationContext> =
|
||||
Object.freeze({
|
||||
id: POSTGRESQL_MAIN_MIGRATION_STREAM_ID,
|
||||
dialect: 'postgresql',
|
||||
migrationIdScheme: 'postgres-prefixed',
|
||||
checksumScheme: 'sha256',
|
||||
migrations: Object.freeze([
|
||||
pg0001SchemaCapabilityMigration,
|
||||
pg0002RunCoreMigration,
|
||||
pg0003RunRetryPolicyMigration,
|
||||
]),
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
import { definePostgresSqlMigration } from './sqlMigration';
|
||||
|
||||
export const POSTGRESQL_SCHEMA_CAPABILITY_TABLE = 'schema_capabilities';
|
||||
|
||||
export const pg0001SchemaCapabilityMigration = definePostgresSqlMigration({
|
||||
id: 'pg-0001-schema-capability',
|
||||
statements: [
|
||||
`
|
||||
CREATE TABLE "ql3"."${POSTGRESQL_SCHEMA_CAPABILITY_TABLE}" (
|
||||
contract_name varchar(64) PRIMARY KEY,
|
||||
contract_version integer NOT NULL
|
||||
CONSTRAINT ql3_schema_capabilities_version_check
|
||||
CHECK (contract_version >= 0),
|
||||
migration_id varchar(128) NOT NULL,
|
||||
capabilities jsonb NOT NULL
|
||||
CONSTRAINT ql3_schema_capabilities_payload_check
|
||||
CHECK (jsonb_typeof(capabilities) = 'object'),
|
||||
updated_at_ms bigint NOT NULL
|
||||
CONSTRAINT ql3_schema_capabilities_updated_at_check
|
||||
CHECK (updated_at_ms >= 0),
|
||||
CONSTRAINT ql3_schema_capabilities_migration_fk
|
||||
FOREIGN KEY (migration_id)
|
||||
REFERENCES "ql3"."schema_migrations" (migration_id)
|
||||
DEFERRABLE INITIALLY DEFERRED
|
||||
)
|
||||
`.trim(),
|
||||
],
|
||||
});
|
||||
@@ -0,0 +1,224 @@
|
||||
import { definePostgresSqlMigration } from './sqlMigration';
|
||||
|
||||
export const POSTGRESQL_RUN_TABLE = 'runs';
|
||||
export const POSTGRESQL_RUN_ATTEMPT_TABLE = 'run_attempts';
|
||||
export const POSTGRESQL_RUN_EVENT_TABLE = 'run_events';
|
||||
|
||||
export const pg0002RunCoreMigration = definePostgresSqlMigration({
|
||||
id: 'pg-0002-run-core',
|
||||
statements: [
|
||||
`
|
||||
CREATE TABLE "ql3"."${POSTGRESQL_RUN_TABLE}" (
|
||||
id varchar(36) PRIMARY KEY,
|
||||
project_id varchar(128) NOT NULL,
|
||||
task_id varchar(255) NOT NULL,
|
||||
task_revision varchar(128) NOT NULL,
|
||||
task_name varchar(255),
|
||||
task_snapshot_ref varchar(512),
|
||||
parent_run_id varchar(36),
|
||||
retry_of_run_id varchar(36),
|
||||
trigger_id varchar(36),
|
||||
trigger_type varchar(64) NOT NULL,
|
||||
execution_origin varchar(64) NOT NULL,
|
||||
execution_owner varchar(16) NOT NULL
|
||||
CONSTRAINT ql3_runs_execution_owner_check
|
||||
CHECK (execution_owner = 'runtime'),
|
||||
triggered_by varchar(255),
|
||||
request_id varchar(128),
|
||||
scheduled_for_ms bigint
|
||||
CONSTRAINT ql3_runs_scheduled_for_check
|
||||
CHECK (scheduled_for_ms IS NULL OR scheduled_for_ms >= 0),
|
||||
status varchar(32) NOT NULL
|
||||
CONSTRAINT ql3_runs_status_check
|
||||
CHECK (status IN (
|
||||
'created', 'queued', 'dispatching', 'running', 'waiting_approval',
|
||||
'retry_wait', 'lost', 'succeeded', 'failed', 'cancelled', 'timed_out'
|
||||
)),
|
||||
version integer NOT NULL DEFAULT 0
|
||||
CONSTRAINT ql3_runs_version_check
|
||||
CHECK (version >= 0),
|
||||
event_sequence integer NOT NULL DEFAULT 0
|
||||
CONSTRAINT ql3_runs_event_sequence_check
|
||||
CHECK (event_sequence >= 0),
|
||||
priority integer NOT NULL DEFAULT 0,
|
||||
idempotency_key varchar(255),
|
||||
input_ref varchar(512),
|
||||
output_ref varchar(512),
|
||||
created_at_ms bigint NOT NULL
|
||||
CONSTRAINT ql3_runs_created_at_check
|
||||
CHECK (created_at_ms >= 0),
|
||||
queued_at_ms bigint
|
||||
CONSTRAINT ql3_runs_queued_at_check
|
||||
CHECK (queued_at_ms IS NULL OR queued_at_ms >= 0),
|
||||
started_at_ms bigint
|
||||
CONSTRAINT ql3_runs_started_at_check
|
||||
CHECK (started_at_ms IS NULL OR started_at_ms >= 0),
|
||||
finished_at_ms bigint
|
||||
CONSTRAINT ql3_runs_finished_at_check
|
||||
CHECK (finished_at_ms IS NULL OR finished_at_ms >= 0),
|
||||
cancel_requested_at_ms bigint
|
||||
CONSTRAINT ql3_runs_cancel_requested_at_check
|
||||
CHECK (cancel_requested_at_ms IS NULL OR cancel_requested_at_ms >= 0),
|
||||
cancel_reason varchar(16)
|
||||
CONSTRAINT ql3_runs_cancel_reason_check
|
||||
CHECK (cancel_reason IS NULL OR cancel_reason IN (
|
||||
'user', 'policy', 'shutdown', 'reconcile', 'timeout'
|
||||
)),
|
||||
error_code varchar(128),
|
||||
error_summary varchar(1024),
|
||||
CONSTRAINT ql3_runs_parent_fk
|
||||
FOREIGN KEY (parent_run_id) REFERENCES "ql3"."${POSTGRESQL_RUN_TABLE}" (id),
|
||||
CONSTRAINT ql3_runs_retry_of_fk
|
||||
FOREIGN KEY (retry_of_run_id) REFERENCES "ql3"."${POSTGRESQL_RUN_TABLE}" (id)
|
||||
)
|
||||
`.trim(),
|
||||
`
|
||||
CREATE UNIQUE INDEX ql3_runs_project_idempotency_uidx
|
||||
ON "ql3"."${POSTGRESQL_RUN_TABLE}" (project_id, idempotency_key)
|
||||
WHERE idempotency_key IS NOT NULL
|
||||
`.trim(),
|
||||
`
|
||||
CREATE INDEX ql3_runs_project_created_idx
|
||||
ON "ql3"."${POSTGRESQL_RUN_TABLE}" (project_id, created_at_ms, id)
|
||||
`.trim(),
|
||||
`
|
||||
CREATE INDEX ql3_runs_task_created_idx
|
||||
ON "ql3"."${POSTGRESQL_RUN_TABLE}" (task_id, created_at_ms, id)
|
||||
`.trim(),
|
||||
`
|
||||
CREATE INDEX ql3_runs_dispatch_candidates_idx
|
||||
ON "ql3"."${POSTGRESQL_RUN_TABLE}" (priority DESC, queued_at_ms, id)
|
||||
WHERE execution_owner = 'runtime'
|
||||
AND status IN ('queued', 'dispatching')
|
||||
AND cancel_requested_at_ms IS NULL
|
||||
AND queued_at_ms IS NOT NULL
|
||||
`.trim(),
|
||||
`
|
||||
CREATE TABLE "ql3"."${POSTGRESQL_RUN_ATTEMPT_TABLE}" (
|
||||
id varchar(36) PRIMARY KEY,
|
||||
run_id varchar(36) NOT NULL,
|
||||
step_run_id varchar(36),
|
||||
attempt integer NOT NULL
|
||||
CONSTRAINT ql3_run_attempts_attempt_check
|
||||
CHECK (attempt >= 1),
|
||||
status varchar(32) NOT NULL
|
||||
CONSTRAINT ql3_run_attempts_status_check
|
||||
CHECK (status IN (
|
||||
'claimed', 'starting', 'running', 'succeeded', 'failed',
|
||||
'cancelled', 'timed_out', 'lost'
|
||||
)),
|
||||
executor_type varchar(64) NOT NULL,
|
||||
worker_id varchar(128),
|
||||
executor_handle varchar(2048),
|
||||
pid integer
|
||||
CONSTRAINT ql3_run_attempts_pid_check
|
||||
CHECK (pid IS NULL OR pid >= 1),
|
||||
log_artifact_id varchar(36),
|
||||
lease_token varchar(128),
|
||||
lease_expires_at_ms bigint
|
||||
CONSTRAINT ql3_run_attempts_lease_expiry_check
|
||||
CHECK (lease_expires_at_ms IS NULL OR lease_expires_at_ms >= 0),
|
||||
deadline_at_ms bigint
|
||||
CONSTRAINT ql3_run_attempts_deadline_check
|
||||
CHECK (deadline_at_ms IS NULL OR deadline_at_ms >= 0),
|
||||
callback_token_hash varchar(128),
|
||||
callback_sequence integer NOT NULL DEFAULT 0
|
||||
CONSTRAINT ql3_run_attempts_callback_sequence_check
|
||||
CHECK (callback_sequence >= 0),
|
||||
created_at_ms bigint NOT NULL
|
||||
CONSTRAINT ql3_run_attempts_created_at_check
|
||||
CHECK (created_at_ms >= 0),
|
||||
started_at_ms bigint
|
||||
CONSTRAINT ql3_run_attempts_started_at_check
|
||||
CHECK (started_at_ms IS NULL OR started_at_ms >= 0),
|
||||
finished_at_ms bigint
|
||||
CONSTRAINT ql3_run_attempts_finished_at_check
|
||||
CHECK (finished_at_ms IS NULL OR finished_at_ms >= 0),
|
||||
exit_code integer,
|
||||
error_code varchar(128),
|
||||
error_summary varchar(1024),
|
||||
CONSTRAINT ql3_run_attempts_run_fk
|
||||
FOREIGN KEY (run_id) REFERENCES "ql3"."${POSTGRESQL_RUN_TABLE}" (id)
|
||||
)
|
||||
`.trim(),
|
||||
`
|
||||
CREATE UNIQUE INDEX ql3_run_attempts_run_attempt_uidx
|
||||
ON "ql3"."${POSTGRESQL_RUN_ATTEMPT_TABLE}" (run_id, attempt)
|
||||
`.trim(),
|
||||
`
|
||||
CREATE INDEX ql3_run_attempts_dispatch_candidates_idx
|
||||
ON "ql3"."${POSTGRESQL_RUN_ATTEMPT_TABLE}" (status, run_id, created_at_ms, id)
|
||||
`.trim(),
|
||||
`
|
||||
CREATE INDEX ql3_run_attempts_lease_idx
|
||||
ON "ql3"."${POSTGRESQL_RUN_ATTEMPT_TABLE}" (lease_expires_at_ms, id)
|
||||
WHERE lease_expires_at_ms IS NOT NULL
|
||||
`.trim(),
|
||||
`
|
||||
CREATE TABLE "ql3"."${POSTGRESQL_RUN_EVENT_TABLE}" (
|
||||
id varchar(36) PRIMARY KEY,
|
||||
run_id varchar(36) NOT NULL,
|
||||
sequence integer NOT NULL
|
||||
CONSTRAINT ql3_run_events_sequence_check
|
||||
CHECK (sequence >= 1),
|
||||
type varchar(128) NOT NULL,
|
||||
dedupe_key varchar(255),
|
||||
actor_type varchar(64) NOT NULL
|
||||
CONSTRAINT ql3_run_events_actor_type_check
|
||||
CHECK (actor_type IN (
|
||||
'user', 'api_app', 'trigger', 'agent', 'mcp_client', 'worker',
|
||||
'executor', 'system', 'legacy_shell', 'scheduler', 'reconciler',
|
||||
'compatibility'
|
||||
)),
|
||||
actor_id varchar(255),
|
||||
attempt_id varchar(36),
|
||||
step_run_id varchar(36),
|
||||
payload jsonb NOT NULL
|
||||
CONSTRAINT ql3_run_events_payload_check
|
||||
CHECK (jsonb_typeof(payload) = 'object'),
|
||||
created_at_ms bigint NOT NULL
|
||||
CONSTRAINT ql3_run_events_created_at_check
|
||||
CHECK (created_at_ms >= 0),
|
||||
CONSTRAINT ql3_run_events_run_fk
|
||||
FOREIGN KEY (run_id) REFERENCES "ql3"."${POSTGRESQL_RUN_TABLE}" (id),
|
||||
CONSTRAINT ql3_run_events_attempt_fk
|
||||
FOREIGN KEY (attempt_id) REFERENCES "ql3"."${POSTGRESQL_RUN_ATTEMPT_TABLE}" (id)
|
||||
)
|
||||
`.trim(),
|
||||
`
|
||||
CREATE UNIQUE INDEX ql3_run_events_run_sequence_uidx
|
||||
ON "ql3"."${POSTGRESQL_RUN_EVENT_TABLE}" (run_id, sequence)
|
||||
`.trim(),
|
||||
`
|
||||
CREATE UNIQUE INDEX ql3_run_events_run_dedupe_uidx
|
||||
ON "ql3"."${POSTGRESQL_RUN_EVENT_TABLE}" (run_id, dedupe_key)
|
||||
WHERE dedupe_key IS NOT NULL
|
||||
`.trim(),
|
||||
`
|
||||
CREATE INDEX ql3_run_events_run_created_idx
|
||||
ON "ql3"."${POSTGRESQL_RUN_EVENT_TABLE}" (run_id, created_at_ms, id)
|
||||
`.trim(),
|
||||
`
|
||||
INSERT INTO "ql3"."schema_capabilities" (
|
||||
contract_name,
|
||||
contract_version,
|
||||
migration_id,
|
||||
capabilities,
|
||||
updated_at_ms
|
||||
)
|
||||
VALUES (
|
||||
'control-core',
|
||||
1,
|
||||
'pg-0002-run-core',
|
||||
'{"run_core":1}'::jsonb,
|
||||
floor(extract(epoch FROM transaction_timestamp()) * 1000)::bigint
|
||||
)
|
||||
ON CONFLICT (contract_name) DO UPDATE
|
||||
SET contract_version = EXCLUDED.contract_version,
|
||||
migration_id = EXCLUDED.migration_id,
|
||||
capabilities = EXCLUDED.capabilities,
|
||||
updated_at_ms = EXCLUDED.updated_at_ms
|
||||
WHERE "schema_capabilities".contract_version < EXCLUDED.contract_version
|
||||
`.trim(),
|
||||
],
|
||||
});
|
||||
@@ -0,0 +1,78 @@
|
||||
import { definePostgresSqlMigration } from './sqlMigration';
|
||||
|
||||
export const POSTGRESQL_RUN_RETRY_POLICY_TABLE = 'run_retry_policies';
|
||||
|
||||
export const pg0003RunRetryPolicyMigration = definePostgresSqlMigration({
|
||||
id: 'pg-0003-run-retry-policy',
|
||||
statements: [
|
||||
`
|
||||
ALTER TABLE "ql3"."runs"
|
||||
ADD COLUMN legacy_cron_id integer
|
||||
CONSTRAINT ql3_runs_legacy_cron_id_check
|
||||
CHECK (legacy_cron_id IS NULL OR legacy_cron_id >= 1)
|
||||
`.trim(),
|
||||
`
|
||||
CREATE TABLE "ql3"."${POSTGRESQL_RUN_RETRY_POLICY_TABLE}" (
|
||||
run_id varchar(36) PRIMARY KEY,
|
||||
max_attempts integer NOT NULL
|
||||
CONSTRAINT ql3_run_retry_policies_max_attempts_check
|
||||
CHECK (max_attempts BETWEEN 1 AND 16),
|
||||
retry_on_lost boolean NOT NULL,
|
||||
safety varchar(16) NOT NULL
|
||||
CONSTRAINT ql3_run_retry_policies_safety_check
|
||||
CHECK (safety IN ('unknown', 'idempotent', 'deduplicated')),
|
||||
backoff_base_ms bigint NOT NULL
|
||||
CONSTRAINT ql3_run_retry_policies_backoff_base_check
|
||||
CHECK (backoff_base_ms BETWEEN 0 AND 86400000),
|
||||
backoff_max_ms bigint NOT NULL
|
||||
CONSTRAINT ql3_run_retry_policies_backoff_max_check
|
||||
CHECK (backoff_max_ms BETWEEN backoff_base_ms AND 86400000),
|
||||
next_attempt_at_ms bigint
|
||||
CONSTRAINT ql3_run_retry_policies_next_attempt_check
|
||||
CHECK (next_attempt_at_ms IS NULL OR next_attempt_at_ms >= 0),
|
||||
version integer NOT NULL DEFAULT 0
|
||||
CONSTRAINT ql3_run_retry_policies_version_check
|
||||
CHECK (version >= 0),
|
||||
created_at_ms bigint NOT NULL
|
||||
CONSTRAINT ql3_run_retry_policies_created_at_check
|
||||
CHECK (created_at_ms >= 0),
|
||||
updated_at_ms bigint NOT NULL
|
||||
CONSTRAINT ql3_run_retry_policies_updated_at_check
|
||||
CHECK (updated_at_ms >= created_at_ms),
|
||||
CONSTRAINT ql3_run_retry_policies_run_fk
|
||||
FOREIGN KEY (run_id) REFERENCES "ql3"."runs" (id) ON DELETE CASCADE
|
||||
)
|
||||
`.trim(),
|
||||
`
|
||||
CREATE INDEX ql3_run_retry_policies_due_idx
|
||||
ON "ql3"."${POSTGRESQL_RUN_RETRY_POLICY_TABLE}" (next_attempt_at_ms, run_id)
|
||||
WHERE next_attempt_at_ms IS NOT NULL
|
||||
`.trim(),
|
||||
`
|
||||
CREATE INDEX ql3_runs_lost_retry_idx
|
||||
ON "ql3"."runs" (execution_owner, status, id)
|
||||
`.trim(),
|
||||
`
|
||||
DO $ql3$
|
||||
BEGIN
|
||||
UPDATE "ql3"."schema_capabilities"
|
||||
SET contract_version = 2,
|
||||
migration_id = 'pg-0003-run-retry-policy',
|
||||
capabilities = '{"run_core":1,"run_retry_policy":1}'::jsonb,
|
||||
updated_at_ms = floor(
|
||||
extract(epoch FROM transaction_timestamp()) * 1000
|
||||
)::bigint
|
||||
WHERE contract_name = 'control-core'
|
||||
AND contract_version = 1
|
||||
AND migration_id = 'pg-0002-run-core'
|
||||
AND capabilities = '{"run_core":1}'::jsonb;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
RAISE EXCEPTION 'control-core capability is not at version 1'
|
||||
USING ERRCODE = 'check_violation';
|
||||
END IF;
|
||||
END
|
||||
$ql3$
|
||||
`.trim(),
|
||||
],
|
||||
});
|
||||
@@ -0,0 +1,206 @@
|
||||
export interface PostgresSchemaContractTable {
|
||||
readonly name: string;
|
||||
readonly columns: readonly string[];
|
||||
}
|
||||
|
||||
export interface PostgresSchemaContract {
|
||||
readonly schema: 'ql3';
|
||||
readonly contractName: 'control-core';
|
||||
readonly contractVersion: 2;
|
||||
readonly migrationId: 'pg-0003-run-retry-policy';
|
||||
readonly minimumServerMajor: 16;
|
||||
readonly maximumServerMajor: 18;
|
||||
readonly capabilities: Readonly<{
|
||||
run_core: 1;
|
||||
run_retry_policy: 1;
|
||||
}>;
|
||||
readonly tables: readonly PostgresSchemaContractTable[];
|
||||
readonly indexes: readonly string[];
|
||||
readonly checks: readonly string[];
|
||||
readonly foreignKeys: readonly string[];
|
||||
}
|
||||
|
||||
function table(
|
||||
name: string,
|
||||
columns: readonly string[],
|
||||
): PostgresSchemaContractTable {
|
||||
return Object.freeze({ name, columns: Object.freeze([...columns]) });
|
||||
}
|
||||
|
||||
export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
Object.freeze({
|
||||
schema: 'ql3',
|
||||
contractName: 'control-core',
|
||||
contractVersion: 2,
|
||||
migrationId: 'pg-0003-run-retry-policy',
|
||||
minimumServerMajor: 16,
|
||||
maximumServerMajor: 18,
|
||||
capabilities: Object.freeze({ run_core: 1, run_retry_policy: 1 }),
|
||||
tables: Object.freeze([
|
||||
table('schema_migrations', [
|
||||
'migration_id',
|
||||
'stream_id',
|
||||
'dialect',
|
||||
'checksum',
|
||||
'applied_at_ms',
|
||||
]),
|
||||
table('schema_capabilities', [
|
||||
'contract_name',
|
||||
'contract_version',
|
||||
'migration_id',
|
||||
'capabilities',
|
||||
'updated_at_ms',
|
||||
]),
|
||||
table('runs', [
|
||||
'id',
|
||||
'project_id',
|
||||
'task_id',
|
||||
'task_revision',
|
||||
'task_name',
|
||||
'task_snapshot_ref',
|
||||
'legacy_cron_id',
|
||||
'parent_run_id',
|
||||
'retry_of_run_id',
|
||||
'trigger_id',
|
||||
'trigger_type',
|
||||
'execution_origin',
|
||||
'execution_owner',
|
||||
'triggered_by',
|
||||
'request_id',
|
||||
'scheduled_for_ms',
|
||||
'status',
|
||||
'version',
|
||||
'event_sequence',
|
||||
'priority',
|
||||
'idempotency_key',
|
||||
'input_ref',
|
||||
'output_ref',
|
||||
'created_at_ms',
|
||||
'queued_at_ms',
|
||||
'started_at_ms',
|
||||
'finished_at_ms',
|
||||
'cancel_requested_at_ms',
|
||||
'cancel_reason',
|
||||
'error_code',
|
||||
'error_summary',
|
||||
]),
|
||||
table('run_attempts', [
|
||||
'id',
|
||||
'run_id',
|
||||
'step_run_id',
|
||||
'attempt',
|
||||
'status',
|
||||
'executor_type',
|
||||
'worker_id',
|
||||
'executor_handle',
|
||||
'pid',
|
||||
'log_artifact_id',
|
||||
'lease_token',
|
||||
'lease_expires_at_ms',
|
||||
'deadline_at_ms',
|
||||
'callback_token_hash',
|
||||
'callback_sequence',
|
||||
'created_at_ms',
|
||||
'started_at_ms',
|
||||
'finished_at_ms',
|
||||
'exit_code',
|
||||
'error_code',
|
||||
'error_summary',
|
||||
]),
|
||||
table('run_events', [
|
||||
'id',
|
||||
'run_id',
|
||||
'sequence',
|
||||
'type',
|
||||
'dedupe_key',
|
||||
'actor_type',
|
||||
'actor_id',
|
||||
'attempt_id',
|
||||
'step_run_id',
|
||||
'payload',
|
||||
'created_at_ms',
|
||||
]),
|
||||
table('run_retry_policies', [
|
||||
'run_id',
|
||||
'max_attempts',
|
||||
'retry_on_lost',
|
||||
'safety',
|
||||
'backoff_base_ms',
|
||||
'backoff_max_ms',
|
||||
'next_attempt_at_ms',
|
||||
'version',
|
||||
'created_at_ms',
|
||||
'updated_at_ms',
|
||||
]),
|
||||
]),
|
||||
indexes: Object.freeze([
|
||||
'schema_migrations_pkey',
|
||||
'schema_capabilities_pkey',
|
||||
'runs_pkey',
|
||||
'ql3_runs_project_idempotency_uidx',
|
||||
'ql3_runs_project_created_idx',
|
||||
'ql3_runs_task_created_idx',
|
||||
'ql3_runs_dispatch_candidates_idx',
|
||||
'run_attempts_pkey',
|
||||
'ql3_run_attempts_run_attempt_uidx',
|
||||
'ql3_run_attempts_dispatch_candidates_idx',
|
||||
'ql3_run_attempts_lease_idx',
|
||||
'run_events_pkey',
|
||||
'ql3_run_events_run_sequence_uidx',
|
||||
'ql3_run_events_run_dedupe_uidx',
|
||||
'ql3_run_events_run_created_idx',
|
||||
'run_retry_policies_pkey',
|
||||
'ql3_run_retry_policies_due_idx',
|
||||
'ql3_runs_lost_retry_idx',
|
||||
]),
|
||||
checks: Object.freeze([
|
||||
'ql3_schema_migrations_dialect_check',
|
||||
'ql3_schema_migrations_checksum_check',
|
||||
'ql3_schema_migrations_applied_at_check',
|
||||
'ql3_schema_capabilities_version_check',
|
||||
'ql3_schema_capabilities_payload_check',
|
||||
'ql3_schema_capabilities_updated_at_check',
|
||||
'ql3_runs_legacy_cron_id_check',
|
||||
'ql3_runs_execution_owner_check',
|
||||
'ql3_runs_scheduled_for_check',
|
||||
'ql3_runs_status_check',
|
||||
'ql3_runs_version_check',
|
||||
'ql3_runs_event_sequence_check',
|
||||
'ql3_runs_created_at_check',
|
||||
'ql3_runs_queued_at_check',
|
||||
'ql3_runs_started_at_check',
|
||||
'ql3_runs_finished_at_check',
|
||||
'ql3_runs_cancel_requested_at_check',
|
||||
'ql3_runs_cancel_reason_check',
|
||||
'ql3_run_attempts_attempt_check',
|
||||
'ql3_run_attempts_status_check',
|
||||
'ql3_run_attempts_pid_check',
|
||||
'ql3_run_attempts_lease_expiry_check',
|
||||
'ql3_run_attempts_deadline_check',
|
||||
'ql3_run_attempts_callback_sequence_check',
|
||||
'ql3_run_attempts_created_at_check',
|
||||
'ql3_run_attempts_started_at_check',
|
||||
'ql3_run_attempts_finished_at_check',
|
||||
'ql3_run_events_sequence_check',
|
||||
'ql3_run_events_actor_type_check',
|
||||
'ql3_run_events_payload_check',
|
||||
'ql3_run_events_created_at_check',
|
||||
'ql3_run_retry_policies_max_attempts_check',
|
||||
'ql3_run_retry_policies_safety_check',
|
||||
'ql3_run_retry_policies_backoff_base_check',
|
||||
'ql3_run_retry_policies_backoff_max_check',
|
||||
'ql3_run_retry_policies_next_attempt_check',
|
||||
'ql3_run_retry_policies_version_check',
|
||||
'ql3_run_retry_policies_created_at_check',
|
||||
'ql3_run_retry_policies_updated_at_check',
|
||||
]),
|
||||
foreignKeys: Object.freeze([
|
||||
'ql3_schema_capabilities_migration_fk',
|
||||
'ql3_runs_parent_fk',
|
||||
'ql3_runs_retry_of_fk',
|
||||
'ql3_run_attempts_run_fk',
|
||||
'ql3_run_events_run_fk',
|
||||
'ql3_run_events_attempt_fk',
|
||||
'ql3_run_retry_policies_run_fk',
|
||||
]),
|
||||
});
|
||||
@@ -0,0 +1,470 @@
|
||||
import {
|
||||
readPostgresMigrationHistory,
|
||||
type PostgresMigrationQueryable,
|
||||
} from '../adapters/postgresMigrationStreamStore';
|
||||
import { auditMigrationStreamHistory } from '../core/migrationStream';
|
||||
import { postgresqlMainMigrationStream } from '.';
|
||||
import {
|
||||
postgresqlControlSchemaContract,
|
||||
type PostgresSchemaContract,
|
||||
} from './schemaContract';
|
||||
|
||||
export const POSTGRES_SCHEMA_READINESS_ERROR_CODES = [
|
||||
'server_version_unsupported',
|
||||
'server_not_writable_primary',
|
||||
'migration_history_invalid',
|
||||
'capability_invalid',
|
||||
'schema_contract_invalid',
|
||||
'runtime_role_invalid',
|
||||
] as const;
|
||||
|
||||
export type PostgresSchemaReadinessErrorCode =
|
||||
(typeof POSTGRES_SCHEMA_READINESS_ERROR_CODES)[number];
|
||||
|
||||
export class PostgresSchemaReadinessError extends Error {
|
||||
constructor(
|
||||
readonly code: PostgresSchemaReadinessErrorCode,
|
||||
readonly facts: readonly string[] = [],
|
||||
) {
|
||||
super(`PostgreSQL schema is not ready: ${code}`);
|
||||
this.name = 'PostgresSchemaReadinessError';
|
||||
}
|
||||
}
|
||||
|
||||
export interface PostgresSchemaReadinessReport {
|
||||
readonly ready: true;
|
||||
readonly writablePrimary: true;
|
||||
readonly serverVersionNum: number;
|
||||
readonly serverMajor: number;
|
||||
readonly currentUser: string;
|
||||
readonly contractName: string;
|
||||
readonly contractVersion: number;
|
||||
readonly migrationIds: readonly string[];
|
||||
}
|
||||
|
||||
interface ServerRow extends Record<string, unknown> {
|
||||
serverVersionNum: unknown;
|
||||
currentUser: unknown;
|
||||
inRecovery: unknown;
|
||||
transactionReadOnly: unknown;
|
||||
}
|
||||
|
||||
interface CapabilityRow extends Record<string, unknown> {
|
||||
contractName: unknown;
|
||||
contractVersion: unknown;
|
||||
migrationId: unknown;
|
||||
capabilities: unknown;
|
||||
}
|
||||
|
||||
interface ColumnRow extends Record<string, unknown> {
|
||||
tableName: unknown;
|
||||
columnName: unknown;
|
||||
}
|
||||
|
||||
interface IndexRow extends Record<string, unknown> {
|
||||
indexName: unknown;
|
||||
}
|
||||
|
||||
interface ConstraintRow extends Record<string, unknown> {
|
||||
constraintName: unknown;
|
||||
constraintType: unknown;
|
||||
}
|
||||
|
||||
interface SchemaPrivilegeRow extends Record<string, unknown> {
|
||||
schemaUsage: unknown;
|
||||
schemaCreate: unknown;
|
||||
}
|
||||
|
||||
interface TablePrivilegeRow extends Record<string, unknown> {
|
||||
tableName: unknown;
|
||||
selectAllowed: unknown;
|
||||
insertAllowed: unknown;
|
||||
updateAllowed: unknown;
|
||||
deleteAllowed: unknown;
|
||||
isOwner: unknown;
|
||||
}
|
||||
|
||||
const REQUIRED_RUNTIME_PRIVILEGES = Object.freeze({
|
||||
schema_migrations: Object.freeze({
|
||||
select: true,
|
||||
insert: false,
|
||||
update: false,
|
||||
delete: false,
|
||||
}),
|
||||
schema_capabilities: Object.freeze({
|
||||
select: true,
|
||||
insert: false,
|
||||
update: false,
|
||||
delete: false,
|
||||
}),
|
||||
runs: Object.freeze({
|
||||
select: true,
|
||||
insert: true,
|
||||
update: true,
|
||||
delete: false,
|
||||
}),
|
||||
run_attempts: Object.freeze({
|
||||
select: true,
|
||||
insert: true,
|
||||
update: true,
|
||||
delete: false,
|
||||
}),
|
||||
run_events: Object.freeze({
|
||||
select: true,
|
||||
insert: true,
|
||||
update: false,
|
||||
delete: false,
|
||||
}),
|
||||
run_retry_policies: Object.freeze({
|
||||
select: true,
|
||||
insert: true,
|
||||
update: true,
|
||||
delete: false,
|
||||
}),
|
||||
});
|
||||
|
||||
function safeInteger(value: unknown): number | null {
|
||||
if (typeof value === 'number' && Number.isSafeInteger(value)) return value;
|
||||
if (typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value)) {
|
||||
const parsed = Number(value);
|
||||
return Number.isSafeInteger(parsed) ? parsed : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function sorted(values: Iterable<string>): string[] {
|
||||
return [...values].sort((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
function exactJsonObject(
|
||||
actual: unknown,
|
||||
expected: Readonly<Record<string, unknown>>,
|
||||
): boolean {
|
||||
if (!actual || typeof actual !== 'object' || Array.isArray(actual)) {
|
||||
return false;
|
||||
}
|
||||
const actualObject = actual as Record<string, unknown>;
|
||||
const actualKeys = sorted(Object.keys(actualObject));
|
||||
const expectedKeys = sorted(Object.keys(expected));
|
||||
return (
|
||||
actualKeys.length === expectedKeys.length &&
|
||||
actualKeys.every(
|
||||
(key, index) =>
|
||||
key === expectedKeys[index] && actualObject[key] === expected[key],
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
async function readServer(
|
||||
queryable: PostgresMigrationQueryable,
|
||||
contract: PostgresSchemaContract,
|
||||
): Promise<{
|
||||
writablePrimary: true;
|
||||
serverVersionNum: number;
|
||||
serverMajor: number;
|
||||
currentUser: string;
|
||||
}> {
|
||||
const result = await queryable.query<ServerRow>(
|
||||
`
|
||||
SELECT
|
||||
current_setting('server_version_num') AS "serverVersionNum",
|
||||
current_user AS "currentUser",
|
||||
pg_is_in_recovery() AS "inRecovery",
|
||||
current_setting('transaction_read_only') AS "transactionReadOnly"
|
||||
`.trim(),
|
||||
);
|
||||
const row = result.rows[0];
|
||||
const serverVersionNum = safeInteger(row?.serverVersionNum);
|
||||
const currentUser = row?.currentUser;
|
||||
const inRecovery = row?.inRecovery;
|
||||
const transactionReadOnly = row?.transactionReadOnly;
|
||||
const serverMajor =
|
||||
serverVersionNum === null ? null : Math.floor(serverVersionNum / 10_000);
|
||||
if (
|
||||
result.rows.length !== 1 ||
|
||||
serverVersionNum === null ||
|
||||
serverMajor === null ||
|
||||
serverMajor < contract.minimumServerMajor ||
|
||||
serverMajor > contract.maximumServerMajor ||
|
||||
typeof currentUser !== 'string' ||
|
||||
currentUser.length === 0
|
||||
) {
|
||||
throw new PostgresSchemaReadinessError('server_version_unsupported', [
|
||||
String(serverVersionNum ?? 'invalid'),
|
||||
]);
|
||||
}
|
||||
if (inRecovery !== false || transactionReadOnly !== 'off') {
|
||||
throw new PostgresSchemaReadinessError('server_not_writable_primary', [
|
||||
`in-recovery:${String(inRecovery)}`,
|
||||
`transaction-read-only:${String(transactionReadOnly)}`,
|
||||
]);
|
||||
}
|
||||
return {
|
||||
writablePrimary: true,
|
||||
serverVersionNum,
|
||||
serverMajor,
|
||||
currentUser,
|
||||
};
|
||||
}
|
||||
|
||||
async function assertHistory(
|
||||
queryable: PostgresMigrationQueryable,
|
||||
): Promise<readonly string[]> {
|
||||
const history = await readPostgresMigrationHistory(queryable);
|
||||
try {
|
||||
auditMigrationStreamHistory(history, postgresqlMainMigrationStream);
|
||||
return Object.freeze(history.map(({ migrationId }) => migrationId));
|
||||
} catch (error) {
|
||||
throw new PostgresSchemaReadinessError('migration_history_invalid', [
|
||||
error instanceof Error ? error.name : 'UnknownError',
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
async function assertCapability(
|
||||
queryable: PostgresMigrationQueryable,
|
||||
contract: PostgresSchemaContract,
|
||||
): Promise<void> {
|
||||
const result = await queryable.query<CapabilityRow>(
|
||||
`
|
||||
SELECT
|
||||
contract_name AS "contractName",
|
||||
contract_version AS "contractVersion",
|
||||
migration_id AS "migrationId",
|
||||
capabilities
|
||||
FROM "${contract.schema}"."schema_capabilities"
|
||||
WHERE contract_name = $1
|
||||
`.trim(),
|
||||
[contract.contractName],
|
||||
);
|
||||
const row = result.rows[0];
|
||||
if (
|
||||
result.rows.length !== 1 ||
|
||||
!row ||
|
||||
row.contractName !== contract.contractName ||
|
||||
safeInteger(row.contractVersion) !== contract.contractVersion ||
|
||||
row.migrationId !== contract.migrationId ||
|
||||
!exactJsonObject(row.capabilities, contract.capabilities)
|
||||
) {
|
||||
throw new PostgresSchemaReadinessError('capability_invalid');
|
||||
}
|
||||
}
|
||||
|
||||
async function assertSchemaContract(
|
||||
queryable: PostgresMigrationQueryable,
|
||||
contract: PostgresSchemaContract,
|
||||
): Promise<void> {
|
||||
const [columnsResult, indexesResult, constraintsResult] = await Promise.all([
|
||||
queryable.query<ColumnRow>(
|
||||
`
|
||||
SELECT table_name AS "tableName", column_name AS "columnName"
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = $1
|
||||
ORDER BY table_name, ordinal_position
|
||||
`.trim(),
|
||||
[contract.schema],
|
||||
),
|
||||
queryable.query<IndexRow>(
|
||||
`
|
||||
SELECT indexname AS "indexName"
|
||||
FROM pg_indexes
|
||||
WHERE schemaname = $1
|
||||
ORDER BY indexname
|
||||
`.trim(),
|
||||
[contract.schema],
|
||||
),
|
||||
queryable.query<ConstraintRow>(
|
||||
`
|
||||
SELECT
|
||||
constraints.conname AS "constraintName",
|
||||
CASE constraints.contype
|
||||
WHEN 'c' THEN 'check'
|
||||
WHEN 'f' THEN 'foreign_key'
|
||||
END AS "constraintType"
|
||||
FROM pg_constraint constraints
|
||||
JOIN pg_class tables ON tables.oid = constraints.conrelid
|
||||
JOIN pg_namespace schemas ON schemas.oid = tables.relnamespace
|
||||
WHERE schemas.nspname = $1
|
||||
AND constraints.contype IN ('c', 'f')
|
||||
ORDER BY constraints.contype, constraints.conname
|
||||
`.trim(),
|
||||
[contract.schema],
|
||||
),
|
||||
]);
|
||||
const actualTables = new Map<string, Set<string>>();
|
||||
for (const row of columnsResult.rows) {
|
||||
if (
|
||||
typeof row.tableName !== 'string' ||
|
||||
typeof row.columnName !== 'string'
|
||||
) {
|
||||
throw new PostgresSchemaReadinessError('schema_contract_invalid');
|
||||
}
|
||||
const columns = actualTables.get(row.tableName) ?? new Set<string>();
|
||||
columns.add(row.columnName);
|
||||
actualTables.set(row.tableName, columns);
|
||||
}
|
||||
const expectedTables = new Map(
|
||||
contract.tables.map((table) => [table.name, new Set(table.columns)]),
|
||||
);
|
||||
const findings: string[] = [];
|
||||
for (const [tableName, expectedColumns] of expectedTables) {
|
||||
const actualColumns = actualTables.get(tableName);
|
||||
if (!actualColumns) {
|
||||
findings.push(`missing-table:${tableName}`);
|
||||
continue;
|
||||
}
|
||||
for (const column of expectedColumns) {
|
||||
if (!actualColumns.has(column)) {
|
||||
findings.push(`missing-column:${tableName}.${column}`);
|
||||
}
|
||||
}
|
||||
for (const column of actualColumns) {
|
||||
if (!expectedColumns.has(column)) {
|
||||
findings.push(`unknown-column:${tableName}.${column}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const tableName of actualTables.keys()) {
|
||||
if (!expectedTables.has(tableName))
|
||||
findings.push(`unknown-table:${tableName}`);
|
||||
}
|
||||
const actualIndexes = new Set<string>();
|
||||
for (const row of indexesResult.rows) {
|
||||
if (typeof row.indexName !== 'string') {
|
||||
throw new PostgresSchemaReadinessError('schema_contract_invalid');
|
||||
}
|
||||
actualIndexes.add(row.indexName);
|
||||
}
|
||||
const expectedIndexes = new Set(contract.indexes);
|
||||
for (const index of expectedIndexes) {
|
||||
if (!actualIndexes.has(index)) findings.push(`missing-index:${index}`);
|
||||
}
|
||||
for (const index of actualIndexes) {
|
||||
if (!expectedIndexes.has(index)) findings.push(`unknown-index:${index}`);
|
||||
}
|
||||
const actualChecks = new Set<string>();
|
||||
const actualForeignKeys = new Set<string>();
|
||||
for (const row of constraintsResult.rows) {
|
||||
if (
|
||||
typeof row.constraintName !== 'string' ||
|
||||
(row.constraintType !== 'check' && row.constraintType !== 'foreign_key')
|
||||
) {
|
||||
throw new PostgresSchemaReadinessError('schema_contract_invalid');
|
||||
}
|
||||
const target =
|
||||
row.constraintType === 'check' ? actualChecks : actualForeignKeys;
|
||||
target.add(row.constraintName);
|
||||
}
|
||||
for (const check of contract.checks) {
|
||||
if (!actualChecks.has(check)) findings.push(`missing-check:${check}`);
|
||||
}
|
||||
for (const check of actualChecks) {
|
||||
if (!contract.checks.includes(check))
|
||||
findings.push(`unknown-check:${check}`);
|
||||
}
|
||||
for (const foreignKey of contract.foreignKeys) {
|
||||
if (!actualForeignKeys.has(foreignKey)) {
|
||||
findings.push(`missing-foreign-key:${foreignKey}`);
|
||||
}
|
||||
}
|
||||
for (const foreignKey of actualForeignKeys) {
|
||||
if (!contract.foreignKeys.includes(foreignKey)) {
|
||||
findings.push(`unknown-foreign-key:${foreignKey}`);
|
||||
}
|
||||
}
|
||||
if (findings.length > 0) {
|
||||
throw new PostgresSchemaReadinessError(
|
||||
'schema_contract_invalid',
|
||||
sorted(findings),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function assertRuntimeRole(
|
||||
queryable: PostgresMigrationQueryable,
|
||||
contract: PostgresSchemaContract,
|
||||
): Promise<void> {
|
||||
const schemaResult = await queryable.query<SchemaPrivilegeRow>(
|
||||
`
|
||||
SELECT
|
||||
has_schema_privilege(current_user, $1, 'USAGE') AS "schemaUsage",
|
||||
has_schema_privilege(current_user, $1, 'CREATE') AS "schemaCreate"
|
||||
`.trim(),
|
||||
[contract.schema],
|
||||
);
|
||||
const schema = schemaResult.rows[0];
|
||||
const tableNames = Object.keys(REQUIRED_RUNTIME_PRIVILEGES);
|
||||
const tableResult = await queryable.query<TablePrivilegeRow>(
|
||||
`
|
||||
SELECT
|
||||
requested.table_name AS "tableName",
|
||||
has_table_privilege(current_user, format('%I.%I', $1, requested.table_name), 'SELECT') AS "selectAllowed",
|
||||
has_table_privilege(current_user, format('%I.%I', $1, requested.table_name), 'INSERT') AS "insertAllowed",
|
||||
has_table_privilege(current_user, format('%I.%I', $1, requested.table_name), 'UPDATE') AS "updateAllowed",
|
||||
has_table_privilege(current_user, format('%I.%I', $1, requested.table_name), 'DELETE') AS "deleteAllowed",
|
||||
pg_get_userbyid(classes.relowner) = current_user AS "isOwner"
|
||||
FROM unnest($2::text[]) AS requested(table_name)
|
||||
JOIN pg_namespace namespaces ON namespaces.nspname = $1
|
||||
JOIN pg_class classes
|
||||
ON classes.relnamespace = namespaces.oid
|
||||
AND classes.relname = requested.table_name
|
||||
ORDER BY requested.table_name
|
||||
`.trim(),
|
||||
[contract.schema, tableNames],
|
||||
);
|
||||
const findings: string[] = [];
|
||||
if (
|
||||
schemaResult.rows.length !== 1 ||
|
||||
schema?.schemaUsage !== true ||
|
||||
schema?.schemaCreate !== false
|
||||
) {
|
||||
findings.push('schema-privileges');
|
||||
}
|
||||
const privilegesByTable = new Map(
|
||||
tableResult.rows.map((row) => [row.tableName, row]),
|
||||
);
|
||||
for (const tableName of tableNames) {
|
||||
const expected =
|
||||
REQUIRED_RUNTIME_PRIVILEGES[
|
||||
tableName as keyof typeof REQUIRED_RUNTIME_PRIVILEGES
|
||||
];
|
||||
const actual = privilegesByTable.get(tableName);
|
||||
if (
|
||||
!actual ||
|
||||
actual.selectAllowed !== expected.select ||
|
||||
actual.insertAllowed !== expected.insert ||
|
||||
actual.updateAllowed !== expected.update ||
|
||||
actual.deleteAllowed !== expected.delete ||
|
||||
actual.isOwner !== false
|
||||
) {
|
||||
findings.push(`table-privileges:${tableName}`);
|
||||
}
|
||||
}
|
||||
if (privilegesByTable.size !== tableNames.length) {
|
||||
findings.push('table-privilege-row-count');
|
||||
}
|
||||
if (findings.length > 0) {
|
||||
throw new PostgresSchemaReadinessError(
|
||||
'runtime_role_invalid',
|
||||
sorted(findings),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function assertPostgresSchemaReady(
|
||||
queryable: PostgresMigrationQueryable,
|
||||
contract: PostgresSchemaContract = postgresqlControlSchemaContract,
|
||||
): Promise<PostgresSchemaReadinessReport> {
|
||||
const server = await readServer(queryable, contract);
|
||||
const migrationIds = await assertHistory(queryable);
|
||||
await assertCapability(queryable, contract);
|
||||
await assertSchemaContract(queryable, contract);
|
||||
await assertRuntimeRole(queryable, contract);
|
||||
return Object.freeze({
|
||||
ready: true,
|
||||
...server,
|
||||
contractName: contract.contractName,
|
||||
contractVersion: contract.contractVersion,
|
||||
migrationIds,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { createHash } from 'crypto';
|
||||
import type { MigrationStreamStep } from '../core/migrationStream';
|
||||
import type { PostgresMigrationContext } from '../adapters/postgresMigrationStreamStore';
|
||||
|
||||
export interface PostgresSqlMigrationDefinition {
|
||||
readonly id: string;
|
||||
readonly statements: readonly string[];
|
||||
}
|
||||
|
||||
export function checksumPostgresStatements(
|
||||
statements: readonly string[],
|
||||
): string {
|
||||
return createHash('sha256')
|
||||
.update(
|
||||
JSON.stringify({
|
||||
format: 1,
|
||||
statements,
|
||||
}),
|
||||
)
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
export function definePostgresSqlMigration(
|
||||
definition: PostgresSqlMigrationDefinition,
|
||||
): MigrationStreamStep<PostgresMigrationContext> {
|
||||
const statements = Object.freeze([...definition.statements]);
|
||||
return Object.freeze({
|
||||
id: definition.id,
|
||||
checksum: checksumPostgresStatements(statements),
|
||||
async up(context: PostgresMigrationContext) {
|
||||
for (const statement of statements) await context.query(statement);
|
||||
},
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user