mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-22 19:29:13 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,68 @@
|
||||
export const LOCAL_MODEL_INVOCATION_MIGRATION_STREAM_ID =
|
||||
'ql3-ai-model-invocation-sqlite';
|
||||
export const LOCAL_MODEL_INVOCATION_MIGRATION_ID = '9001-ai-model-invocations';
|
||||
export const LOCAL_MODEL_INVOCATION_USAGE_MIGRATION_ID =
|
||||
'9002-ai-model-usage-ledger';
|
||||
export const LOCAL_MODEL_INVOCATION_QUOTA_MIGRATION_ID =
|
||||
'9003-ai-model-usage-quota';
|
||||
export const LOCAL_MODEL_INVOCATION_PRICING_MIGRATION_ID =
|
||||
'9004-ai-model-pricing-snapshots';
|
||||
export const LOCAL_MODEL_PRICE_CATALOG_MIGRATION_ID =
|
||||
'9005-ai-model-price-catalog';
|
||||
export const LOCAL_MODEL_PRICE_CATALOG_AUTHORIZATION_MIGRATION_ID =
|
||||
'9006-ai-model-price-catalog-authorizations';
|
||||
export const LOCAL_MODEL_INVOCATION_FEATURE_ACTIVATION_MIGRATION_ID =
|
||||
'9007-ai-feature-activation';
|
||||
export const LOCAL_PLUGIN_PACKAGE_PROMPT_ADMISSION_MIGRATION_ID =
|
||||
'9008-ai-plugin-package-prompt-admissions';
|
||||
export const LOCAL_PLUGIN_PACKAGE_PROMPT_FINALIZATION_MIGRATION_ID =
|
||||
'9009-ai-plugin-package-prompt-finalizations';
|
||||
export const LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_ARTIFACT_MIGRATION_ID =
|
||||
'9010-ai-plugin-package-prompt-output-artifacts';
|
||||
export const LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_TOMBSTONE_MIGRATION_ID =
|
||||
'9011-ai-plugin-package-prompt-output-tombstones';
|
||||
export const LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_MIGRATION_ID =
|
||||
'9012-ai-plugin-package-prompt-output-key-retirements';
|
||||
export const LOCAL_MODEL_PROVIDER_CREDENTIAL_CATALOG_MIGRATION_ID =
|
||||
'9013-ai-model-provider-credential-catalog';
|
||||
export const POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID =
|
||||
'ql3-ai-model-invocation-postgresql';
|
||||
export const POSTGRES_MODEL_INVOCATION_MIGRATION_ID =
|
||||
'pg-9001-ai-model-invocations';
|
||||
export const POSTGRES_MODEL_INVOCATION_USAGE_MIGRATION_ID =
|
||||
'pg-9002-ai-model-usage-ledger';
|
||||
export const POSTGRES_MODEL_INVOCATION_QUOTA_MIGRATION_ID =
|
||||
'pg-9003-ai-model-usage-quota';
|
||||
export const POSTGRES_MODEL_INVOCATION_PRICING_MIGRATION_ID =
|
||||
'pg-9004-ai-model-pricing-snapshots';
|
||||
export const POSTGRES_MODEL_PRICE_CATALOG_MIGRATION_ID =
|
||||
'pg-9005-ai-model-price-catalog';
|
||||
export const POSTGRES_MODEL_PRICE_CATALOG_AUTHORIZATION_MIGRATION_ID =
|
||||
'pg-9006-ai-model-price-catalog-authorizations';
|
||||
export const POSTGRES_PLUGIN_PACKAGE_PROMPT_ADMISSION_MIGRATION_ID =
|
||||
'pg-9007-ai-plugin-package-prompt-admissions';
|
||||
export const POSTGRES_PLUGIN_PACKAGE_PROMPT_FINALIZATION_MIGRATION_ID =
|
||||
'pg-9008-ai-plugin-package-prompt-finalizations';
|
||||
export const POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_ARTIFACT_MIGRATION_ID =
|
||||
'pg-9009-ai-plugin-package-prompt-output-artifacts';
|
||||
export const POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_TOMBSTONE_MIGRATION_ID =
|
||||
'pg-9010-ai-plugin-package-prompt-output-tombstones';
|
||||
export const POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_MIGRATION_ID =
|
||||
'pg-9011-ai-plugin-package-prompt-output-key-retirements';
|
||||
export const POSTGRES_MODEL_PROVIDER_CREDENTIAL_CATALOG_MIGRATION_ID =
|
||||
'pg-9012-ai-model-provider-credential-catalog';
|
||||
export const POSTGRES_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_MIGRATION_ID =
|
||||
'pg-9013-ai-model-provider-credential-management-boundary';
|
||||
export const POSTGRES_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_MIGRATION_ID =
|
||||
'pg-9014-ai-model-provider-credential-management-identity-ledger';
|
||||
export const POSTGRES_MODEL_PROVIDER_CREDENTIAL_TEST_CONNECTION_MIGRATION_ID =
|
||||
'pg-9015-ai-model-provider-credential-test-connection';
|
||||
export const POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_ROTATION_MIGRATION_ID =
|
||||
'pg-9016-ai-plugin-package-prompt-output-key-rotation';
|
||||
export const POSTGRES_PLUGIN_PACKAGE_PROMPT_PRODUCT_AUTHORIZATION_MIGRATION_ID =
|
||||
'pg-9017-ai-plugin-package-prompt-product-authorization';
|
||||
export const POSTGRES_MODEL_INVOCATION_SCHEMA = 'ql3_ai';
|
||||
export const LOCAL_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE =
|
||||
'QingLong3AiSchemaMigrations';
|
||||
export const POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE =
|
||||
'ai_schema_migrations';
|
||||
@@ -0,0 +1,259 @@
|
||||
import type {
|
||||
PostgresPool,
|
||||
PostgresQueryable,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
MigrationStreamHistoryCorruptionError,
|
||||
runMigrationStream,
|
||||
type MigrationStreamDefinition,
|
||||
type MigrationStreamRecord,
|
||||
type MigrationStreamStore,
|
||||
type MigrationStreamTransaction,
|
||||
} from '@qinglong/runtime-core/migration-stream';
|
||||
|
||||
import {
|
||||
POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID,
|
||||
POSTGRES_MODEL_INVOCATION_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_USAGE_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_QUOTA_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_PRICING_MIGRATION_ID,
|
||||
POSTGRES_MODEL_PRICE_CATALOG_MIGRATION_ID,
|
||||
POSTGRES_MODEL_PRICE_CATALOG_AUTHORIZATION_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_ADMISSION_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_FINALIZATION_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_ARTIFACT_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_TOMBSTONE_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_MIGRATION_ID,
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_CATALOG_MIGRATION_ID,
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_MIGRATION_ID,
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_MIGRATION_ID,
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_TEST_CONNECTION_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_ROTATION_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_PRODUCT_AUTHORIZATION_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_SCHEMA,
|
||||
POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE,
|
||||
} from './identities';
|
||||
|
||||
import { postgresCatalogMigrations } from './postgres/catalog';
|
||||
import { postgresCoreMigrations } from './postgres/core';
|
||||
import { postgresCredentialMigrations } from './postgres/credential';
|
||||
import {
|
||||
postgresPromptBaseMigrations,
|
||||
postgresPromptExtensionMigrations,
|
||||
} from './postgres/prompt';
|
||||
import { postgresUsagePricingMigrations } from './postgres/usagePricing';
|
||||
import { historyRecord, type HistoryRow } from './shared';
|
||||
|
||||
const POSTGRES_HISTORY_IDENTITY = Object.freeze({
|
||||
migrationIds: Object.freeze([
|
||||
POSTGRES_MODEL_INVOCATION_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_USAGE_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_QUOTA_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_PRICING_MIGRATION_ID,
|
||||
POSTGRES_MODEL_PRICE_CATALOG_MIGRATION_ID,
|
||||
POSTGRES_MODEL_PRICE_CATALOG_AUTHORIZATION_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_ADMISSION_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_FINALIZATION_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_ARTIFACT_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_TOMBSTONE_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_ROTATION_MIGRATION_ID,
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_CATALOG_MIGRATION_ID,
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_MIGRATION_ID,
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_MIGRATION_ID,
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_TEST_CONNECTION_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_PRODUCT_AUTHORIZATION_MIGRATION_ID,
|
||||
]),
|
||||
streamId: POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID,
|
||||
dialect: 'postgresql' as const,
|
||||
});
|
||||
|
||||
async function readPostgresHistory(
|
||||
queryable: PostgresQueryable,
|
||||
migrationId?: string,
|
||||
): Promise<readonly MigrationStreamRecord[]> {
|
||||
const result = await queryable.query<HistoryRow>(
|
||||
`SELECT
|
||||
migration_id AS "migrationId",
|
||||
stream_id AS "streamId",
|
||||
dialect,
|
||||
checksum,
|
||||
applied_at_ms AS "appliedAtMs"
|
||||
FROM "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE}"
|
||||
WHERE stream_id = $1
|
||||
${migrationId === undefined ? '' : 'AND migration_id = $2'}
|
||||
ORDER BY migration_id`,
|
||||
migrationId === undefined
|
||||
? [POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID]
|
||||
: [POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID, migrationId],
|
||||
);
|
||||
return result.rows.map((row) =>
|
||||
historyRecord(row, POSTGRES_HISTORY_IDENTITY),
|
||||
);
|
||||
}
|
||||
|
||||
class PostgresModelInvocationMigrationStore
|
||||
implements MigrationStreamStore<PostgresQueryable>
|
||||
{
|
||||
constructor(private readonly pool: PostgresPool) {}
|
||||
|
||||
async ensureHistory(): Promise<void> {
|
||||
const result = await this.pool.query<Record<string, unknown>>(
|
||||
`SELECT
|
||||
to_regclass('ql3.schema_migrations') IS NOT NULL AS history,
|
||||
to_regclass('ql3.runs') IS NOT NULL AS runs,
|
||||
to_regclass('ql3.run_events') IS NOT NULL AS events,
|
||||
to_regclass('ql3.step_runs') IS NOT NULL AS steps,
|
||||
to_regclass('ql3.step_run_mutations') IS NOT NULL AS mutations`,
|
||||
);
|
||||
const row = result.rows[0];
|
||||
if (
|
||||
result.rows.length !== 1 ||
|
||||
row?.history !== true ||
|
||||
row.runs !== true ||
|
||||
row.events !== true ||
|
||||
row.steps !== true ||
|
||||
row.mutations !== true
|
||||
) {
|
||||
throw new TypeError(
|
||||
'PostgreSQL ModelInvocation feature requires the main migration stream',
|
||||
);
|
||||
}
|
||||
await this.pool.query(
|
||||
`CREATE SCHEMA IF NOT EXISTS "${POSTGRES_MODEL_INVOCATION_SCHEMA}"`,
|
||||
);
|
||||
await this.pool.query(
|
||||
`REVOKE ALL ON SCHEMA "${POSTGRES_MODEL_INVOCATION_SCHEMA}" FROM PUBLIC`,
|
||||
);
|
||||
await this.pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE}" (
|
||||
migration_id varchar(128) PRIMARY KEY,
|
||||
stream_id varchar(64) NOT NULL
|
||||
CONSTRAINT ql3_ai_schema_migrations_stream_check
|
||||
CHECK (stream_id = '${POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID}'),
|
||||
dialect varchar(16) NOT NULL
|
||||
CONSTRAINT ql3_ai_schema_migrations_dialect_check
|
||||
CHECK (dialect = 'postgresql'),
|
||||
checksum char(64) NOT NULL
|
||||
CONSTRAINT ql3_ai_schema_migrations_checksum_check
|
||||
CHECK (checksum ~ '^[0-9a-f]{64}$'),
|
||||
applied_at_ms bigint NOT NULL
|
||||
CONSTRAINT ql3_ai_schema_migrations_applied_at_check
|
||||
CHECK (applied_at_ms >= 0)
|
||||
)
|
||||
`);
|
||||
await this.pool.query(
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE}"
|
||||
FROM PUBLIC`,
|
||||
);
|
||||
}
|
||||
|
||||
async listAll(): Promise<readonly MigrationStreamRecord[]> {
|
||||
return readPostgresHistory(this.pool);
|
||||
}
|
||||
|
||||
async findById(migrationId: string): Promise<MigrationStreamRecord | null> {
|
||||
const rows = await readPostgresHistory(this.pool, migrationId);
|
||||
if (rows.length > 1) {
|
||||
throw new MigrationStreamHistoryCorruptionError(migrationId);
|
||||
}
|
||||
return rows[0] ?? null;
|
||||
}
|
||||
|
||||
async transaction<T>(
|
||||
work: (
|
||||
transaction: MigrationStreamTransaction<PostgresQueryable>,
|
||||
) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const client = await this.pool.connect();
|
||||
let began = false;
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
began = true;
|
||||
await client.query(`SELECT set_config('statement_timeout', $1, true)`, [
|
||||
'15s',
|
||||
]);
|
||||
await client.query(`SELECT set_config('lock_timeout', $1, true)`, ['5s']);
|
||||
await client.query(
|
||||
`SELECT set_config('idle_in_transaction_session_timeout', $1, true)`,
|
||||
['15s'],
|
||||
);
|
||||
await client.query(
|
||||
'SELECT pg_advisory_xact_lock($1, $2)',
|
||||
[0x514c, 0x0301],
|
||||
);
|
||||
const result = await work({
|
||||
context: client,
|
||||
findById: async (migrationId) => {
|
||||
const rows = await readPostgresHistory(client, migrationId);
|
||||
return rows[0] ?? null;
|
||||
},
|
||||
insert: async (record) => {
|
||||
if (
|
||||
record.streamId !== POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID ||
|
||||
!POSTGRES_HISTORY_IDENTITY.migrationIds.includes(
|
||||
record.migrationId,
|
||||
) ||
|
||||
record.dialect !== 'postgresql'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'PostgreSQL ModelInvocation migration identity is invalid',
|
||||
);
|
||||
}
|
||||
await client.query(
|
||||
`INSERT INTO "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE}"
|
||||
(migration_id, stream_id, dialect, checksum, applied_at_ms)
|
||||
VALUES ($1, $2, $3, $4, $5)`,
|
||||
[
|
||||
record.migrationId,
|
||||
record.streamId,
|
||||
record.dialect,
|
||||
record.checksum,
|
||||
record.appliedAtMs,
|
||||
],
|
||||
);
|
||||
},
|
||||
});
|
||||
await client.query('COMMIT');
|
||||
began = false;
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (began) {
|
||||
try {
|
||||
await client.query('ROLLBACK');
|
||||
} catch {
|
||||
// Preserve the migration failure.
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const postgresModelInvocationMigrationDefinition: MigrationStreamDefinition<PostgresQueryable> =
|
||||
Object.freeze({
|
||||
id: POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID,
|
||||
dialect: 'postgresql',
|
||||
migrationIdScheme: 'postgres-prefixed',
|
||||
checksumScheme: 'sha256',
|
||||
migrations: Object.freeze([
|
||||
...postgresCoreMigrations,
|
||||
...postgresUsagePricingMigrations,
|
||||
...postgresCatalogMigrations,
|
||||
...postgresPromptBaseMigrations,
|
||||
...postgresCredentialMigrations,
|
||||
...postgresPromptExtensionMigrations,
|
||||
]),
|
||||
});
|
||||
|
||||
export async function migratePostgresModelInvocationFeature(
|
||||
pool: PostgresPool,
|
||||
): Promise<void> {
|
||||
await runMigrationStream({
|
||||
stream: postgresModelInvocationMigrationDefinition,
|
||||
store: new PostgresModelInvocationMigrationStore(pool),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
import type { PostgresQueryable } from '@qinglong/runtime-core';
|
||||
|
||||
import {
|
||||
POSTGRES_MODEL_PRICE_CATALOG_MIGRATION_ID,
|
||||
POSTGRES_MODEL_PRICE_CATALOG_AUTHORIZATION_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_SCHEMA,
|
||||
} from '../identities';
|
||||
|
||||
import { defineSqlMigration } from '../shared';
|
||||
|
||||
const POSTGRES_MODEL_PRICE_CATALOG_PUBLICATION_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_publications" (
|
||||
provider varchar(128) NOT NULL,
|
||||
model varchar(128) NOT NULL,
|
||||
price_revision varchar(128) NOT NULL,
|
||||
catalog_digest char(64) NOT NULL,
|
||||
mutation_id varchar(128) NOT NULL UNIQUE,
|
||||
command_digest char(64) NOT NULL,
|
||||
publication_digest char(64) NOT NULL,
|
||||
published_at_ms bigint NOT NULL,
|
||||
published_by_user_id varchar(128) NOT NULL,
|
||||
publication_json jsonb NOT NULL,
|
||||
PRIMARY KEY (provider, model, price_revision),
|
||||
CONSTRAINT ql3_model_price_catalog_publication_identity_check CHECK (
|
||||
provider ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
model ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
price_revision ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
mutation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
published_by_user_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$'
|
||||
),
|
||||
CONSTRAINT ql3_model_price_catalog_publication_value_check CHECK (
|
||||
published_at_ms >= 0 AND
|
||||
catalog_digest ~ '^[0-9a-f]{64}$' AND
|
||||
command_digest ~ '^[0-9a-f]{64}$' AND
|
||||
publication_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_model_price_catalog_publication_json_check CHECK (
|
||||
jsonb_typeof(publication_json) = 'object' AND
|
||||
octet_length(publication_json::text) BETWEEN 2 AND 24576 AND
|
||||
publication_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-price-catalog-publication@v1',
|
||||
'mutationId', mutation_id,
|
||||
'publishedByUserId', published_by_user_id,
|
||||
'commandDigest', command_digest,
|
||||
'publicationDigest', publication_digest
|
||||
) AND
|
||||
publication_json->'entry' @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-price-catalog-entry@v1',
|
||||
'provider', provider,
|
||||
'model', model,
|
||||
'priceRevision', price_revision,
|
||||
'catalogDigest', catalog_digest,
|
||||
'publishedAtMs', published_at_ms
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const POSTGRES_MODEL_PRICE_CATALOG_HEAD_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_heads" (
|
||||
provider varchar(128) NOT NULL,
|
||||
model varchar(128) NOT NULL,
|
||||
generation integer NOT NULL,
|
||||
previous_head_digest char(64),
|
||||
active_price_revision varchar(128),
|
||||
active_catalog_digest char(64),
|
||||
revoked_price_revision varchar(128),
|
||||
revoked_catalog_digest char(64),
|
||||
action varchar(16) NOT NULL,
|
||||
mutation_id varchar(128) NOT NULL UNIQUE,
|
||||
changed_by_user_id varchar(128) NOT NULL,
|
||||
changed_at_ms bigint NOT NULL,
|
||||
command_digest char(64) NOT NULL,
|
||||
head_digest char(64) NOT NULL UNIQUE,
|
||||
head_json jsonb NOT NULL,
|
||||
PRIMARY KEY (provider, model, generation),
|
||||
FOREIGN KEY (previous_head_digest)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_heads"
|
||||
(head_digest) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (
|
||||
provider, model, active_price_revision, active_catalog_digest
|
||||
) REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_publications" (
|
||||
provider, model, price_revision, catalog_digest
|
||||
) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (
|
||||
provider, model, revoked_price_revision, revoked_catalog_digest
|
||||
) REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_publications" (
|
||||
provider, model, price_revision, catalog_digest
|
||||
) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_price_catalog_head_identity_check CHECK (
|
||||
provider ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
model ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
mutation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
changed_by_user_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$'
|
||||
),
|
||||
CONSTRAINT ql3_model_price_catalog_head_value_check CHECK (
|
||||
generation BETWEEN 1 AND 2147483647 AND
|
||||
((generation = 1 AND previous_head_digest IS NULL) OR
|
||||
(generation > 1 AND previous_head_digest IS NOT NULL)) AND
|
||||
((active_price_revision IS NULL AND active_catalog_digest IS NULL) OR
|
||||
(active_price_revision IS NOT NULL AND
|
||||
active_catalog_digest IS NOT NULL)) AND
|
||||
((revoked_price_revision IS NULL AND revoked_catalog_digest IS NULL) OR
|
||||
(revoked_price_revision IS NOT NULL AND
|
||||
revoked_catalog_digest IS NOT NULL)) AND
|
||||
(revoked_price_revision IS NULL OR
|
||||
revoked_price_revision <> active_price_revision) AND
|
||||
action IN ('activate', 'deactivate', 'revoke') AND
|
||||
((action = 'revoke') = (revoked_price_revision IS NOT NULL)) AND
|
||||
changed_at_ms >= 0 AND
|
||||
(previous_head_digest IS NULL OR
|
||||
previous_head_digest ~ '^[0-9a-f]{64}$') AND
|
||||
(active_catalog_digest IS NULL OR
|
||||
active_catalog_digest ~ '^[0-9a-f]{64}$') AND
|
||||
(revoked_catalog_digest IS NULL OR
|
||||
revoked_catalog_digest ~ '^[0-9a-f]{64}$') AND
|
||||
command_digest ~ '^[0-9a-f]{64}$' AND
|
||||
head_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_model_price_catalog_head_json_check CHECK (
|
||||
jsonb_typeof(head_json) = 'object' AND
|
||||
octet_length(head_json::text) BETWEEN 2 AND 24576 AND
|
||||
head_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-price-catalog-head@v1',
|
||||
'provider', provider,
|
||||
'model', model,
|
||||
'generation', generation,
|
||||
'previousHeadDigest', previous_head_digest,
|
||||
'activePriceRevision', active_price_revision,
|
||||
'activeCatalogDigest', active_catalog_digest,
|
||||
'revokedPriceRevision', revoked_price_revision,
|
||||
'revokedCatalogDigest', revoked_catalog_digest,
|
||||
'action', action,
|
||||
'mutationId', mutation_id,
|
||||
'changedByUserId', changed_by_user_id,
|
||||
'changedAtMs', changed_at_ms,
|
||||
'commandDigest', command_digest,
|
||||
'headDigest', head_digest
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const postgresModelPriceCatalogMigration =
|
||||
defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_MODEL_PRICE_CATALOG_MIGRATION_ID,
|
||||
[
|
||||
POSTGRES_MODEL_PRICE_CATALOG_PUBLICATION_TABLE_SQL,
|
||||
`CREATE UNIQUE INDEX ql3_model_price_catalog_publication_identity_digest_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_publications"
|
||||
(provider, model, price_revision, catalog_digest)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_price_catalog_publication_digest_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_publications"
|
||||
(publication_digest)`,
|
||||
POSTGRES_MODEL_PRICE_CATALOG_HEAD_TABLE_SQL,
|
||||
`CREATE INDEX ql3_model_price_catalog_head_current_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_heads"
|
||||
(provider, model, generation DESC)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_price_catalog_revoked_revision_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_heads"
|
||||
(provider, model, revoked_price_revision)
|
||||
WHERE revoked_price_revision IS NOT NULL`,
|
||||
`GRANT USAGE ON SCHEMA "${POSTGRES_MODEL_INVOCATION_SCHEMA}"
|
||||
TO ql3_admin`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_publications",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_heads"
|
||||
FROM PUBLIC`,
|
||||
`GRANT SELECT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_publications",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_heads"
|
||||
TO ql3_runtime`,
|
||||
`GRANT SELECT, INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_publications",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_heads"
|
||||
TO ql3_admin`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
const POSTGRES_MODEL_PRICE_CATALOG_AUTHORIZATION_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_authorizations" (
|
||||
authorization_id varchar(128) PRIMARY KEY,
|
||||
request_id varchar(128) NOT NULL,
|
||||
operation varchar(16) NOT NULL,
|
||||
provider varchar(128) NOT NULL,
|
||||
model varchar(128) NOT NULL,
|
||||
price_revision varchar(128),
|
||||
catalog_command_digest char(64) NOT NULL UNIQUE,
|
||||
publication_digest char(64) UNIQUE,
|
||||
head_digest char(64) UNIQUE,
|
||||
result_digest char(64) NOT NULL UNIQUE,
|
||||
user_id varchar(128) NOT NULL,
|
||||
authentication_id varchar(128) NOT NULL,
|
||||
assurance varchar(32) NOT NULL,
|
||||
authenticated_at_ms bigint NOT NULL,
|
||||
expires_at_ms bigint NOT NULL,
|
||||
policy_revision varchar(128) NOT NULL,
|
||||
policy_decision_digest char(64) NOT NULL,
|
||||
decision_mode varchar(32) NOT NULL,
|
||||
command_digest char(64) NOT NULL UNIQUE,
|
||||
committed_at_ms bigint NOT NULL,
|
||||
authorization_digest char(64) NOT NULL UNIQUE,
|
||||
reasons_json jsonb NOT NULL,
|
||||
authorization_json jsonb NOT NULL,
|
||||
FOREIGN KEY (publication_digest)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_publications"
|
||||
(publication_digest) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (head_digest)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_heads"
|
||||
(head_digest) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_price_catalog_authorization_identity_check CHECK (
|
||||
authorization_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
request_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND
|
||||
provider ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
model ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
(price_revision IS NULL OR
|
||||
price_revision ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$') AND
|
||||
user_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
authentication_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND
|
||||
policy_revision ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$'
|
||||
),
|
||||
CONSTRAINT ql3_model_price_catalog_authorization_value_check CHECK (
|
||||
operation IN ('publish', 'activate', 'deactivate', 'revoke') AND
|
||||
((operation = 'deactivate' AND price_revision IS NULL) OR
|
||||
(operation <> 'deactivate' AND price_revision IS NOT NULL)) AND
|
||||
((operation = 'publish' AND publication_digest IS NOT NULL AND
|
||||
head_digest IS NULL AND result_digest = publication_digest) OR
|
||||
(operation <> 'publish' AND publication_digest IS NULL AND
|
||||
head_digest IS NOT NULL AND result_digest = head_digest)) AND
|
||||
assurance IN ('multi_factor', 'hardware', 'local_console') AND
|
||||
decision_mode IN ('human_confirmation', 'separation_of_duty') AND
|
||||
authenticated_at_ms >= 0 AND
|
||||
expires_at_ms > authenticated_at_ms AND
|
||||
committed_at_ms >= authenticated_at_ms AND
|
||||
committed_at_ms < expires_at_ms AND
|
||||
committed_at_ms - authenticated_at_ms <= 300000 AND
|
||||
catalog_command_digest ~ '^[0-9a-f]{64}$' AND
|
||||
result_digest ~ '^[0-9a-f]{64}$' AND
|
||||
policy_decision_digest ~ '^[0-9a-f]{64}$' AND
|
||||
command_digest ~ '^[0-9a-f]{64}$' AND
|
||||
authorization_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_model_price_catalog_authorization_reasons_check CHECK (
|
||||
jsonb_typeof(reasons_json) = 'array' AND
|
||||
jsonb_array_length(reasons_json) BETWEEN 1 AND 8 AND
|
||||
octet_length(reasons_json::text) BETWEEN 5 AND 2048
|
||||
),
|
||||
CONSTRAINT ql3_model_price_catalog_authorization_json_check CHECK (
|
||||
jsonb_typeof(authorization_json) = 'object' AND
|
||||
octet_length(authorization_json::text) BETWEEN 2 AND 32768 AND
|
||||
authorization_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-price-catalog-authorization@v1',
|
||||
'authorizationId', authorization_id,
|
||||
'requestId', request_id,
|
||||
'operation', operation,
|
||||
'provider', provider,
|
||||
'model', model,
|
||||
'priceRevision', price_revision,
|
||||
'catalogCommandDigest', catalog_command_digest,
|
||||
'resultDigest', result_digest,
|
||||
'decisionMode', decision_mode,
|
||||
'commandDigest', command_digest,
|
||||
'committedAtMs', committed_at_ms,
|
||||
'authorizationDigest', authorization_digest
|
||||
) AND
|
||||
authorization_json->'principal' @> jsonb_build_object(
|
||||
'subject', jsonb_build_object('type', 'user', 'id', user_id),
|
||||
'authenticationId', authentication_id,
|
||||
'assurance', assurance,
|
||||
'authenticatedAtMs', authenticated_at_ms,
|
||||
'expiresAtMs', expires_at_ms
|
||||
) AND
|
||||
authorization_json->'policy' @> jsonb_build_object(
|
||||
'revision', policy_revision,
|
||||
'decisionDigest', policy_decision_digest,
|
||||
'reasons', reasons_json
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const postgresModelPriceCatalogAuthorizationMigration =
|
||||
defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_MODEL_PRICE_CATALOG_AUTHORIZATION_MIGRATION_ID,
|
||||
[
|
||||
POSTGRES_MODEL_PRICE_CATALOG_AUTHORIZATION_TABLE_SQL,
|
||||
`CREATE INDEX ql3_model_price_catalog_authorization_target_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_authorizations"
|
||||
(provider, model, operation, committed_at_ms DESC)`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_authorizations"
|
||||
FROM PUBLIC`,
|
||||
`GRANT SELECT, INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_price_catalog_authorizations"
|
||||
TO ql3_admin`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
export const postgresCatalogMigrations = Object.freeze([
|
||||
postgresModelPriceCatalogMigration,
|
||||
postgresModelPriceCatalogAuthorizationMigration,
|
||||
]);
|
||||
@@ -0,0 +1,249 @@
|
||||
import type { PostgresQueryable } from '@qinglong/runtime-core';
|
||||
|
||||
import {
|
||||
POSTGRES_MODEL_INVOCATION_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_SCHEMA,
|
||||
} from '../identities';
|
||||
|
||||
import { defineSqlMigration } from '../shared';
|
||||
|
||||
const POSTGRES_START_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_starts" (
|
||||
invocation_id varchar(128) PRIMARY KEY,
|
||||
project_id varchar(128) NOT NULL,
|
||||
run_id varchar(36) NOT NULL,
|
||||
step_run_id varchar(128) NOT NULL,
|
||||
trace_id varchar(128) NOT NULL,
|
||||
provider varchar(128) NOT NULL,
|
||||
model varchar(128) NOT NULL,
|
||||
policy_revision varchar(128) NOT NULL,
|
||||
request_digest varchar(71) NOT NULL,
|
||||
input_bytes integer NOT NULL,
|
||||
max_output_tokens integer NOT NULL,
|
||||
deadline_at_ms bigint NOT NULL,
|
||||
admitted_at_ms bigint NOT NULL,
|
||||
mutation_id varchar(128) NOT NULL,
|
||||
mutation_digest char(64) NOT NULL,
|
||||
run_event_id varchar(36) NOT NULL,
|
||||
start_digest char(64) NOT NULL,
|
||||
record_json jsonb NOT NULL,
|
||||
FOREIGN KEY (run_id, step_run_id)
|
||||
REFERENCES "ql3"."step_runs" (run_id, id) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (mutation_id)
|
||||
REFERENCES "ql3"."step_run_mutations" (mutation_id) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (run_event_id)
|
||||
REFERENCES "ql3"."run_events" (id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_invocation_starts_identity_check CHECK (
|
||||
invocation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
project_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND
|
||||
step_run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
trace_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
provider ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
model ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
policy_revision ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
mutation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_starts_value_check CHECK (
|
||||
request_digest ~ '^sha256:[0-9a-f]{64}$' AND
|
||||
mutation_digest ~ '^[0-9a-f]{64}$' AND
|
||||
start_digest ~ '^[0-9a-f]{64}$' AND
|
||||
input_bytes BETWEEN 1 AND 262144 AND
|
||||
max_output_tokens BETWEEN 1 AND 32768 AND
|
||||
admitted_at_ms >= 0 AND deadline_at_ms > admitted_at_ms
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_starts_json_check CHECK (
|
||||
jsonb_typeof(record_json) = 'object' AND
|
||||
octet_length(record_json::text) BETWEEN 2 AND 24576 AND
|
||||
record_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-invocation-start@v1',
|
||||
'invocationId', invocation_id,
|
||||
'projectId', project_id,
|
||||
'runId', run_id,
|
||||
'stepRunId', step_run_id,
|
||||
'traceId', trace_id,
|
||||
'provider', provider,
|
||||
'model', model,
|
||||
'policyRevision', policy_revision,
|
||||
'requestDigest', request_digest,
|
||||
'inputBytes', input_bytes,
|
||||
'maxOutputTokens', max_output_tokens,
|
||||
'deadlineAtMs', deadline_at_ms,
|
||||
'admittedAtMs', admitted_at_ms,
|
||||
'stepRunMutationId', mutation_id,
|
||||
'stepRunMutationDigest', mutation_digest,
|
||||
'runEventId', run_event_id,
|
||||
'startDigest', start_digest
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const POSTGRES_COMPLETION_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_completions" (
|
||||
invocation_id varchar(128) PRIMARY KEY,
|
||||
project_id varchar(128) NOT NULL,
|
||||
run_id varchar(36) NOT NULL,
|
||||
step_run_id varchar(128) NOT NULL,
|
||||
trace_id varchar(128) NOT NULL,
|
||||
start_digest char(64) NOT NULL,
|
||||
outcome varchar(32) NOT NULL,
|
||||
output_bytes integer NOT NULL,
|
||||
error_code varchar(64),
|
||||
completed_at_ms bigint NOT NULL,
|
||||
mutation_id varchar(128) NOT NULL,
|
||||
mutation_digest char(64) NOT NULL,
|
||||
run_event_id varchar(36) NOT NULL,
|
||||
completion_digest char(64) NOT NULL,
|
||||
record_json jsonb NOT NULL,
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_starts" (invocation_id)
|
||||
ON DELETE RESTRICT,
|
||||
FOREIGN KEY (run_id, step_run_id)
|
||||
REFERENCES "ql3"."step_runs" (run_id, id) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (mutation_id)
|
||||
REFERENCES "ql3"."step_run_mutations" (mutation_id) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (run_event_id)
|
||||
REFERENCES "ql3"."run_events" (id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_invocation_completions_value_check CHECK (
|
||||
outcome IN ('succeeded', 'failed', 'timed_out', 'outcome_unknown') AND
|
||||
output_bytes BETWEEN 0 AND 1048576 AND completed_at_ms >= 0 AND
|
||||
((outcome = 'succeeded' AND error_code IS NULL) OR
|
||||
(outcome <> 'succeeded' AND
|
||||
error_code ~ '^[A-Z][A-Z0-9_]{0,63}$')) AND
|
||||
start_digest ~ '^[0-9a-f]{64}$' AND
|
||||
mutation_digest ~ '^[0-9a-f]{64}$' AND
|
||||
completion_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_completions_json_check CHECK (
|
||||
jsonb_typeof(record_json) = 'object' AND
|
||||
octet_length(record_json::text) BETWEEN 2 AND 24576 AND
|
||||
record_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-invocation-completion@v1',
|
||||
'invocationId', invocation_id,
|
||||
'projectId', project_id,
|
||||
'runId', run_id,
|
||||
'stepRunId', step_run_id,
|
||||
'traceId', trace_id,
|
||||
'startDigest', start_digest,
|
||||
'outcome', outcome,
|
||||
'outputBytes', output_bytes,
|
||||
'errorCode', error_code,
|
||||
'completedAtMs', completed_at_ms,
|
||||
'stepRunMutationId', mutation_id,
|
||||
'stepRunMutationDigest', mutation_digest,
|
||||
'runEventId', run_event_id,
|
||||
'completionDigest', completion_digest
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const POSTGRES_RESOLUTION_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_resolutions" (
|
||||
resolution_id varchar(128) PRIMARY KEY,
|
||||
invocation_id varchar(128) NOT NULL UNIQUE,
|
||||
project_id varchar(128) NOT NULL,
|
||||
run_id varchar(36) NOT NULL,
|
||||
step_run_id varchar(128) NOT NULL,
|
||||
trace_id varchar(128) NOT NULL,
|
||||
completion_digest char(64) NOT NULL,
|
||||
decision varchar(16) NOT NULL,
|
||||
resolved_by_user_id varchar(128) NOT NULL,
|
||||
resolved_at_ms bigint NOT NULL,
|
||||
mutation_id varchar(128) NOT NULL UNIQUE,
|
||||
mutation_digest char(64) NOT NULL,
|
||||
run_event_id varchar(36) NOT NULL UNIQUE,
|
||||
resolution_digest char(64) NOT NULL UNIQUE,
|
||||
record_json jsonb NOT NULL,
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_completions" (invocation_id)
|
||||
ON DELETE RESTRICT,
|
||||
FOREIGN KEY (run_id, step_run_id)
|
||||
REFERENCES "ql3"."step_runs" (run_id, id) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (mutation_id)
|
||||
REFERENCES "ql3"."step_run_mutations" (mutation_id) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (run_event_id)
|
||||
REFERENCES "ql3"."run_events" (id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_invocation_resolutions_value_check CHECK (
|
||||
decision IN ('retry', 'fail', 'cancel') AND
|
||||
resolved_at_ms >= 0 AND
|
||||
resolution_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
invocation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
project_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND
|
||||
step_run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
trace_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
resolved_by_user_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
mutation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND
|
||||
completion_digest ~ '^[0-9a-f]{64}$' AND
|
||||
mutation_digest ~ '^[0-9a-f]{64}$' AND
|
||||
resolution_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_resolutions_json_check CHECK (
|
||||
jsonb_typeof(record_json) = 'object' AND
|
||||
octet_length(record_json::text) BETWEEN 2 AND 24576 AND
|
||||
record_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-invocation-resolution@v1',
|
||||
'resolutionId', resolution_id,
|
||||
'invocationId', invocation_id,
|
||||
'projectId', project_id,
|
||||
'runId', run_id,
|
||||
'stepRunId', step_run_id,
|
||||
'traceId', trace_id,
|
||||
'completionDigest', completion_digest,
|
||||
'decision', decision,
|
||||
'resolvedByUserId', resolved_by_user_id,
|
||||
'resolvedAtMs', resolved_at_ms,
|
||||
'stepRunMutationId', mutation_id,
|
||||
'stepRunMutationDigest', mutation_digest,
|
||||
'runEventId', run_event_id,
|
||||
'resolutionDigest', resolution_digest
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const postgresMigration = defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_MODEL_INVOCATION_MIGRATION_ID,
|
||||
[
|
||||
POSTGRES_START_TABLE_SQL,
|
||||
`CREATE INDEX ql3_model_invocation_starts_step_history_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_starts"
|
||||
(run_id, step_run_id, admitted_at_ms, invocation_id)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_starts_mutation_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_starts" (mutation_id)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_starts_event_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_starts" (run_event_id)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_starts_digest_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_starts" (start_digest)`,
|
||||
POSTGRES_COMPLETION_TABLE_SQL,
|
||||
`CREATE INDEX ql3_model_invocation_completions_step_history_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_completions"
|
||||
(run_id, step_run_id, completed_at_ms, invocation_id)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_completions_mutation_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_completions" (mutation_id)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_completions_event_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_completions" (run_event_id)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_completions_digest_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_completions" (completion_digest)`,
|
||||
POSTGRES_RESOLUTION_TABLE_SQL,
|
||||
`CREATE INDEX ql3_model_invocation_resolutions_step_history_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_resolutions"
|
||||
(run_id, step_run_id, resolved_at_ms, resolution_id)`,
|
||||
`GRANT USAGE ON SCHEMA "${POSTGRES_MODEL_INVOCATION_SCHEMA}"
|
||||
TO ql3_runtime`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_starts",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_completions",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_resolutions"
|
||||
FROM PUBLIC`,
|
||||
`GRANT SELECT, INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_starts",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_completions",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_resolutions"
|
||||
TO ql3_runtime`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
export const postgresCoreMigrations = Object.freeze([
|
||||
postgresMigration,
|
||||
]);
|
||||
@@ -0,0 +1,513 @@
|
||||
import type { PostgresQueryable } from '@qinglong/runtime-core';
|
||||
|
||||
import {
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_CATALOG_MIGRATION_ID,
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_MIGRATION_ID,
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_MIGRATION_ID,
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_TEST_CONNECTION_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_SCHEMA,
|
||||
} from '../identities';
|
||||
|
||||
import { defineSqlMigration } from '../shared';
|
||||
|
||||
const POSTGRES_MODEL_PROVIDER_CREDENTIAL_BINDING_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_bindings" (
|
||||
project_id varchar(128) NOT NULL,
|
||||
provider varchar(128) NOT NULL,
|
||||
revision varchar(128) NOT NULL,
|
||||
secret_ref varchar(512) NOT NULL,
|
||||
scheme varchar(16) NOT NULL,
|
||||
binding_digest varchar(71) NOT NULL UNIQUE,
|
||||
binding_json jsonb NOT NULL,
|
||||
PRIMARY KEY (project_id, provider, revision),
|
||||
CONSTRAINT ql3_ai_model_provider_credential_binding_project_fk
|
||||
FOREIGN KEY (project_id) REFERENCES "ql3"."projects" (id)
|
||||
ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_model_provider_credential_binding_identity_check CHECK (
|
||||
project_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
provider ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
revision ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
length(secret_ref) BETWEEN 1 AND 512 AND
|
||||
secret_ref LIKE 'qlsecret:v1:%' AND
|
||||
scheme = 'bearer' AND
|
||||
binding_digest ~ '^sha256:[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_model_provider_credential_binding_json_check CHECK (
|
||||
jsonb_typeof(binding_json) = 'object' AND
|
||||
octet_length(binding_json::text) BETWEEN 2 AND 4096 AND
|
||||
binding_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-provider-credential-binding@v1',
|
||||
'projectId', project_id, 'provider', provider,
|
||||
'revision', revision, 'secretRef', secret_ref, 'scheme', scheme
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const POSTGRES_MODEL_PROVIDER_CREDENTIAL_TRANSITION_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_transitions" (
|
||||
project_id varchar(128) NOT NULL,
|
||||
provider varchar(128) NOT NULL,
|
||||
generation integer NOT NULL,
|
||||
action varchar(16) NOT NULL,
|
||||
active_binding_revision varchar(128),
|
||||
active_binding_digest char(64),
|
||||
previous_transition_digest char(64),
|
||||
mutation_id varchar(128) NOT NULL UNIQUE,
|
||||
changed_by_type varchar(32) NOT NULL,
|
||||
changed_by_id varchar(128) NOT NULL,
|
||||
changed_at_ms bigint NOT NULL,
|
||||
command_digest char(64) NOT NULL,
|
||||
transition_digest char(64) NOT NULL UNIQUE,
|
||||
transition_json jsonb NOT NULL,
|
||||
PRIMARY KEY (project_id, provider, generation),
|
||||
CONSTRAINT ql3_ai_model_provider_credential_transition_project_fk
|
||||
FOREIGN KEY (project_id) REFERENCES "ql3"."projects" (id)
|
||||
ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_model_provider_credential_transition_binding_fk
|
||||
FOREIGN KEY (project_id, provider, active_binding_revision)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_bindings"
|
||||
(project_id, provider, revision) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_model_provider_credential_transition_identity_check CHECK (
|
||||
project_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
provider ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
generation BETWEEN 1 AND 2147483647 AND
|
||||
action IN ('bind', 'revoke') AND
|
||||
mutation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
changed_by_type IN ('user', 'api_app', 'mcp_client', 'agent', 'system', 'worker') AND
|
||||
changed_by_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
changed_at_ms >= 0
|
||||
),
|
||||
CONSTRAINT ql3_ai_model_provider_credential_transition_state_check CHECK (
|
||||
(action = 'bind' AND active_binding_revision IS NOT NULL AND
|
||||
active_binding_digest ~ '^[0-9a-f]{64}$') OR
|
||||
(action = 'revoke' AND active_binding_revision IS NULL AND
|
||||
active_binding_digest IS NULL)
|
||||
),
|
||||
CONSTRAINT ql3_ai_model_provider_credential_transition_digest_check CHECK (
|
||||
(previous_transition_digest IS NULL OR
|
||||
previous_transition_digest ~ '^[0-9a-f]{64}$') AND
|
||||
command_digest ~ '^[0-9a-f]{64}$' AND
|
||||
transition_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_model_provider_credential_transition_json_check CHECK (
|
||||
jsonb_typeof(transition_json) = 'object' AND
|
||||
octet_length(transition_json::text) BETWEEN 2 AND 8192 AND
|
||||
transition_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-provider-credential-transition@v1',
|
||||
'mutationId', mutation_id, 'projectId', project_id,
|
||||
'provider', provider, 'generation', generation, 'action', action,
|
||||
'activeBindingRevision', active_binding_revision,
|
||||
'activeBindingDigest', active_binding_digest,
|
||||
'previousTransitionDigest', previous_transition_digest,
|
||||
'changedAtMs', changed_at_ms, 'commandDigest', command_digest,
|
||||
'transitionDigest', transition_digest
|
||||
) AND
|
||||
transition_json -> 'changedBy' @> jsonb_build_object(
|
||||
'type', changed_by_type, 'id', changed_by_id
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const POSTGRES_MODEL_PROVIDER_CREDENTIAL_AUDIT_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_audits" (
|
||||
project_id varchar(128) NOT NULL,
|
||||
provider varchar(128) NOT NULL,
|
||||
request_id varchar(128) NOT NULL,
|
||||
operation varchar(16) NOT NULL,
|
||||
binding_revision varchar(128) NOT NULL,
|
||||
binding_digest varchar(71) NOT NULL,
|
||||
occurred_at_ms bigint NOT NULL,
|
||||
audit_digest char(64) PRIMARY KEY,
|
||||
audit_json jsonb NOT NULL,
|
||||
CONSTRAINT ql3_ai_model_provider_credential_audit_identity_uidx
|
||||
UNIQUE (project_id, provider, request_id, operation),
|
||||
CONSTRAINT ql3_ai_model_provider_credential_audit_project_fk
|
||||
FOREIGN KEY (project_id) REFERENCES "ql3"."projects" (id)
|
||||
ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_model_provider_credential_audit_identity_check CHECK (
|
||||
project_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
provider ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
request_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
operation IN ('list_models', 'generate', 'stream') AND
|
||||
binding_revision ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
binding_digest ~ '^sha256:[0-9a-f]{64}$' AND
|
||||
occurred_at_ms >= 0 AND audit_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_model_provider_credential_audit_json_check CHECK (
|
||||
jsonb_typeof(audit_json) = 'object' AND
|
||||
octet_length(audit_json::text) BETWEEN 2 AND 4096 AND
|
||||
audit_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-provider-credential-audit@v1',
|
||||
'projectId', project_id, 'provider', provider,
|
||||
'requestId', request_id, 'operation', operation,
|
||||
'bindingRevision', binding_revision,
|
||||
'bindingDigest', binding_digest, 'occurredAtMs', occurred_at_ms
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const postgresModelProviderCredentialCatalogMigration =
|
||||
defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_CATALOG_MIGRATION_ID,
|
||||
[
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_BINDING_TABLE_SQL,
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_TRANSITION_TABLE_SQL,
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_AUDIT_TABLE_SQL,
|
||||
`CREATE INDEX ql3_ai_model_provider_credential_transition_current_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_transitions"
|
||||
(project_id, provider, generation DESC)`,
|
||||
`CREATE INDEX ql3_ai_model_provider_credential_audit_project_time_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_audits"
|
||||
(project_id, occurred_at_ms DESC, request_id)`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_bindings",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_transitions",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_audits"
|
||||
FROM PUBLIC`,
|
||||
`GRANT USAGE ON SCHEMA "${POSTGRES_MODEL_INVOCATION_SCHEMA}"
|
||||
TO ql3_runtime, ql3_ai_maintenance`,
|
||||
`GRANT SELECT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_bindings",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_transitions"
|
||||
TO ql3_runtime`,
|
||||
`GRANT SELECT, INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_audits"
|
||||
TO ql3_runtime`,
|
||||
`GRANT SELECT, INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_bindings",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_transitions"
|
||||
TO ql3_ai_maintenance`,
|
||||
`GRANT SELECT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_audits"
|
||||
TO ql3_ai_maintenance`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
const postgresModelProviderCredentialManagementMigration =
|
||||
defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_MIGRATION_ID,
|
||||
[
|
||||
`DO $ql3_ai$
|
||||
BEGIN
|
||||
EXECUTE format(
|
||||
'GRANT CONNECT ON DATABASE %I TO ql3_ai_credential_manager',
|
||||
current_database()
|
||||
);
|
||||
END
|
||||
$ql3_ai$`,
|
||||
`REVOKE CREATE ON SCHEMA "ql3", "${POSTGRES_MODEL_INVOCATION_SCHEMA}"
|
||||
FROM ql3_ai_credential_manager`,
|
||||
`REVOKE ALL ON ALL TABLES IN SCHEMA "${POSTGRES_MODEL_INVOCATION_SCHEMA}"
|
||||
FROM ql3_ai_credential_manager`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_bindings",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_transitions",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_audits"
|
||||
FROM ql3_ai_maintenance`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"ql3"."projects",
|
||||
"ql3"."project_role_bindings",
|
||||
"ql3"."security_audit_events"
|
||||
FROM ql3_ai_credential_manager`,
|
||||
`GRANT USAGE ON SCHEMA "ql3", "${POSTGRES_MODEL_INVOCATION_SCHEMA}"
|
||||
TO ql3_ai_credential_manager`,
|
||||
`GRANT SELECT ON TABLE
|
||||
"ql3"."projects",
|
||||
"ql3"."project_role_bindings",
|
||||
"ql3"."security_audit_events"
|
||||
TO ql3_ai_credential_manager`,
|
||||
`GRANT INSERT ON TABLE "ql3"."security_audit_events"
|
||||
TO ql3_ai_credential_manager`,
|
||||
`GRANT SELECT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."ai_schema_migrations",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_bindings",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_transitions",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_audits"
|
||||
TO ql3_ai_credential_manager`,
|
||||
`GRANT INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_bindings",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_transitions"
|
||||
TO ql3_ai_credential_manager`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
const postgresModelProviderCredentialManagementIdentityMigration =
|
||||
defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_MIGRATION_ID,
|
||||
[
|
||||
`CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_management_identity_keyset_ledger" (
|
||||
authority varchar(64) PRIMARY KEY,
|
||||
generation bigint NOT NULL,
|
||||
digest varchar(43) NOT NULL,
|
||||
issuer varchar(512) NOT NULL,
|
||||
audience varchar(256) NOT NULL,
|
||||
active_key_ids jsonb NOT NULL,
|
||||
revoked_key_ids jsonb NOT NULL,
|
||||
updated_at_ms bigint NOT NULL,
|
||||
CONSTRAINT ql3_ai_provider_credential_identity_authority_check
|
||||
CHECK (authority = 'model-provider-credential-management'),
|
||||
CONSTRAINT ql3_ai_provider_credential_identity_generation_check
|
||||
CHECK (generation >= 1 AND updated_at_ms >= 0),
|
||||
CONSTRAINT ql3_ai_provider_credential_identity_digest_check
|
||||
CHECK (digest ~ '^[A-Za-z0-9_-]{43}$'),
|
||||
CONSTRAINT ql3_ai_provider_credential_identity_trust_domain_check
|
||||
CHECK (
|
||||
char_length(issuer) BETWEEN 1 AND 512 AND
|
||||
issuer !~ '[[:cntrl:]]' AND
|
||||
char_length(audience) BETWEEN 1 AND 256 AND
|
||||
audience !~ '[[:cntrl:]]'
|
||||
),
|
||||
CONSTRAINT ql3_ai_provider_credential_identity_keys_check
|
||||
CHECK (
|
||||
jsonb_typeof(active_key_ids) = 'array' AND
|
||||
jsonb_array_length(active_key_ids) BETWEEN 1 AND 8 AND
|
||||
octet_length(active_key_ids::text) BETWEEN 3 AND 8192 AND
|
||||
jsonb_typeof(revoked_key_ids) = 'array' AND
|
||||
jsonb_array_length(revoked_key_ids) BETWEEN 0 AND 64 AND
|
||||
octet_length(revoked_key_ids::text) BETWEEN 2 AND 16384
|
||||
)
|
||||
)`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_management_identity_keyset_ledger"
|
||||
FROM PUBLIC, ql3_ai_maintenance`,
|
||||
`GRANT SELECT, INSERT, UPDATE ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_management_identity_keyset_ledger"
|
||||
TO ql3_ai_credential_manager`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
const postgresModelProviderCredentialTestConnectionMigration =
|
||||
defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_MODEL_PROVIDER_CREDENTIAL_TEST_CONNECTION_MIGRATION_ID,
|
||||
[
|
||||
`CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_plans" (
|
||||
test_id uuid PRIMARY KEY,
|
||||
request_id varchar(128) NOT NULL,
|
||||
project_id varchar(128) NOT NULL,
|
||||
provider varchar(128) NOT NULL,
|
||||
adapter varchar(32) NOT NULL,
|
||||
base_url varchar(1024) NOT NULL,
|
||||
endpoint_revision varchar(128) NOT NULL,
|
||||
endpoint_config_digest char(64) NOT NULL,
|
||||
deadline_ms integer NOT NULL,
|
||||
max_response_bytes integer NOT NULL,
|
||||
max_models integer NOT NULL,
|
||||
max_cost_microusd bigint NOT NULL,
|
||||
retry_limit integer NOT NULL,
|
||||
requested_by_type varchar(32) NOT NULL,
|
||||
requested_by_id varchar(128) NOT NULL,
|
||||
project_version integer NOT NULL,
|
||||
binding_version integer NOT NULL,
|
||||
planned_at_ms bigint NOT NULL,
|
||||
expires_at_ms bigint NOT NULL,
|
||||
plan_digest char(64) NOT NULL UNIQUE,
|
||||
plan_json jsonb NOT NULL,
|
||||
CONSTRAINT ql3_ai_provider_credential_test_plan_project_fk
|
||||
FOREIGN KEY (project_id) REFERENCES "ql3"."projects" (id)
|
||||
ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_provider_credential_test_plan_identity_uidx
|
||||
UNIQUE (project_id, request_id),
|
||||
CONSTRAINT ql3_ai_provider_credential_test_plan_digest_uidx
|
||||
UNIQUE (test_id, plan_digest),
|
||||
CONSTRAINT ql3_ai_provider_credential_test_plan_identity_check CHECK (
|
||||
project_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
provider ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
request_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND
|
||||
endpoint_revision ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
requested_by_type = 'user' AND
|
||||
char_length(requested_by_id) BETWEEN 1 AND 128 AND
|
||||
requested_by_id !~ '[[:cntrl:]]'
|
||||
),
|
||||
CONSTRAINT ql3_ai_provider_credential_test_plan_budget_check CHECK (
|
||||
adapter = 'openai-compatible' AND
|
||||
base_url ~ '^https://[^[:cntrl:]]+/$' AND
|
||||
endpoint_config_digest ~ '^[0-9a-f]{64}$' AND
|
||||
deadline_ms BETWEEN 1000 AND 15000 AND
|
||||
max_response_bytes BETWEEN 1024 AND 262144 AND
|
||||
max_models BETWEEN 1 AND 256 AND
|
||||
max_cost_microusd = 0 AND retry_limit = 0
|
||||
),
|
||||
CONSTRAINT ql3_ai_provider_credential_test_plan_fence_check CHECK (
|
||||
project_version BETWEEN 1 AND 2147483647 AND
|
||||
binding_version BETWEEN 1 AND 2147483647 AND
|
||||
planned_at_ms >= 0 AND expires_at_ms > planned_at_ms AND
|
||||
expires_at_ms - planned_at_ms <= 300000 AND
|
||||
plan_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_provider_credential_test_plan_json_check CHECK (
|
||||
jsonb_typeof(plan_json) = 'object' AND
|
||||
octet_length(plan_json::text) BETWEEN 2 AND 16384 AND
|
||||
plan_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-provider-credential-test-plan@v1',
|
||||
'testId', test_id::text, 'requestId', request_id,
|
||||
'projectId', project_id, 'provider', provider,
|
||||
'plannedAtMs', planned_at_ms, 'expiresAtMs', expires_at_ms,
|
||||
'planDigest', plan_digest
|
||||
) AND
|
||||
plan_json -> 'endpoint' @> jsonb_build_object(
|
||||
'provider', provider, 'adapter', adapter, 'baseUrl', base_url,
|
||||
'revision', endpoint_revision,
|
||||
'configDigest', endpoint_config_digest,
|
||||
'deadlineMs', deadline_ms,
|
||||
'maxResponseBytes', max_response_bytes,
|
||||
'maxModels', max_models, 'maxCostMicrousd', max_cost_microusd,
|
||||
'retryLimit', retry_limit
|
||||
) AND
|
||||
plan_json -> 'requestedBy' @> jsonb_build_object(
|
||||
'type', requested_by_type, 'id', requested_by_id
|
||||
) AND
|
||||
plan_json -> 'fence' @> jsonb_build_object(
|
||||
'projectVersion', project_version,
|
||||
'bindingVersion', binding_version
|
||||
)
|
||||
)
|
||||
)`,
|
||||
`CREATE INDEX ql3_ai_provider_credential_test_plan_expiry_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_plans"
|
||||
(expires_at_ms, test_id)`,
|
||||
`CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_quota_buckets" (
|
||||
project_id varchar(128) NOT NULL,
|
||||
subject_id varchar(128) NOT NULL,
|
||||
window_started_at_ms bigint NOT NULL,
|
||||
consumed_count integer NOT NULL,
|
||||
receipt_ids jsonb NOT NULL,
|
||||
updated_at_ms bigint NOT NULL,
|
||||
PRIMARY KEY (project_id, subject_id),
|
||||
CONSTRAINT ql3_ai_provider_credential_test_quota_project_fk
|
||||
FOREIGN KEY (project_id) REFERENCES "ql3"."projects" (id)
|
||||
ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_provider_credential_test_quota_identity_check CHECK (
|
||||
project_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
char_length(subject_id) BETWEEN 1 AND 128 AND
|
||||
subject_id !~ '[[:cntrl:]]'
|
||||
),
|
||||
CONSTRAINT ql3_ai_provider_credential_test_quota_value_check CHECK (
|
||||
window_started_at_ms >= 0 AND
|
||||
consumed_count BETWEEN 1 AND 32 AND
|
||||
updated_at_ms >= window_started_at_ms AND
|
||||
jsonb_typeof(receipt_ids) = 'array' AND
|
||||
jsonb_array_length(receipt_ids) BETWEEN 1 AND 32 AND
|
||||
octet_length(receipt_ids::text) BETWEEN 3 AND 8192
|
||||
)
|
||||
)`,
|
||||
`CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_executions" (
|
||||
execution_id uuid PRIMARY KEY,
|
||||
test_id uuid NOT NULL UNIQUE,
|
||||
plan_digest char(64) NOT NULL,
|
||||
started_at_ms bigint NOT NULL,
|
||||
execution_digest char(64) NOT NULL UNIQUE,
|
||||
execution_json jsonb NOT NULL,
|
||||
CONSTRAINT ql3_ai_provider_credential_test_execution_plan_fk
|
||||
FOREIGN KEY (test_id, plan_digest)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_plans"
|
||||
(test_id, plan_digest) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_provider_credential_test_execution_value_check CHECK (
|
||||
plan_digest ~ '^[0-9a-f]{64}$' AND started_at_ms >= 0 AND
|
||||
execution_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_provider_credential_test_execution_json_check CHECK (
|
||||
jsonb_typeof(execution_json) = 'object' AND
|
||||
octet_length(execution_json::text) BETWEEN 2 AND 4096 AND
|
||||
execution_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-provider-credential-test-execution@v1',
|
||||
'executionId', execution_id::text, 'testId', test_id::text,
|
||||
'planDigest', plan_digest, 'startedAtMs', started_at_ms,
|
||||
'executionDigest', execution_digest
|
||||
)
|
||||
)
|
||||
)`,
|
||||
`ALTER TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_executions"
|
||||
ADD CONSTRAINT ql3_ai_provider_credential_test_execution_identity_uidx
|
||||
UNIQUE (execution_id, test_id, plan_digest)`,
|
||||
`CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_results" (
|
||||
execution_id uuid PRIMARY KEY,
|
||||
test_id uuid NOT NULL UNIQUE,
|
||||
plan_digest char(64) NOT NULL,
|
||||
outcome varchar(16) NOT NULL,
|
||||
model_count integer,
|
||||
duration_ms integer NOT NULL,
|
||||
completed_at_ms bigint NOT NULL,
|
||||
result_digest char(64) NOT NULL UNIQUE,
|
||||
result_json jsonb NOT NULL,
|
||||
CONSTRAINT ql3_ai_provider_credential_test_result_execution_fk
|
||||
FOREIGN KEY (execution_id, test_id, plan_digest)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_executions"
|
||||
(execution_id, test_id, plan_digest) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_provider_credential_test_result_value_check CHECK (
|
||||
plan_digest ~ '^[0-9a-f]{64}$' AND
|
||||
((outcome = 'reachable' AND model_count BETWEEN 0 AND 256) OR
|
||||
(outcome = 'unreachable' AND model_count IS NULL)) AND
|
||||
duration_ms BETWEEN 0 AND 15000 AND completed_at_ms >= 0 AND
|
||||
result_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_provider_credential_test_result_json_check CHECK (
|
||||
jsonb_typeof(result_json) = 'object' AND
|
||||
octet_length(result_json::text) BETWEEN 2 AND 4096 AND
|
||||
result_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-provider-credential-test-result@v1',
|
||||
'executionId', execution_id::text, 'testId', test_id::text,
|
||||
'planDigest', plan_digest, 'outcome', outcome,
|
||||
'modelCount', model_count, 'durationMs', duration_ms,
|
||||
'completedAtMs', completed_at_ms, 'resultDigest', result_digest
|
||||
)
|
||||
)
|
||||
)`,
|
||||
`DO $ql3_ai$
|
||||
BEGIN
|
||||
EXECUTE format(
|
||||
'GRANT CONNECT ON DATABASE %I TO ql3_ai_credential_tester',
|
||||
current_database()
|
||||
);
|
||||
END
|
||||
$ql3_ai$`,
|
||||
`REVOKE CREATE ON SCHEMA "ql3", "${POSTGRES_MODEL_INVOCATION_SCHEMA}"
|
||||
FROM ql3_ai_credential_tester`,
|
||||
`REVOKE ALL ON ALL TABLES IN SCHEMA "ql3", "${POSTGRES_MODEL_INVOCATION_SCHEMA}"
|
||||
FROM ql3_ai_credential_tester`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_plans",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_quota_buckets",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_executions",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_results"
|
||||
FROM PUBLIC, ql3_runtime, ql3_ai_maintenance, ql3_admin,
|
||||
ql3_package_manager, ql3_package_executor, ql3_worker_ingress,
|
||||
ql3_ai_credential_manager, ql3_ai_credential_tester`,
|
||||
`GRANT USAGE ON SCHEMA "${POSTGRES_MODEL_INVOCATION_SCHEMA}"
|
||||
TO ql3_ai_credential_tester`,
|
||||
`GRANT SELECT, INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_plans"
|
||||
TO ql3_ai_credential_manager`,
|
||||
`GRANT SELECT, INSERT, UPDATE ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_quota_buckets"
|
||||
TO ql3_ai_credential_manager`,
|
||||
`GRANT SELECT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_executions",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_results"
|
||||
TO ql3_ai_credential_manager`,
|
||||
`GRANT SELECT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."ai_schema_migrations",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_bindings",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_transitions",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_audits",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_plans",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_executions",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_results"
|
||||
TO ql3_ai_credential_tester`,
|
||||
`GRANT INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_audits",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_executions",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_provider_credential_test_results"
|
||||
TO ql3_ai_credential_tester`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
export const postgresCredentialMigrations = Object.freeze([
|
||||
postgresModelProviderCredentialCatalogMigration,
|
||||
postgresModelProviderCredentialManagementMigration,
|
||||
postgresModelProviderCredentialManagementIdentityMigration,
|
||||
postgresModelProviderCredentialTestConnectionMigration,
|
||||
]);
|
||||
@@ -0,0 +1,855 @@
|
||||
import type { PostgresQueryable } from '@qinglong/runtime-core';
|
||||
|
||||
import {
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_ADMISSION_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_FINALIZATION_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_ARTIFACT_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_TOMBSTONE_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_ROTATION_MIGRATION_ID,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_PRODUCT_AUTHORIZATION_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_SCHEMA,
|
||||
POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE,
|
||||
} from '../identities';
|
||||
|
||||
import { defineSqlMigration } from '../shared';
|
||||
|
||||
const POSTGRES_PLUGIN_PACKAGE_PROMPT_ADMISSION_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_admissions" (
|
||||
request_id varchar(128) PRIMARY KEY,
|
||||
invocation_id varchar(128) NOT NULL UNIQUE,
|
||||
plan_digest char(64) NOT NULL UNIQUE,
|
||||
run_id varchar(36) NOT NULL UNIQUE,
|
||||
step_run_id varchar(128) NOT NULL UNIQUE,
|
||||
project_id varchar(128) NOT NULL,
|
||||
package_name varchar(63) NOT NULL,
|
||||
installation_id varchar(128) NOT NULL,
|
||||
lock_digest char(64) NOT NULL,
|
||||
generation integer NOT NULL,
|
||||
generation_digest char(64) NOT NULL,
|
||||
materialized_revision_digest char(64) NOT NULL,
|
||||
publication_digest char(64) NOT NULL,
|
||||
prompt_id varchar(128) NOT NULL,
|
||||
prompt_definition_digest char(64) NOT NULL,
|
||||
parameter_digest char(64) NOT NULL,
|
||||
model_request_digest varchar(71) NOT NULL,
|
||||
admitted_at_ms bigint NOT NULL,
|
||||
receipt_digest char(64) NOT NULL UNIQUE,
|
||||
plan_json jsonb NOT NULL,
|
||||
receipt_json jsonb NOT NULL,
|
||||
CONSTRAINT ql3_ai_prompt_admission_run_fk
|
||||
FOREIGN KEY (run_id) REFERENCES "ql3"."runs" (id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_admission_step_fk
|
||||
FOREIGN KEY (run_id, step_run_id)
|
||||
REFERENCES "ql3"."step_runs" (run_id, id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_admission_publication_fk
|
||||
FOREIGN KEY (publication_digest)
|
||||
REFERENCES "ql3"."plugin_package_automation_publications" (
|
||||
publication_digest
|
||||
) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_admission_identity_check CHECK (
|
||||
request_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
invocation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$' AND
|
||||
step_run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
project_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
package_name ~ '^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$' AND
|
||||
installation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
prompt_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
generation BETWEEN 1 AND 2147483647 AND admitted_at_ms >= 0
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_admission_digest_check CHECK (
|
||||
plan_digest ~ '^[0-9a-f]{64}$' AND
|
||||
lock_digest ~ '^[0-9a-f]{64}$' AND
|
||||
generation_digest ~ '^[0-9a-f]{64}$' AND
|
||||
materialized_revision_digest ~ '^[0-9a-f]{64}$' AND
|
||||
publication_digest ~ '^[0-9a-f]{64}$' AND
|
||||
prompt_definition_digest ~ '^[0-9a-f]{64}$' AND
|
||||
parameter_digest ~ '^[0-9a-f]{64}$' AND
|
||||
model_request_digest ~ '^sha256:[0-9a-f]{64}$' AND
|
||||
receipt_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_admission_json_check CHECK (
|
||||
jsonb_typeof(plan_json) = 'object' AND
|
||||
octet_length(plan_json::text) BETWEEN 2 AND 32768 AND
|
||||
plan_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/plugin-package-prompt-execution-plan@v1',
|
||||
'requestId', request_id, 'invocationId', invocation_id,
|
||||
'planDigest', plan_digest, 'runId', run_id, 'stepRunId', step_run_id,
|
||||
'parameterDigest', parameter_digest,
|
||||
'modelRequestDigest', model_request_digest,
|
||||
'plannedAtMs', admitted_at_ms
|
||||
) AND
|
||||
plan_json -> 'target' @> jsonb_build_object(
|
||||
'projectId', project_id, 'packageName', package_name,
|
||||
'installationId', installation_id, 'lockDigest', lock_digest,
|
||||
'generation', generation, 'generationDigest', generation_digest,
|
||||
'materializedRevisionDigest', materialized_revision_digest,
|
||||
'publicationDigest', publication_digest, 'promptId', prompt_id,
|
||||
'promptDefinitionDigest', prompt_definition_digest
|
||||
) AND
|
||||
jsonb_typeof(receipt_json) = 'object' AND
|
||||
octet_length(receipt_json::text) BETWEEN 2 AND 16384 AND
|
||||
receipt_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/plugin-package-prompt-admission-receipt@v1',
|
||||
'requestId', request_id, 'invocationId', invocation_id,
|
||||
'planDigest', plan_digest, 'runId', run_id, 'stepRunId', step_run_id,
|
||||
'publicationDigest', publication_digest, 'promptId', prompt_id,
|
||||
'admittedAtMs', admitted_at_ms, 'receiptDigest', receipt_digest
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const postgresPluginPackagePromptAdmissionMigration =
|
||||
defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_ADMISSION_MIGRATION_ID,
|
||||
[
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_ADMISSION_TABLE_SQL,
|
||||
`CREATE UNIQUE INDEX ql3_ai_prompt_admission_run_step_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_admissions"
|
||||
(run_id, step_run_id)`,
|
||||
`CREATE INDEX ql3_ai_prompt_admission_target_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_admissions"
|
||||
(project_id, package_name, admitted_at_ms, request_id)`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_admissions"
|
||||
FROM PUBLIC`,
|
||||
`GRANT SELECT, INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_admissions"
|
||||
TO ql3_runtime`,
|
||||
`CREATE FUNCTION
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."plugin_package_prompt_admission_snapshot"(
|
||||
p_project_id varchar,
|
||||
p_package_name varchar,
|
||||
p_publication_digest char(64),
|
||||
p_requested_by_subject_type varchar,
|
||||
p_requested_by_subject_id varchar,
|
||||
p_project_version integer,
|
||||
p_binding_version integer
|
||||
)
|
||||
RETURNS TABLE(publication_json jsonb, revision_json jsonb)
|
||||
LANGUAGE plpgsql
|
||||
VOLATILE
|
||||
SECURITY DEFINER
|
||||
SET search_path = pg_catalog, ql3, ql3_ai
|
||||
AS $ql3_ai$
|
||||
BEGIN
|
||||
IF NOT pg_has_role(session_user, 'ql3_runtime', 'member') THEN
|
||||
RAISE EXCEPTION 'Runtime authority is required'
|
||||
USING ERRCODE = 'insufficient_privilege';
|
||||
END IF;
|
||||
PERFORM 1
|
||||
FROM "ql3"."projects" AS project
|
||||
JOIN "ql3"."project_role_bindings" AS binding
|
||||
ON binding.project_id = project.id
|
||||
AND binding.subject_type = p_requested_by_subject_type
|
||||
AND binding.subject_id = p_requested_by_subject_id
|
||||
AND binding.version = p_binding_version
|
||||
WHERE project.id = p_project_id
|
||||
AND project.status = 'active'
|
||||
AND project.version = p_project_version
|
||||
AND binding.state = 'active'
|
||||
AND binding.version = (
|
||||
SELECT max(candidate.version)
|
||||
FROM "ql3"."project_role_bindings" AS candidate
|
||||
WHERE candidate.project_id = p_project_id
|
||||
AND candidate.subject_type = p_requested_by_subject_type
|
||||
AND candidate.subject_id = p_requested_by_subject_id
|
||||
)
|
||||
FOR SHARE OF project, binding;
|
||||
IF NOT FOUND THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
IF NOT "ql3"."plugin_package_automation_start_allowed"(
|
||||
p_project_id, p_package_name, p_publication_digest
|
||||
) THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
RETURN QUERY
|
||||
SELECT publication.publication_json, revision.revision_json
|
||||
FROM "ql3"."plugin_package_automation_publications" AS publication
|
||||
JOIN "ql3"."plugin_package_materialized_revisions" AS revision
|
||||
ON revision.generation_digest = publication.generation_digest
|
||||
AND revision.revision_digest =
|
||||
publication.materialized_revision_digest
|
||||
WHERE publication.project_id = p_project_id
|
||||
AND publication.package_name = p_package_name
|
||||
AND publication.publication_digest = p_publication_digest
|
||||
AND publication.state = 'active'
|
||||
FOR SHARE OF publication, revision;
|
||||
END
|
||||
$ql3_ai$`,
|
||||
`REVOKE ALL ON FUNCTION
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."plugin_package_prompt_admission_snapshot"(
|
||||
varchar, varchar, char(64), varchar, varchar, integer, integer
|
||||
)
|
||||
FROM PUBLIC, ql3_runtime, ql3_admin, ql3_package_manager,
|
||||
ql3_package_executor, ql3_worker_ingress`,
|
||||
`GRANT EXECUTE ON FUNCTION
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."plugin_package_prompt_admission_snapshot"(
|
||||
varchar, varchar, char(64), varchar, varchar, integer, integer
|
||||
)
|
||||
TO ql3_runtime`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
const POSTGRES_PLUGIN_PACKAGE_PROMPT_FINALIZATION_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_finalizations" (
|
||||
request_id varchar(128) PRIMARY KEY,
|
||||
invocation_id varchar(128) NOT NULL UNIQUE,
|
||||
plan_digest char(64) NOT NULL UNIQUE,
|
||||
run_id varchar(36) NOT NULL UNIQUE,
|
||||
step_run_id varchar(128) NOT NULL UNIQUE,
|
||||
terminal_evidence_kind varchar(16) NOT NULL,
|
||||
terminal_evidence_digest char(64) NOT NULL UNIQUE,
|
||||
final_step_run_digest char(64) NOT NULL,
|
||||
run_status varchar(16) NOT NULL,
|
||||
event_id varchar(36) NOT NULL UNIQUE,
|
||||
final_run_version integer NOT NULL,
|
||||
final_run_event_sequence integer NOT NULL,
|
||||
finalized_at_ms bigint NOT NULL,
|
||||
receipt_digest char(64) NOT NULL UNIQUE,
|
||||
receipt_json jsonb NOT NULL,
|
||||
CONSTRAINT ql3_ai_prompt_finalization_admission_fk
|
||||
FOREIGN KEY (request_id)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_admissions"
|
||||
(request_id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_finalization_invocation_fk
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_admissions"
|
||||
(invocation_id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_finalization_plan_fk
|
||||
FOREIGN KEY (plan_digest)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_admissions"
|
||||
(plan_digest) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_finalization_run_fk
|
||||
FOREIGN KEY (run_id) REFERENCES "ql3"."runs" (id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_finalization_step_fk
|
||||
FOREIGN KEY (run_id, step_run_id)
|
||||
REFERENCES "ql3"."step_runs" (run_id, id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_finalization_event_fk
|
||||
FOREIGN KEY (event_id) REFERENCES "ql3"."run_events" (id)
|
||||
ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_finalization_identity_check CHECK (
|
||||
request_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
invocation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$' AND
|
||||
step_run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
event_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$' AND
|
||||
terminal_evidence_kind IN ('completion', 'resolution') AND
|
||||
run_status IN ('succeeded', 'failed', 'cancelled', 'timed_out') AND
|
||||
final_run_version BETWEEN 3 AND 2147483647 AND
|
||||
final_run_event_sequence = final_run_version AND finalized_at_ms >= 0
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_finalization_digest_check CHECK (
|
||||
plan_digest ~ '^[0-9a-f]{64}$' AND
|
||||
terminal_evidence_digest ~ '^[0-9a-f]{64}$' AND
|
||||
final_step_run_digest ~ '^[0-9a-f]{64}$' AND
|
||||
receipt_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_finalization_json_check CHECK (
|
||||
jsonb_typeof(receipt_json) = 'object' AND
|
||||
octet_length(receipt_json::text) BETWEEN 2 AND 16384 AND
|
||||
receipt_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/plugin-package-prompt-finalization-receipt@v1',
|
||||
'requestId', request_id, 'invocationId', invocation_id,
|
||||
'planDigest', plan_digest, 'runId', run_id, 'stepRunId', step_run_id,
|
||||
'terminalEvidenceKind', terminal_evidence_kind,
|
||||
'terminalEvidenceDigest', terminal_evidence_digest,
|
||||
'finalStepRunDigest', final_step_run_digest, 'runStatus', run_status,
|
||||
'eventId', event_id, 'finalRunVersion', final_run_version,
|
||||
'finalRunEventSequence', final_run_event_sequence,
|
||||
'finalizedAtMs', finalized_at_ms, 'receiptDigest', receipt_digest
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const postgresPluginPackagePromptFinalizationMigration =
|
||||
defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_FINALIZATION_MIGRATION_ID,
|
||||
[
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_FINALIZATION_TABLE_SQL,
|
||||
`CREATE UNIQUE INDEX ql3_ai_prompt_finalization_run_step_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_finalizations"
|
||||
(run_id, step_run_id)`,
|
||||
`CREATE INDEX ql3_ai_prompt_finalization_status_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_finalizations"
|
||||
(run_status, finalized_at_ms, request_id)`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_finalizations"
|
||||
FROM PUBLIC`,
|
||||
`GRANT SELECT, INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_finalizations"
|
||||
TO ql3_runtime`,
|
||||
`GRANT SELECT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE}"
|
||||
TO ql3_runtime`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
const POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_ARTIFACT_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_artifacts" (
|
||||
artifact_id varchar(128) PRIMARY KEY,
|
||||
project_id varchar(128) NOT NULL,
|
||||
run_id varchar(36) NOT NULL,
|
||||
step_run_id varchar(128) NOT NULL,
|
||||
invocation_id varchar(128) NOT NULL UNIQUE,
|
||||
requested_by_type varchar(32) NOT NULL,
|
||||
requested_by_id varchar(128) NOT NULL,
|
||||
provider varchar(256) NOT NULL,
|
||||
model varchar(256) NOT NULL,
|
||||
content_digest char(64) NOT NULL,
|
||||
output_bytes integer NOT NULL,
|
||||
retention_policy_revision varchar(128) NOT NULL,
|
||||
retention_ms bigint NOT NULL,
|
||||
retention_policy_digest char(64) NOT NULL,
|
||||
retention_eligible_at_ms bigint NOT NULL,
|
||||
key_id varchar(128) NOT NULL,
|
||||
algorithm varchar(16) NOT NULL,
|
||||
plaintext_bytes integer NOT NULL,
|
||||
sealed_at_ms bigint NOT NULL,
|
||||
artifact_digest char(64) NOT NULL UNIQUE,
|
||||
artifact_json jsonb NOT NULL,
|
||||
CONSTRAINT ql3_ai_prompt_output_artifact_admission_fk
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_admissions"
|
||||
(invocation_id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_output_artifact_start_fk
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_starts"
|
||||
(invocation_id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_output_artifact_step_fk
|
||||
FOREIGN KEY (run_id, step_run_id)
|
||||
REFERENCES "ql3"."step_runs" (run_id, id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_output_artifact_identity_check CHECK (
|
||||
artifact_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
project_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$' AND
|
||||
step_run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
invocation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
length(requested_by_type) BETWEEN 1 AND 32 AND
|
||||
requested_by_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
length(provider) BETWEEN 1 AND 256 AND
|
||||
length(model) BETWEEN 1 AND 256 AND
|
||||
retention_policy_revision ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
key_id ~ '^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$' AND
|
||||
algorithm = 'aes-256-gcm'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_output_artifact_value_check CHECK (
|
||||
output_bytes BETWEEN 0 AND 1048576 AND
|
||||
plaintext_bytes BETWEEN 1 AND 1052672 AND
|
||||
retention_ms BETWEEN 3600000 AND 31536000000 AND
|
||||
sealed_at_ms >= 0 AND
|
||||
retention_eligible_at_ms = sealed_at_ms + retention_ms AND
|
||||
content_digest ~ '^[0-9a-f]{64}$' AND
|
||||
retention_policy_digest ~ '^[0-9a-f]{64}$' AND
|
||||
artifact_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_output_artifact_json_check CHECK (
|
||||
jsonb_typeof(artifact_json) = 'object' AND
|
||||
octet_length(artifact_json::text) BETWEEN 2 AND 1572864 AND
|
||||
artifact_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/plugin-package-prompt-output-artifact@v1',
|
||||
'artifactId', artifact_id, 'projectId', project_id,
|
||||
'runId', run_id, 'stepRunId', step_run_id,
|
||||
'invocationId', invocation_id, 'provider', provider, 'model', model,
|
||||
'contentDigest', content_digest, 'outputBytes', output_bytes,
|
||||
'retentionPolicyDigest', retention_policy_digest,
|
||||
'retentionEligibleAtMs', retention_eligible_at_ms,
|
||||
'keyId', key_id, 'algorithm', algorithm,
|
||||
'plaintextBytes', plaintext_bytes, 'sealedAtMs', sealed_at_ms,
|
||||
'artifactDigest', artifact_digest
|
||||
) AND
|
||||
artifact_json -> 'requestedBy' @> jsonb_build_object(
|
||||
'type', requested_by_type, 'id', requested_by_id
|
||||
) AND
|
||||
artifact_json -> 'retentionPolicy' @> jsonb_build_object(
|
||||
'revision', retention_policy_revision, 'retentionMs', retention_ms
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const postgresPluginPackagePromptOutputArtifactMigration =
|
||||
defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_ARTIFACT_MIGRATION_ID,
|
||||
[
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_ARTIFACT_TABLE_SQL,
|
||||
`CREATE INDEX ql3_ai_prompt_output_artifact_retention_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_artifacts"
|
||||
(retention_eligible_at_ms, artifact_id)`,
|
||||
`CREATE INDEX ql3_ai_prompt_output_artifact_project_run_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_artifacts"
|
||||
(project_id, run_id, artifact_id)`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_artifacts"
|
||||
FROM PUBLIC`,
|
||||
`GRANT SELECT, INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_artifacts"
|
||||
TO ql3_runtime`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
const POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_TOMBSTONE_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_artifact_tombstones" (
|
||||
artifact_id varchar(128) PRIMARY KEY,
|
||||
project_id varchar(128) NOT NULL,
|
||||
run_id varchar(36) NOT NULL,
|
||||
step_run_id varchar(128) NOT NULL,
|
||||
invocation_id varchar(128) NOT NULL UNIQUE,
|
||||
artifact_digest char(64) NOT NULL UNIQUE,
|
||||
retention_policy_digest char(64) NOT NULL,
|
||||
retention_eligible_at_ms bigint NOT NULL,
|
||||
key_id varchar(128) NOT NULL,
|
||||
tombstoned_at_ms bigint NOT NULL,
|
||||
tombstone_digest char(64) NOT NULL UNIQUE,
|
||||
tombstone_json jsonb NOT NULL,
|
||||
CONSTRAINT ql3_ai_prompt_output_tombstone_admission_fk
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_admissions"
|
||||
(invocation_id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_output_tombstone_start_fk
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_starts"
|
||||
(invocation_id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_output_tombstone_step_fk
|
||||
FOREIGN KEY (run_id, step_run_id)
|
||||
REFERENCES "ql3"."step_runs" (run_id, id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_output_tombstone_identity_check CHECK (
|
||||
artifact_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
project_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$' AND
|
||||
step_run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
invocation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
key_id ~ '^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_output_tombstone_value_check CHECK (
|
||||
retention_eligible_at_ms >= 0 AND
|
||||
tombstoned_at_ms >= retention_eligible_at_ms AND
|
||||
artifact_digest ~ '^[0-9a-f]{64}$' AND
|
||||
retention_policy_digest ~ '^[0-9a-f]{64}$' AND
|
||||
tombstone_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_output_tombstone_json_check CHECK (
|
||||
jsonb_typeof(tombstone_json) = 'object' AND
|
||||
octet_length(tombstone_json::text) BETWEEN 2 AND 8192 AND
|
||||
tombstone_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/plugin-package-prompt-output-artifact-tombstone@v1',
|
||||
'tombstonedAtMs', tombstoned_at_ms,
|
||||
'tombstoneDigest', tombstone_digest
|
||||
) AND
|
||||
tombstone_json -> 'reference' @> jsonb_build_object(
|
||||
'artifactId', artifact_id, 'projectId', project_id,
|
||||
'runId', run_id, 'stepRunId', step_run_id,
|
||||
'invocationId', invocation_id, 'artifactDigest', artifact_digest,
|
||||
'retentionPolicyDigest', retention_policy_digest,
|
||||
'retentionEligibleAtMs', retention_eligible_at_ms, 'keyId', key_id
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const postgresPluginPackagePromptOutputTombstoneMigration =
|
||||
defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_TOMBSTONE_MIGRATION_ID,
|
||||
[
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_TOMBSTONE_TABLE_SQL,
|
||||
`CREATE INDEX ql3_ai_prompt_output_tombstone_time_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_artifact_tombstones"
|
||||
(tombstoned_at_ms, artifact_id)`,
|
||||
`CREATE INDEX ql3_ai_prompt_output_tombstone_project_run_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_artifact_tombstones"
|
||||
(project_id, run_id, artifact_id)`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_artifact_tombstones"
|
||||
FROM PUBLIC`,
|
||||
`DO $$
|
||||
BEGIN
|
||||
EXECUTE format(
|
||||
'GRANT CONNECT ON DATABASE %I TO ql3_ai_maintenance',
|
||||
current_database()
|
||||
);
|
||||
END
|
||||
$$`,
|
||||
`GRANT SELECT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_artifact_tombstones"
|
||||
TO ql3_runtime`,
|
||||
`GRANT USAGE ON SCHEMA "${POSTGRES_MODEL_INVOCATION_SCHEMA}"
|
||||
TO ql3_ai_maintenance`,
|
||||
`GRANT SELECT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE}",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_completions",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_admissions",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_finalizations"
|
||||
TO ql3_ai_maintenance`,
|
||||
`GRANT SELECT, DELETE ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_artifacts"
|
||||
TO ql3_ai_maintenance`,
|
||||
`GRANT SELECT, INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_artifact_tombstones"
|
||||
TO ql3_ai_maintenance`,
|
||||
`GRANT USAGE ON SCHEMA "ql3" TO ql3_ai_maintenance`,
|
||||
`GRANT SELECT ON TABLE "ql3"."runs", "ql3"."step_runs"
|
||||
TO ql3_ai_maintenance`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
const POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_PREPARATION_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_key_retirement_preparations" (
|
||||
key_id varchar(128) PRIMARY KEY,
|
||||
retirement_id varchar(128) NOT NULL UNIQUE,
|
||||
request_id varchar(128) NOT NULL UNIQUE,
|
||||
mutation_id varchar(128) NOT NULL UNIQUE,
|
||||
catalog_digest char(64) NOT NULL,
|
||||
material_proof char(64) NOT NULL,
|
||||
prepared_at_ms bigint NOT NULL,
|
||||
preparation_digest char(64) NOT NULL UNIQUE,
|
||||
preparation_json jsonb NOT NULL,
|
||||
CONSTRAINT ql3_ai_prompt_output_key_retirement_preparation_fk_uidx
|
||||
UNIQUE (key_id, preparation_digest),
|
||||
CONSTRAINT ql3_ai_prompt_output_key_retirement_preparation_identity_check CHECK (
|
||||
key_id ~ '^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$' AND
|
||||
retirement_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
request_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
mutation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_output_key_retirement_preparation_value_check CHECK (
|
||||
prepared_at_ms >= 0 AND
|
||||
catalog_digest ~ '^[0-9a-f]{64}$' AND
|
||||
material_proof ~ '^[0-9a-f]{64}$' AND
|
||||
preparation_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_output_key_retirement_preparation_json_check CHECK (
|
||||
jsonb_typeof(preparation_json) = 'object' AND
|
||||
octet_length(preparation_json::text) BETWEEN 2 AND 8192 AND
|
||||
preparation_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/plugin-package-prompt-output-key-retirement-preparation@v1',
|
||||
'keyId', key_id, 'retirementId', retirement_id,
|
||||
'requestId', request_id, 'mutationId', mutation_id,
|
||||
'catalogDigest', catalog_digest, 'materialProof', material_proof,
|
||||
'preparedAtMs', prepared_at_ms,
|
||||
'preparationDigest', preparation_digest
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_COMPLETION_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_key_retirement_completions" (
|
||||
key_id varchar(128) PRIMARY KEY,
|
||||
retirement_id varchar(128) NOT NULL UNIQUE,
|
||||
request_id varchar(128) NOT NULL UNIQUE,
|
||||
mutation_id varchar(128) NOT NULL UNIQUE,
|
||||
preparation_digest char(64) NOT NULL UNIQUE,
|
||||
retired_catalog_digest char(64) NOT NULL,
|
||||
absence_proof char(64) NOT NULL,
|
||||
completed_at_ms bigint NOT NULL,
|
||||
completion_digest char(64) NOT NULL UNIQUE,
|
||||
completion_json jsonb NOT NULL,
|
||||
CONSTRAINT ql3_ai_prompt_output_key_retirement_completion_preparation_fk
|
||||
FOREIGN KEY (key_id, preparation_digest)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_key_retirement_preparations"
|
||||
(key_id, preparation_digest) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_output_key_retirement_completion_identity_check CHECK (
|
||||
key_id ~ '^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$' AND
|
||||
retirement_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
request_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
mutation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_output_key_retirement_completion_value_check CHECK (
|
||||
completed_at_ms >= 0 AND
|
||||
preparation_digest ~ '^[0-9a-f]{64}$' AND
|
||||
retired_catalog_digest ~ '^[0-9a-f]{64}$' AND
|
||||
absence_proof ~ '^[0-9a-f]{64}$' AND
|
||||
completion_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_output_key_retirement_completion_json_check CHECK (
|
||||
jsonb_typeof(completion_json) = 'object' AND
|
||||
octet_length(completion_json::text) BETWEEN 2 AND 8192 AND
|
||||
completion_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/plugin-package-prompt-output-key-retirement-completion@v1',
|
||||
'keyId', key_id, 'retirementId', retirement_id,
|
||||
'requestId', request_id, 'mutationId', mutation_id,
|
||||
'preparationDigest', preparation_digest,
|
||||
'retiredCatalogDigest', retired_catalog_digest,
|
||||
'absenceProof', absence_proof, 'completedAtMs', completed_at_ms,
|
||||
'completionDigest', completion_digest
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const postgresPluginPackagePromptOutputKeyRetirementMigration =
|
||||
defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_MIGRATION_ID,
|
||||
[
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_PREPARATION_TABLE_SQL,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_COMPLETION_TABLE_SQL,
|
||||
`CREATE INDEX ql3_ai_prompt_output_key_retirement_completion_time_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_key_retirement_completions"
|
||||
(completed_at_ms, key_id)`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_key_retirement_preparations",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_key_retirement_completions"
|
||||
FROM PUBLIC`,
|
||||
`GRANT SELECT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_key_retirement_preparations"
|
||||
TO ql3_runtime`,
|
||||
`GRANT SELECT, INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_key_retirement_preparations",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_key_retirement_completions"
|
||||
TO ql3_ai_maintenance`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
const POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_ROTATION_PREPARATION_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_key_rotation_preparations" (
|
||||
rotation_id varchar(128) PRIMARY KEY,
|
||||
request_id varchar(128) NOT NULL UNIQUE,
|
||||
mutation_id varchar(128) NOT NULL UNIQUE,
|
||||
expected_secret_uid varchar(128) NOT NULL,
|
||||
expected_active_key_id varchar(128) NOT NULL,
|
||||
expected_catalog_digest char(64) NOT NULL,
|
||||
new_key_id varchar(128) NOT NULL UNIQUE,
|
||||
material_proof char(64) NOT NULL,
|
||||
prepared_at_ms bigint NOT NULL,
|
||||
preparation_digest char(64) NOT NULL UNIQUE,
|
||||
preparation_json jsonb NOT NULL,
|
||||
CONSTRAINT ql3_ai_prompt_key_rotation_preparation_fk_uidx
|
||||
UNIQUE (rotation_id, preparation_digest),
|
||||
CONSTRAINT ql3_ai_prompt_key_rotation_source_uidx
|
||||
UNIQUE (expected_secret_uid, expected_catalog_digest),
|
||||
CONSTRAINT ql3_ai_prompt_key_rotation_preparation_identity_check CHECK (
|
||||
rotation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
request_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
mutation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
expected_secret_uid ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND
|
||||
expected_active_key_id ~ '^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$' AND
|
||||
new_key_id ~ '^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$' AND
|
||||
expected_active_key_id <> new_key_id
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_key_rotation_preparation_value_check CHECK (
|
||||
prepared_at_ms >= 0 AND
|
||||
expected_catalog_digest ~ '^[0-9a-f]{64}$' AND
|
||||
material_proof ~ '^[0-9a-f]{64}$' AND
|
||||
preparation_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_key_rotation_preparation_json_check CHECK (
|
||||
jsonb_typeof(preparation_json) = 'object' AND
|
||||
octet_length(preparation_json::text) BETWEEN 2 AND 8192 AND
|
||||
preparation_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/plugin-package-prompt-output-key-rotation-preparation@v1',
|
||||
'rotationId', rotation_id, 'requestId', request_id,
|
||||
'mutationId', mutation_id, 'expectedSecretUid', expected_secret_uid,
|
||||
'expectedActiveKeyId', expected_active_key_id,
|
||||
'expectedCatalogDigest', expected_catalog_digest,
|
||||
'newKeyId', new_key_id, 'materialProof', material_proof,
|
||||
'preparedAtMs', prepared_at_ms, 'preparationDigest', preparation_digest
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_ROTATION_COMPLETION_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_key_rotation_completions" (
|
||||
rotation_id varchar(128) PRIMARY KEY,
|
||||
request_id varchar(128) NOT NULL UNIQUE,
|
||||
mutation_id varchar(128) NOT NULL UNIQUE,
|
||||
preparation_digest char(64) NOT NULL UNIQUE,
|
||||
generation bigint NOT NULL,
|
||||
previous_active_key_id varchar(128) NOT NULL,
|
||||
active_key_id varchar(128) NOT NULL,
|
||||
catalog_digest char(64) NOT NULL,
|
||||
material_proof char(64) NOT NULL,
|
||||
completed_at_ms bigint NOT NULL,
|
||||
completion_digest char(64) NOT NULL UNIQUE,
|
||||
completion_json jsonb NOT NULL,
|
||||
CONSTRAINT ql3_ai_prompt_key_rotation_completion_preparation_fk
|
||||
FOREIGN KEY (rotation_id, preparation_digest)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_key_rotation_preparations"
|
||||
(rotation_id, preparation_digest) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_key_rotation_completion_identity_check CHECK (
|
||||
rotation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
request_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
mutation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
previous_active_key_id ~ '^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$' AND
|
||||
active_key_id ~ '^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$' AND
|
||||
previous_active_key_id <> active_key_id
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_key_rotation_completion_value_check CHECK (
|
||||
generation BETWEEN 2 AND 9007199254740991 AND completed_at_ms >= 0 AND
|
||||
preparation_digest ~ '^[0-9a-f]{64}$' AND
|
||||
catalog_digest ~ '^[0-9a-f]{64}$' AND
|
||||
material_proof ~ '^[0-9a-f]{64}$' AND
|
||||
completion_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_key_rotation_completion_json_check CHECK (
|
||||
jsonb_typeof(completion_json) = 'object' AND
|
||||
octet_length(completion_json::text) BETWEEN 2 AND 8192 AND
|
||||
completion_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/plugin-package-prompt-output-key-rotation-completion@v1',
|
||||
'rotationId', rotation_id, 'requestId', request_id,
|
||||
'mutationId', mutation_id, 'preparationDigest', preparation_digest,
|
||||
'generation', generation, 'previousActiveKeyId', previous_active_key_id,
|
||||
'activeKeyId', active_key_id, 'catalogDigest', catalog_digest,
|
||||
'materialProof', material_proof, 'completedAtMs', completed_at_ms,
|
||||
'completionDigest', completion_digest
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const postgresPluginPackagePromptOutputKeyRotationMigration =
|
||||
defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_ROTATION_MIGRATION_ID,
|
||||
[
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_ROTATION_PREPARATION_TABLE_SQL,
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_ROTATION_COMPLETION_TABLE_SQL,
|
||||
`CREATE INDEX ql3_ai_prompt_key_rotation_completion_time_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_key_rotation_completions"
|
||||
(completed_at_ms, rotation_id)`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_key_rotation_preparations",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_key_rotation_completions"
|
||||
FROM PUBLIC, ql3_runtime, ql3_admin, ql3_package_manager,
|
||||
ql3_package_executor, ql3_automation_manager, ql3_worker_ingress,
|
||||
ql3_worker_credential_manager, ql3_worker_credential_executor,
|
||||
ql3_ai_credential_manager, ql3_ai_credential_tester`,
|
||||
`GRANT SELECT, INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_key_rotation_preparations",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_prompt_output_key_rotation_completions"
|
||||
TO ql3_ai_maintenance`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
const postgresPluginPackagePromptProductAuthorizationMigration =
|
||||
defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_PRODUCT_AUTHORIZATION_MIGRATION_ID,
|
||||
[
|
||||
`CREATE FUNCTION
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."plugin_package_prompt_authorize_admission"(
|
||||
p_credential_id varchar,
|
||||
p_credential_version integer,
|
||||
p_project_id varchar,
|
||||
p_subject_type varchar,
|
||||
p_subject_id varchar,
|
||||
p_project_version integer,
|
||||
p_binding_version integer,
|
||||
p_audit_event_id uuid,
|
||||
p_request_id varchar,
|
||||
p_planned_at_ms bigint,
|
||||
p_replay boolean
|
||||
)
|
||||
RETURNS boolean
|
||||
LANGUAGE plpgsql
|
||||
VOLATILE
|
||||
SECURITY DEFINER
|
||||
SET search_path = pg_catalog, ql3, ql3_ai
|
||||
AS $ql3_ai$
|
||||
BEGIN
|
||||
IF NOT pg_has_role(session_user, 'ql3_runtime', 'member') THEN
|
||||
RAISE EXCEPTION 'Runtime authority is required'
|
||||
USING ERRCODE = 'insufficient_privilege';
|
||||
END IF;
|
||||
PERFORM pg_advisory_xact_lock(
|
||||
hashtextextended('ql3-api-credential:' || p_credential_id, 0)
|
||||
);
|
||||
PERFORM pg_advisory_xact_lock(
|
||||
hashtextextended(
|
||||
'ql3-identity:' || p_subject_type || ':' || p_subject_id,
|
||||
0
|
||||
)
|
||||
);
|
||||
PERFORM 1
|
||||
FROM "ql3"."api_credentials" AS credential
|
||||
JOIN "ql3"."identity_subjects" AS identity
|
||||
ON identity.subject_type = credential.subject_type
|
||||
AND identity.subject_id = credential.subject_id
|
||||
WHERE credential.credential_id = p_credential_id
|
||||
AND credential.version = p_credential_version
|
||||
AND credential.version = (
|
||||
SELECT max(candidate.version)
|
||||
FROM "ql3"."api_credentials" AS candidate
|
||||
WHERE candidate.credential_id = p_credential_id
|
||||
)
|
||||
AND credential.state = 'active'
|
||||
AND credential.subject_type = p_subject_type
|
||||
AND credential.subject_id = p_subject_id
|
||||
AND credential.not_before_at_ms <=
|
||||
floor(extract(epoch FROM clock_timestamp()) * 1000)::bigint
|
||||
AND credential.expires_at_ms >
|
||||
floor(extract(epoch FROM clock_timestamp()) * 1000)::bigint
|
||||
AND identity.status = 'active'
|
||||
FOR SHARE OF credential, identity;
|
||||
IF NOT FOUND THEN
|
||||
RETURN false;
|
||||
END IF;
|
||||
PERFORM 1
|
||||
FROM "ql3"."projects" AS project
|
||||
JOIN "ql3"."project_role_bindings" AS binding
|
||||
ON binding.project_id = project.id
|
||||
AND binding.subject_type = p_subject_type
|
||||
AND binding.subject_id = p_subject_id
|
||||
AND binding.version = p_binding_version
|
||||
WHERE project.id = p_project_id
|
||||
AND project.status = 'active'
|
||||
AND project.version = p_project_version
|
||||
AND binding.state = 'active'
|
||||
AND binding.role IN ('owner', 'admin', 'operator')
|
||||
AND p_subject_type <> 'agent'
|
||||
AND binding.version = (
|
||||
SELECT max(candidate.version)
|
||||
FROM "ql3"."project_role_bindings" AS candidate
|
||||
WHERE candidate.project_id = p_project_id
|
||||
AND candidate.subject_type = p_subject_type
|
||||
AND candidate.subject_id = p_subject_id
|
||||
)
|
||||
FOR SHARE OF project, binding;
|
||||
IF NOT FOUND THEN
|
||||
RETURN false;
|
||||
END IF;
|
||||
IF NOT p_replay THEN
|
||||
INSERT INTO "ql3"."security_audit_events" (
|
||||
event_id, request_id, operation_id, project_id,
|
||||
subject_type, subject_id, authentication_id, outcome, reasons,
|
||||
project_version, binding_version, occurred_at_ms
|
||||
) VALUES (
|
||||
p_audit_event_id, p_request_id, 'prompt.execute', p_project_id,
|
||||
p_subject_type, p_subject_id,
|
||||
'api_credential:' || p_credential_id || ':' || p_credential_version,
|
||||
'allowed', '["project_policy_allowed"]'::jsonb,
|
||||
p_project_version, p_binding_version, p_planned_at_ms
|
||||
);
|
||||
END IF;
|
||||
RETURN true;
|
||||
END
|
||||
$ql3_ai$`,
|
||||
`REVOKE ALL ON FUNCTION
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."plugin_package_prompt_authorize_admission"(
|
||||
varchar, integer, varchar, varchar, varchar, integer, integer,
|
||||
uuid, varchar, bigint, boolean
|
||||
)
|
||||
FROM PUBLIC, ql3_runtime, ql3_admin, ql3_package_manager,
|
||||
ql3_package_executor, ql3_worker_ingress`,
|
||||
`GRANT EXECUTE ON FUNCTION
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."plugin_package_prompt_authorize_admission"(
|
||||
varchar, integer, varchar, varchar, varchar, integer, integer,
|
||||
uuid, varchar, bigint, boolean
|
||||
)
|
||||
TO ql3_runtime`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
export const postgresPromptBaseMigrations = Object.freeze([
|
||||
postgresPluginPackagePromptAdmissionMigration,
|
||||
postgresPluginPackagePromptFinalizationMigration,
|
||||
postgresPluginPackagePromptOutputArtifactMigration,
|
||||
postgresPluginPackagePromptOutputTombstoneMigration,
|
||||
postgresPluginPackagePromptOutputKeyRetirementMigration,
|
||||
]);
|
||||
|
||||
export const postgresPromptExtensionMigrations = Object.freeze([
|
||||
postgresPluginPackagePromptOutputKeyRotationMigration,
|
||||
postgresPluginPackagePromptProductAuthorizationMigration,
|
||||
]);
|
||||
@@ -0,0 +1,377 @@
|
||||
import type { PostgresQueryable } from '@qinglong/runtime-core';
|
||||
|
||||
import {
|
||||
POSTGRES_MODEL_INVOCATION_USAGE_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_QUOTA_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_PRICING_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_SCHEMA,
|
||||
} from '../identities';
|
||||
|
||||
import { defineSqlMigration } from '../shared';
|
||||
|
||||
const POSTGRES_USAGE_LEDGER_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_usage_ledger" (
|
||||
invocation_id varchar(128) PRIMARY KEY,
|
||||
project_id varchar(128) NOT NULL,
|
||||
run_id varchar(36) NOT NULL,
|
||||
step_run_id varchar(128) NOT NULL,
|
||||
trace_id varchar(128) NOT NULL,
|
||||
provider varchar(128) NOT NULL,
|
||||
model varchar(128) NOT NULL,
|
||||
policy_revision varchar(128) NOT NULL,
|
||||
completion_digest char(64) NOT NULL,
|
||||
outcome varchar(32) NOT NULL,
|
||||
settled_at_ms bigint NOT NULL,
|
||||
input_bytes integer NOT NULL,
|
||||
output_bytes integer NOT NULL,
|
||||
input_tokens bigint NOT NULL,
|
||||
output_tokens bigint NOT NULL,
|
||||
total_tokens bigint NOT NULL,
|
||||
cost_micros bigint,
|
||||
ledger_digest char(64) NOT NULL,
|
||||
record_json jsonb NOT NULL,
|
||||
FOREIGN KEY (invocation_id, completion_digest)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_completions"
|
||||
(invocation_id, completion_digest) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_invocation_usage_identity_check CHECK (
|
||||
invocation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
project_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND
|
||||
step_run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
trace_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
provider ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
model ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
policy_revision ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_usage_value_check CHECK (
|
||||
outcome IN ('succeeded', 'failed', 'timed_out', 'outcome_unknown') AND
|
||||
settled_at_ms >= 0 AND
|
||||
input_bytes BETWEEN 1 AND 262144 AND
|
||||
output_bytes BETWEEN 0 AND 1048576 AND
|
||||
input_tokens >= 0 AND output_tokens >= 0 AND
|
||||
total_tokens = input_tokens + output_tokens AND
|
||||
(cost_micros IS NULL OR cost_micros >= 0) AND
|
||||
completion_digest ~ '^[0-9a-f]{64}$' AND
|
||||
ledger_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_usage_json_check CHECK (
|
||||
jsonb_typeof(record_json) = 'object' AND
|
||||
octet_length(record_json::text) BETWEEN 2 AND 24576 AND
|
||||
record_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-invocation-usage-ledger@v1',
|
||||
'invocationId', invocation_id,
|
||||
'projectId', project_id,
|
||||
'runId', run_id,
|
||||
'stepRunId', step_run_id,
|
||||
'traceId', trace_id,
|
||||
'provider', provider,
|
||||
'model', model,
|
||||
'policyRevision', policy_revision,
|
||||
'completionDigest', completion_digest,
|
||||
'outcome', outcome,
|
||||
'settledAtMs', settled_at_ms,
|
||||
'inputBytes', input_bytes,
|
||||
'outputBytes', output_bytes,
|
||||
'inputTokens', input_tokens,
|
||||
'outputTokens', output_tokens,
|
||||
'totalTokens', total_tokens,
|
||||
'costMicros', cost_micros,
|
||||
'ledgerDigest', ledger_digest
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const postgresUsageMigration = defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_MODEL_INVOCATION_USAGE_MIGRATION_ID,
|
||||
[
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_completions_identity_digest_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_completions"
|
||||
(invocation_id, completion_digest)`,
|
||||
POSTGRES_USAGE_LEDGER_TABLE_SQL,
|
||||
`CREATE INDEX ql3_model_invocation_usage_project_time_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_usage_ledger"
|
||||
(project_id, settled_at_ms, invocation_id)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_usage_digest_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_usage_ledger"
|
||||
(ledger_digest)`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_usage_ledger"
|
||||
FROM PUBLIC`,
|
||||
`GRANT SELECT, INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_usage_ledger"
|
||||
TO ql3_runtime`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
const POSTGRES_QUOTA_RESERVATION_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_quota_reservations" (
|
||||
invocation_id varchar(128) PRIMARY KEY,
|
||||
project_id varchar(128) NOT NULL,
|
||||
model_policy_revision varchar(128) NOT NULL,
|
||||
quota_policy_revision varchar(128) NOT NULL,
|
||||
window_ms bigint NOT NULL,
|
||||
window_start_ms bigint NOT NULL,
|
||||
window_end_ms bigint NOT NULL,
|
||||
max_invocations integer NOT NULL,
|
||||
max_tokens bigint NOT NULL,
|
||||
max_cost_micros bigint,
|
||||
reserved_tokens bigint NOT NULL,
|
||||
reserved_cost_micros bigint,
|
||||
reserved_at_ms bigint NOT NULL,
|
||||
admission_digest char(64) NOT NULL,
|
||||
reservation_digest char(64) NOT NULL,
|
||||
record_json jsonb NOT NULL,
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_starts"
|
||||
(invocation_id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_invocation_quota_reservation_value_check CHECK (
|
||||
window_ms IN (60000, 3600000, 86400000) AND
|
||||
window_start_ms >= 0 AND window_end_ms = window_start_ms + window_ms AND
|
||||
reserved_at_ms >= window_start_ms AND reserved_at_ms < window_end_ms AND
|
||||
max_invocations BETWEEN 1 AND 100000 AND
|
||||
max_tokens BETWEEN 1 AND 1000000000000 AND
|
||||
(max_cost_micros IS NULL OR
|
||||
max_cost_micros BETWEEN 0 AND 1000000000000000) AND
|
||||
reserved_tokens BETWEEN 1 AND 1000000000000 AND
|
||||
(reserved_cost_micros IS NULL OR
|
||||
reserved_cost_micros BETWEEN 0 AND 1000000000000000) AND
|
||||
(max_cost_micros IS NULL OR reserved_cost_micros IS NOT NULL) AND
|
||||
admission_digest ~ '^[0-9a-f]{64}$' AND
|
||||
reservation_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_quota_reservation_json_check CHECK (
|
||||
jsonb_typeof(record_json) = 'object' AND
|
||||
octet_length(record_json::text) BETWEEN 2 AND 24576 AND
|
||||
record_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-invocation-quota-reservation@v1',
|
||||
'invocationId', invocation_id,
|
||||
'projectId', project_id,
|
||||
'modelPolicyRevision', model_policy_revision,
|
||||
'quotaPolicyRevision', quota_policy_revision,
|
||||
'windowMs', window_ms,
|
||||
'windowStartMs', window_start_ms,
|
||||
'windowEndMs', window_end_ms,
|
||||
'maxInvocations', max_invocations,
|
||||
'maxTokens', max_tokens,
|
||||
'maxCostMicros', max_cost_micros,
|
||||
'reservedTokens', reserved_tokens,
|
||||
'reservedCostMicros', reserved_cost_micros,
|
||||
'reservedAtMs', reserved_at_ms,
|
||||
'admissionDigest', admission_digest,
|
||||
'reservationDigest', reservation_digest
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const POSTGRES_QUOTA_SETTLEMENT_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_quota_settlements" (
|
||||
invocation_id varchar(128) PRIMARY KEY,
|
||||
project_id varchar(128) NOT NULL,
|
||||
reservation_digest char(64) NOT NULL,
|
||||
completion_digest char(64) NOT NULL,
|
||||
effective_tokens bigint NOT NULL,
|
||||
effective_cost_micros bigint,
|
||||
retained_token_reservation boolean NOT NULL,
|
||||
retained_cost_reservation boolean NOT NULL,
|
||||
settled_at_ms bigint NOT NULL,
|
||||
settlement_digest char(64) NOT NULL,
|
||||
record_json jsonb NOT NULL,
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_quota_reservations"
|
||||
(invocation_id) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (invocation_id, completion_digest)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_completions"
|
||||
(invocation_id, completion_digest) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_invocation_quota_settlement_value_check CHECK (
|
||||
effective_tokens BETWEEN 0 AND 1000000000000 AND
|
||||
(effective_cost_micros IS NULL OR
|
||||
effective_cost_micros BETWEEN 0 AND 1000000000000000) AND
|
||||
settled_at_ms >= 0 AND
|
||||
reservation_digest ~ '^[0-9a-f]{64}$' AND
|
||||
completion_digest ~ '^[0-9a-f]{64}$' AND
|
||||
settlement_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_quota_settlement_json_check CHECK (
|
||||
jsonb_typeof(record_json) = 'object' AND
|
||||
octet_length(record_json::text) BETWEEN 2 AND 24576 AND
|
||||
record_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-invocation-quota-settlement@v1',
|
||||
'invocationId', invocation_id,
|
||||
'projectId', project_id,
|
||||
'reservationDigest', reservation_digest,
|
||||
'completionDigest', completion_digest,
|
||||
'effectiveTokens', effective_tokens,
|
||||
'effectiveCostMicros', effective_cost_micros,
|
||||
'retainedTokenReservation', retained_token_reservation,
|
||||
'retainedCostReservation', retained_cost_reservation,
|
||||
'settledAtMs', settled_at_ms,
|
||||
'settlementDigest', settlement_digest
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const postgresQuotaMigration = defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_MODEL_INVOCATION_QUOTA_MIGRATION_ID,
|
||||
[
|
||||
POSTGRES_QUOTA_RESERVATION_TABLE_SQL,
|
||||
`CREATE INDEX ql3_model_invocation_quota_reservation_window_idx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_quota_reservations"
|
||||
(project_id, window_start_ms, invocation_id)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_quota_reservation_digest_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_quota_reservations"
|
||||
(reservation_digest)`,
|
||||
POSTGRES_QUOTA_SETTLEMENT_TABLE_SQL,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_quota_settlement_digest_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_quota_settlements"
|
||||
(settlement_digest)`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_quota_reservations",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_quota_settlements"
|
||||
FROM PUBLIC`,
|
||||
`GRANT SELECT, INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_quota_reservations",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_quota_settlements"
|
||||
TO ql3_runtime`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
const POSTGRES_PRICE_QUOTE_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_price_quotes" (
|
||||
invocation_id varchar(128) PRIMARY KEY,
|
||||
project_id varchar(128) NOT NULL,
|
||||
model_policy_revision varchar(128) NOT NULL,
|
||||
provider varchar(128) NOT NULL,
|
||||
model varchar(128) NOT NULL,
|
||||
price_revision varchar(128) NOT NULL,
|
||||
currency varchar(3) NOT NULL,
|
||||
input_micros_per_million_tokens bigint NOT NULL,
|
||||
output_micros_per_million_tokens bigint NOT NULL,
|
||||
max_total_tokens bigint NOT NULL,
|
||||
max_output_tokens bigint NOT NULL,
|
||||
reserved_cost_micros bigint NOT NULL,
|
||||
catalog_digest char(64) NOT NULL,
|
||||
quote_digest char(64) NOT NULL,
|
||||
record_json jsonb NOT NULL,
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_starts"
|
||||
(invocation_id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_invocation_price_quote_identity_check CHECK (
|
||||
project_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
model_policy_revision ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
provider ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
model ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
price_revision ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_price_quote_value_check CHECK (
|
||||
currency = 'USD' AND
|
||||
input_micros_per_million_tokens BETWEEN 0 AND 1000000000000 AND
|
||||
output_micros_per_million_tokens BETWEEN 0 AND 1000000000000 AND
|
||||
max_total_tokens BETWEEN 1 AND 1000000000000 AND
|
||||
max_output_tokens BETWEEN 1 AND max_total_tokens AND
|
||||
reserved_cost_micros BETWEEN 0 AND 1000000000000000 AND
|
||||
catalog_digest ~ '^[0-9a-f]{64}$' AND
|
||||
quote_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_price_quote_json_check CHECK (
|
||||
jsonb_typeof(record_json) = 'object' AND
|
||||
octet_length(record_json::text) BETWEEN 2 AND 24576 AND
|
||||
record_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-invocation-price-quote@v1',
|
||||
'invocationId', invocation_id,
|
||||
'projectId', project_id,
|
||||
'modelPolicyRevision', model_policy_revision,
|
||||
'provider', provider,
|
||||
'model', model,
|
||||
'priceRevision', price_revision,
|
||||
'currency', currency,
|
||||
'inputMicrosPerMillionTokens', input_micros_per_million_tokens,
|
||||
'outputMicrosPerMillionTokens', output_micros_per_million_tokens,
|
||||
'maxTotalTokens', max_total_tokens,
|
||||
'maxOutputTokens', max_output_tokens,
|
||||
'reservedCostMicros', reserved_cost_micros,
|
||||
'catalogDigest', catalog_digest,
|
||||
'quoteDigest', quote_digest
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const POSTGRES_PRICE_SETTLEMENT_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_price_settlements" (
|
||||
invocation_id varchar(128) PRIMARY KEY,
|
||||
project_id varchar(128) NOT NULL,
|
||||
quote_digest char(64) NOT NULL,
|
||||
completion_digest char(64) NOT NULL,
|
||||
currency varchar(3) NOT NULL,
|
||||
input_tokens bigint NOT NULL,
|
||||
output_tokens bigint NOT NULL,
|
||||
cost_micros bigint NOT NULL,
|
||||
settled_at_ms bigint NOT NULL,
|
||||
settlement_digest char(64) NOT NULL,
|
||||
record_json jsonb NOT NULL,
|
||||
FOREIGN KEY (invocation_id, quote_digest)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_price_quotes"
|
||||
(invocation_id, quote_digest) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (invocation_id, completion_digest)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_completions"
|
||||
(invocation_id, completion_digest) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_invocation_price_settlement_value_check CHECK (
|
||||
currency = 'USD' AND
|
||||
input_tokens >= 0 AND output_tokens >= 0 AND
|
||||
cost_micros BETWEEN 0 AND 1000000000000000 AND
|
||||
settled_at_ms >= 0 AND
|
||||
quote_digest ~ '^[0-9a-f]{64}$' AND
|
||||
completion_digest ~ '^[0-9a-f]{64}$' AND
|
||||
settlement_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_price_settlement_json_check CHECK (
|
||||
jsonb_typeof(record_json) = 'object' AND
|
||||
octet_length(record_json::text) BETWEEN 2 AND 24576 AND
|
||||
record_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/model-invocation-price-settlement@v1',
|
||||
'invocationId', invocation_id,
|
||||
'projectId', project_id,
|
||||
'quoteDigest', quote_digest,
|
||||
'completionDigest', completion_digest,
|
||||
'currency', currency,
|
||||
'inputTokens', input_tokens,
|
||||
'outputTokens', output_tokens,
|
||||
'costMicros', cost_micros,
|
||||
'settledAtMs', settled_at_ms,
|
||||
'settlementDigest', settlement_digest
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const postgresPricingMigration = defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_MODEL_INVOCATION_PRICING_MIGRATION_ID,
|
||||
[
|
||||
POSTGRES_PRICE_QUOTE_TABLE_SQL,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_price_quote_identity_digest_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_price_quotes"
|
||||
(invocation_id, quote_digest)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_price_quote_digest_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_price_quotes"
|
||||
(quote_digest)`,
|
||||
POSTGRES_PRICE_SETTLEMENT_TABLE_SQL,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_price_settlement_digest_uidx
|
||||
ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_price_settlements"
|
||||
(settlement_digest)`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_price_quotes",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_price_settlements"
|
||||
FROM PUBLIC`,
|
||||
`GRANT SELECT, INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_price_quotes",
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_price_settlements"
|
||||
TO ql3_runtime`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
export const postgresUsagePricingMigrations = Object.freeze([
|
||||
postgresUsageMigration,
|
||||
postgresQuotaMigration,
|
||||
postgresPricingMigration,
|
||||
]);
|
||||
@@ -0,0 +1,94 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
import {
|
||||
MigrationStreamHistoryCorruptionError,
|
||||
type MigrationStreamRecord,
|
||||
type MigrationStreamStep,
|
||||
} from '@qinglong/runtime-core/migration-stream';
|
||||
|
||||
export interface HistoryRow extends Record<string, unknown> {
|
||||
readonly migrationId: unknown;
|
||||
readonly streamId: unknown;
|
||||
readonly dialect: unknown;
|
||||
readonly checksum: unknown;
|
||||
readonly appliedAtMs: unknown;
|
||||
}
|
||||
|
||||
function checksum(statements: readonly string[]): string {
|
||||
return createHash('sha256')
|
||||
.update(
|
||||
JSON.stringify({
|
||||
format: 1,
|
||||
statements,
|
||||
}),
|
||||
)
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
export function defineSqlMigration<TContext>(
|
||||
id: string,
|
||||
statementsValue: readonly string[],
|
||||
execute: (context: TContext, statement: string) => void | Promise<void>,
|
||||
): MigrationStreamStep<TContext> {
|
||||
const statements = Object.freeze(
|
||||
statementsValue.map((statement) => statement.trim()),
|
||||
);
|
||||
if (
|
||||
statements.length < 1 ||
|
||||
statements.length > 32 ||
|
||||
statements.some(
|
||||
(statement) => statement.length < 1 || statement.length > 64 * 1024,
|
||||
)
|
||||
) {
|
||||
throw new TypeError('ModelInvocation migration statements are invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
id,
|
||||
checksum: checksum(statements),
|
||||
async up(context: TContext): Promise<void> {
|
||||
for (const statement of statements) await execute(context, statement);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function appliedAtMs(value: unknown, migrationId: string): number {
|
||||
if (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) {
|
||||
return value;
|
||||
}
|
||||
if (typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value)) {
|
||||
const parsed = Number(value);
|
||||
if (Number.isSafeInteger(parsed)) return parsed;
|
||||
}
|
||||
throw new MigrationStreamHistoryCorruptionError(migrationId);
|
||||
}
|
||||
|
||||
export function historyRecord(
|
||||
row: HistoryRow,
|
||||
expected: Readonly<{
|
||||
migrationIds: readonly string[];
|
||||
streamId: string;
|
||||
dialect: 'sqlite' | 'postgresql';
|
||||
}>,
|
||||
): MigrationStreamRecord {
|
||||
const corruptionId =
|
||||
typeof row.migrationId === 'string'
|
||||
? row.migrationId
|
||||
: expected.migrationIds[0] ?? expected.streamId;
|
||||
if (
|
||||
typeof row.migrationId !== 'string' ||
|
||||
!expected.migrationIds.includes(row.migrationId) ||
|
||||
row.streamId !== expected.streamId ||
|
||||
row.dialect !== expected.dialect ||
|
||||
typeof row.checksum !== 'string' ||
|
||||
!/^[0-9a-f]{64}$/.test(row.checksum)
|
||||
) {
|
||||
throw new MigrationStreamHistoryCorruptionError(corruptionId);
|
||||
}
|
||||
return Object.freeze({
|
||||
migrationId: row.migrationId,
|
||||
streamId: row.streamId,
|
||||
dialect: expected.dialect,
|
||||
checksum: row.checksum,
|
||||
appliedAtMs: appliedAtMs(row.appliedAtMs, row.migrationId),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
import { Buffer } from 'node:buffer';
|
||||
import { createHash } from 'node:crypto';
|
||||
import type { DatabaseSync } from 'node:sqlite';
|
||||
|
||||
import {
|
||||
MigrationStreamHistoryCorruptionError,
|
||||
auditMigrationStreamHistory,
|
||||
runMigrationStream,
|
||||
type MigrationStreamDefinition,
|
||||
type MigrationStreamRecord,
|
||||
type MigrationStreamStore,
|
||||
type MigrationStreamTransaction,
|
||||
} from '@qinglong/runtime-core/migration-stream';
|
||||
|
||||
import {
|
||||
LOCAL_MODEL_INVOCATION_MIGRATION_STREAM_ID,
|
||||
LOCAL_MODEL_INVOCATION_MIGRATION_ID,
|
||||
LOCAL_MODEL_INVOCATION_USAGE_MIGRATION_ID,
|
||||
LOCAL_MODEL_INVOCATION_QUOTA_MIGRATION_ID,
|
||||
LOCAL_MODEL_INVOCATION_PRICING_MIGRATION_ID,
|
||||
LOCAL_MODEL_PRICE_CATALOG_MIGRATION_ID,
|
||||
LOCAL_MODEL_PRICE_CATALOG_AUTHORIZATION_MIGRATION_ID,
|
||||
LOCAL_MODEL_INVOCATION_FEATURE_ACTIVATION_MIGRATION_ID,
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_ADMISSION_MIGRATION_ID,
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_FINALIZATION_MIGRATION_ID,
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_ARTIFACT_MIGRATION_ID,
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_TOMBSTONE_MIGRATION_ID,
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_MIGRATION_ID,
|
||||
LOCAL_MODEL_PROVIDER_CREDENTIAL_CATALOG_MIGRATION_ID,
|
||||
LOCAL_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE,
|
||||
} from './identities';
|
||||
|
||||
import { historyRecord, type HistoryRow } from './shared';
|
||||
import { sqliteCatalogMigrations } from './sqlite/catalog';
|
||||
import type { LocalMigrationContext } from './sqlite/context';
|
||||
import { sqliteCoreMigrations } from './sqlite/core';
|
||||
import { sqliteCredentialMigrations } from './sqlite/credential';
|
||||
import { sqlitePromptMigrations } from './sqlite/prompt';
|
||||
import { sqliteUsagePricingMigrations } from './sqlite/usagePricing';
|
||||
|
||||
const LOCAL_HISTORY_IDENTITY = Object.freeze({
|
||||
migrationIds: Object.freeze([
|
||||
LOCAL_MODEL_INVOCATION_MIGRATION_ID,
|
||||
LOCAL_MODEL_INVOCATION_USAGE_MIGRATION_ID,
|
||||
LOCAL_MODEL_INVOCATION_QUOTA_MIGRATION_ID,
|
||||
LOCAL_MODEL_INVOCATION_PRICING_MIGRATION_ID,
|
||||
LOCAL_MODEL_PRICE_CATALOG_MIGRATION_ID,
|
||||
LOCAL_MODEL_PRICE_CATALOG_AUTHORIZATION_MIGRATION_ID,
|
||||
LOCAL_MODEL_INVOCATION_FEATURE_ACTIVATION_MIGRATION_ID,
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_ADMISSION_MIGRATION_ID,
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_FINALIZATION_MIGRATION_ID,
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_ARTIFACT_MIGRATION_ID,
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_TOMBSTONE_MIGRATION_ID,
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_MIGRATION_ID,
|
||||
LOCAL_MODEL_PROVIDER_CREDENTIAL_CATALOG_MIGRATION_ID,
|
||||
]),
|
||||
streamId: LOCAL_MODEL_INVOCATION_MIGRATION_STREAM_ID,
|
||||
dialect: 'sqlite' as const,
|
||||
});
|
||||
|
||||
function readLocalHistory(
|
||||
client: DatabaseSync,
|
||||
migrationId?: string,
|
||||
): readonly MigrationStreamRecord[] {
|
||||
const rows = client
|
||||
.prepare(
|
||||
`SELECT
|
||||
migration_id AS "migrationId",
|
||||
stream_id AS "streamId",
|
||||
dialect,
|
||||
checksum,
|
||||
applied_at_ms AS "appliedAtMs"
|
||||
FROM "${LOCAL_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE}"
|
||||
WHERE stream_id = ?
|
||||
${migrationId === undefined ? '' : 'AND migration_id = ?'}
|
||||
ORDER BY migration_id`,
|
||||
)
|
||||
.all(
|
||||
LOCAL_MODEL_INVOCATION_MIGRATION_STREAM_ID,
|
||||
...(migrationId === undefined ? [] : [migrationId]),
|
||||
) as unknown as HistoryRow[];
|
||||
return rows.map((row) => historyRecord(row, LOCAL_HISTORY_IDENTITY));
|
||||
}
|
||||
|
||||
class LocalModelInvocationMigrationStore
|
||||
implements MigrationStreamStore<LocalMigrationContext>
|
||||
{
|
||||
constructor(private readonly client: DatabaseSync) {}
|
||||
|
||||
async ensureHistory(): Promise<void> {
|
||||
const required = [
|
||||
'QingLong3SchemaMigrations',
|
||||
'Runs',
|
||||
'RunEvents',
|
||||
'StepRuns',
|
||||
'StepRunMutations',
|
||||
];
|
||||
const rows = this.client
|
||||
.prepare(
|
||||
`SELECT name FROM sqlite_schema
|
||||
WHERE type = 'table' AND name IN (${required
|
||||
.map(() => '?')
|
||||
.join(', ')})`,
|
||||
)
|
||||
.all(...required) as unknown as { name?: unknown }[];
|
||||
const names = new Set(
|
||||
rows
|
||||
.map((row) => row.name)
|
||||
.filter((name): name is string => typeof name === 'string'),
|
||||
);
|
||||
if (required.some((name) => !names.has(name))) {
|
||||
throw new TypeError(
|
||||
'Local ModelInvocation feature requires the main SQLite migration stream',
|
||||
);
|
||||
}
|
||||
this.client.exec(`
|
||||
CREATE TABLE IF NOT EXISTS "${LOCAL_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE}" (
|
||||
migration_id TEXT PRIMARY KEY,
|
||||
stream_id TEXT NOT NULL
|
||||
CONSTRAINT ql3_ai_migrations_stream_check
|
||||
CHECK (stream_id = '${LOCAL_MODEL_INVOCATION_MIGRATION_STREAM_ID}'),
|
||||
dialect TEXT NOT NULL
|
||||
CONSTRAINT ql3_ai_migrations_dialect_check CHECK (dialect = 'sqlite'),
|
||||
checksum TEXT NOT NULL
|
||||
CONSTRAINT ql3_ai_migrations_checksum_check
|
||||
CHECK (length(checksum) = 64 AND checksum NOT GLOB '*[^0-9a-f]*'),
|
||||
applied_at_ms INTEGER NOT NULL
|
||||
CONSTRAINT ql3_ai_migrations_applied_at_check CHECK (applied_at_ms >= 0)
|
||||
)
|
||||
`);
|
||||
}
|
||||
|
||||
async listAll(): Promise<readonly MigrationStreamRecord[]> {
|
||||
return readLocalHistory(this.client);
|
||||
}
|
||||
|
||||
async findById(migrationId: string): Promise<MigrationStreamRecord | null> {
|
||||
const rows = readLocalHistory(this.client, migrationId);
|
||||
if (rows.length > 1) {
|
||||
throw new MigrationStreamHistoryCorruptionError(migrationId);
|
||||
}
|
||||
return rows[0] ?? null;
|
||||
}
|
||||
|
||||
async transaction<T>(
|
||||
work: (
|
||||
transaction: MigrationStreamTransaction<LocalMigrationContext>,
|
||||
) => Promise<T>,
|
||||
): Promise<T> {
|
||||
this.client.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
const result = await work({
|
||||
context: { client: this.client },
|
||||
findById: (migrationId) => this.findById(migrationId),
|
||||
insert: async (record) => {
|
||||
if (
|
||||
record.streamId !== LOCAL_MODEL_INVOCATION_MIGRATION_STREAM_ID ||
|
||||
!LOCAL_HISTORY_IDENTITY.migrationIds.includes(record.migrationId) ||
|
||||
record.dialect !== 'sqlite'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Local ModelInvocation migration identity is invalid',
|
||||
);
|
||||
}
|
||||
this.client
|
||||
.prepare(
|
||||
`INSERT INTO "${LOCAL_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE}"
|
||||
(migration_id, stream_id, dialect, checksum, applied_at_ms)
|
||||
VALUES (?, ?, ?, ?, ?)`,
|
||||
)
|
||||
.run(
|
||||
record.migrationId,
|
||||
record.streamId,
|
||||
record.dialect,
|
||||
record.checksum,
|
||||
record.appliedAtMs,
|
||||
);
|
||||
},
|
||||
});
|
||||
this.client.exec('COMMIT');
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (this.client.isTransaction) this.client.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const localModelInvocationMigrationDefinition: MigrationStreamDefinition<LocalMigrationContext> =
|
||||
Object.freeze({
|
||||
id: LOCAL_MODEL_INVOCATION_MIGRATION_STREAM_ID,
|
||||
dialect: 'sqlite',
|
||||
migrationIdScheme: 'sqlite-numbered',
|
||||
checksumScheme: 'sha256',
|
||||
migrations: Object.freeze([
|
||||
...sqliteCoreMigrations,
|
||||
...sqliteUsagePricingMigrations,
|
||||
...sqliteCatalogMigrations,
|
||||
...sqlitePromptMigrations,
|
||||
...sqliteCredentialMigrations,
|
||||
]),
|
||||
});
|
||||
|
||||
export const LOCAL_MODEL_INVOCATION_MIGRATION_PLAN_DIGEST = createHash('sha256')
|
||||
.update(
|
||||
Buffer.from('qinglong/local-model-invocation-migration-plan@v1\0', 'utf8'),
|
||||
)
|
||||
.update(
|
||||
JSON.stringify(
|
||||
localModelInvocationMigrationDefinition.migrations.map(
|
||||
({ id, checksum: migrationChecksum }) => ({
|
||||
id,
|
||||
checksum: migrationChecksum,
|
||||
}),
|
||||
),
|
||||
),
|
||||
'utf8',
|
||||
)
|
||||
.digest('hex');
|
||||
|
||||
export class LocalModelInvocationFeatureNotReadyError extends Error {
|
||||
readonly code = 'LOCAL_MODEL_INVOCATION_FEATURE_NOT_READY';
|
||||
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('The local ModelInvocation feature schema is not ready', options);
|
||||
this.name = 'LocalModelInvocationFeatureNotReadyError';
|
||||
}
|
||||
}
|
||||
|
||||
const LOCAL_MODEL_INVOCATION_FEATURE_TABLES = Object.freeze([
|
||||
'ModelInvocationCompletions',
|
||||
'ModelInvocationFeatureHead',
|
||||
'ModelInvocationFeatureTransitions',
|
||||
'ModelInvocationPriceQuotes',
|
||||
'ModelInvocationPriceSettlements',
|
||||
'ModelInvocationPromptAdmissions',
|
||||
'ModelInvocationPromptFinalizations',
|
||||
'ModelInvocationPromptOutputArtifactTombstones',
|
||||
'ModelInvocationPromptOutputArtifacts',
|
||||
'ModelInvocationPromptOutputKeyRetirementCompletions',
|
||||
'ModelInvocationPromptOutputKeyRetirementPreparations',
|
||||
'ModelInvocationProviderCredentialAudits',
|
||||
'ModelInvocationProviderCredentialBindings',
|
||||
'ModelInvocationProviderCredentialTransitions',
|
||||
'ModelInvocationQuotaReservations',
|
||||
'ModelInvocationQuotaSettlements',
|
||||
'ModelInvocationResolutions',
|
||||
'ModelInvocationStarts',
|
||||
'ModelInvocationUsageLedger',
|
||||
'ModelPriceCatalogAuthorizations',
|
||||
'ModelPriceCatalogHeads',
|
||||
'ModelPriceCatalogPublications',
|
||||
LOCAL_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE,
|
||||
]);
|
||||
|
||||
export function assertLocalModelInvocationFeatureReady(
|
||||
client: DatabaseSync,
|
||||
): void {
|
||||
try {
|
||||
if (
|
||||
!client ||
|
||||
typeof client !== 'object' ||
|
||||
!client.isOpen ||
|
||||
typeof client.prepare !== 'function'
|
||||
) {
|
||||
throw new LocalModelInvocationFeatureNotReadyError();
|
||||
}
|
||||
const tables = (
|
||||
client
|
||||
.prepare(
|
||||
`SELECT name
|
||||
FROM sqlite_schema
|
||||
WHERE type = 'table'
|
||||
AND (
|
||||
name = ?
|
||||
OR name LIKE 'ModelInvocation%'
|
||||
OR name LIKE 'ModelPriceCatalog%'
|
||||
)
|
||||
ORDER BY name`,
|
||||
)
|
||||
.all(LOCAL_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE) as {
|
||||
readonly name?: unknown;
|
||||
}[]
|
||||
).map(({ name }) => name);
|
||||
if (
|
||||
tables.length !== LOCAL_MODEL_INVOCATION_FEATURE_TABLES.length ||
|
||||
tables.some(
|
||||
(table, index) =>
|
||||
table !== LOCAL_MODEL_INVOCATION_FEATURE_TABLES[index],
|
||||
)
|
||||
) {
|
||||
throw new LocalModelInvocationFeatureNotReadyError();
|
||||
}
|
||||
const applied = auditMigrationStreamHistory(
|
||||
readLocalHistory(client),
|
||||
localModelInvocationMigrationDefinition,
|
||||
);
|
||||
if (
|
||||
applied.size !== localModelInvocationMigrationDefinition.migrations.length
|
||||
) {
|
||||
throw new LocalModelInvocationFeatureNotReadyError();
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof LocalModelInvocationFeatureNotReadyError) {
|
||||
throw error;
|
||||
}
|
||||
throw new LocalModelInvocationFeatureNotReadyError({
|
||||
cause: error instanceof Error ? error : undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function migrateLocalModelInvocationFeature(
|
||||
client: DatabaseSync,
|
||||
): Promise<void> {
|
||||
await runMigrationStream({
|
||||
stream: localModelInvocationMigrationDefinition,
|
||||
store: new LocalModelInvocationMigrationStore(client),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,486 @@
|
||||
import type { LocalMigrationContext } from './context';
|
||||
|
||||
import {
|
||||
LOCAL_MODEL_PRICE_CATALOG_MIGRATION_ID,
|
||||
LOCAL_MODEL_PRICE_CATALOG_AUTHORIZATION_MIGRATION_ID,
|
||||
LOCAL_MODEL_INVOCATION_FEATURE_ACTIVATION_MIGRATION_ID,
|
||||
} from '../identities';
|
||||
|
||||
import { defineSqlMigration } from '../shared';
|
||||
|
||||
const LOCAL_MODEL_PRICE_CATALOG_PUBLICATION_TABLE_SQL = `
|
||||
CREATE TABLE "ModelPriceCatalogPublications" (
|
||||
provider TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
price_revision TEXT NOT NULL,
|
||||
catalog_digest TEXT NOT NULL,
|
||||
mutation_id TEXT NOT NULL UNIQUE,
|
||||
command_digest TEXT NOT NULL,
|
||||
publication_digest TEXT NOT NULL,
|
||||
published_at_ms INTEGER NOT NULL,
|
||||
published_by_user_id TEXT NOT NULL,
|
||||
publication_json TEXT NOT NULL,
|
||||
PRIMARY KEY (provider, model, price_revision),
|
||||
CONSTRAINT ql3_model_price_catalog_publication_identity_check CHECK (
|
||||
length(provider) BETWEEN 1 AND 128 AND
|
||||
length(model) BETWEEN 1 AND 128 AND
|
||||
length(price_revision) BETWEEN 1 AND 128 AND
|
||||
length(mutation_id) BETWEEN 1 AND 128 AND
|
||||
length(published_by_user_id) BETWEEN 1 AND 128
|
||||
),
|
||||
CONSTRAINT ql3_model_price_catalog_publication_value_check CHECK (
|
||||
published_at_ms >= 0 AND
|
||||
length(catalog_digest) = 64 AND
|
||||
catalog_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(command_digest) = 64 AND
|
||||
command_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(publication_digest) = 64 AND
|
||||
publication_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_model_price_catalog_publication_json_check CHECK (
|
||||
length(CAST(publication_json AS BLOB)) BETWEEN 2 AND 24576 AND
|
||||
json_valid(publication_json) AND
|
||||
json_type(publication_json) = 'object' AND
|
||||
json_extract(publication_json, '$.schema') =
|
||||
'qinglong/model-price-catalog-publication@v1' AND
|
||||
json_extract(publication_json, '$.entry.schema') =
|
||||
'qinglong/model-price-catalog-entry@v1' AND
|
||||
json_extract(publication_json, '$.entry.provider') = provider AND
|
||||
json_extract(publication_json, '$.entry.model') = model AND
|
||||
json_extract(publication_json, '$.entry.priceRevision') =
|
||||
price_revision AND
|
||||
json_extract(publication_json, '$.entry.catalogDigest') =
|
||||
catalog_digest AND
|
||||
json_extract(publication_json, '$.entry.publishedAtMs') =
|
||||
published_at_ms AND
|
||||
json_extract(publication_json, '$.mutationId') = mutation_id AND
|
||||
json_extract(publication_json, '$.publishedByUserId') =
|
||||
published_by_user_id AND
|
||||
json_extract(publication_json, '$.commandDigest') = command_digest AND
|
||||
json_extract(publication_json, '$.publicationDigest') =
|
||||
publication_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const LOCAL_MODEL_PRICE_CATALOG_HEAD_TABLE_SQL = `
|
||||
CREATE TABLE "ModelPriceCatalogHeads" (
|
||||
provider TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
generation INTEGER NOT NULL,
|
||||
previous_head_digest TEXT,
|
||||
active_price_revision TEXT,
|
||||
active_catalog_digest TEXT,
|
||||
revoked_price_revision TEXT,
|
||||
revoked_catalog_digest TEXT,
|
||||
action TEXT NOT NULL,
|
||||
mutation_id TEXT NOT NULL UNIQUE,
|
||||
changed_by_user_id TEXT NOT NULL,
|
||||
changed_at_ms INTEGER NOT NULL,
|
||||
command_digest TEXT NOT NULL,
|
||||
head_digest TEXT NOT NULL UNIQUE,
|
||||
head_json TEXT NOT NULL,
|
||||
PRIMARY KEY (provider, model, generation),
|
||||
FOREIGN KEY (previous_head_digest)
|
||||
REFERENCES "ModelPriceCatalogHeads" (head_digest) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (
|
||||
provider, model, active_price_revision, active_catalog_digest
|
||||
) REFERENCES "ModelPriceCatalogPublications" (
|
||||
provider, model, price_revision, catalog_digest
|
||||
) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (
|
||||
provider, model, revoked_price_revision, revoked_catalog_digest
|
||||
) REFERENCES "ModelPriceCatalogPublications" (
|
||||
provider, model, price_revision, catalog_digest
|
||||
) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_price_catalog_head_identity_check CHECK (
|
||||
length(provider) BETWEEN 1 AND 128 AND
|
||||
length(model) BETWEEN 1 AND 128 AND
|
||||
length(mutation_id) BETWEEN 1 AND 128 AND
|
||||
length(changed_by_user_id) BETWEEN 1 AND 128
|
||||
),
|
||||
CONSTRAINT ql3_model_price_catalog_head_value_check CHECK (
|
||||
generation BETWEEN 1 AND 2147483647 AND
|
||||
((generation = 1 AND previous_head_digest IS NULL) OR
|
||||
(generation > 1 AND previous_head_digest IS NOT NULL)) AND
|
||||
((active_price_revision IS NULL AND active_catalog_digest IS NULL) OR
|
||||
(active_price_revision IS NOT NULL AND
|
||||
active_catalog_digest IS NOT NULL)) AND
|
||||
((revoked_price_revision IS NULL AND revoked_catalog_digest IS NULL) OR
|
||||
(revoked_price_revision IS NOT NULL AND
|
||||
revoked_catalog_digest IS NOT NULL)) AND
|
||||
(revoked_price_revision IS NULL OR
|
||||
revoked_price_revision <> active_price_revision) AND
|
||||
action IN ('activate', 'deactivate', 'revoke') AND
|
||||
((action = 'revoke') = (revoked_price_revision IS NOT NULL)) AND
|
||||
changed_at_ms >= 0 AND
|
||||
(previous_head_digest IS NULL OR (
|
||||
length(previous_head_digest) = 64 AND
|
||||
previous_head_digest NOT GLOB '*[^0-9a-f]*'
|
||||
)) AND
|
||||
(active_catalog_digest IS NULL OR (
|
||||
length(active_catalog_digest) = 64 AND
|
||||
active_catalog_digest NOT GLOB '*[^0-9a-f]*'
|
||||
)) AND
|
||||
(revoked_catalog_digest IS NULL OR (
|
||||
length(revoked_catalog_digest) = 64 AND
|
||||
revoked_catalog_digest NOT GLOB '*[^0-9a-f]*'
|
||||
)) AND
|
||||
length(command_digest) = 64 AND
|
||||
command_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(head_digest) = 64 AND head_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_model_price_catalog_head_json_check CHECK (
|
||||
length(CAST(head_json AS BLOB)) BETWEEN 2 AND 24576 AND
|
||||
json_valid(head_json) AND json_type(head_json) = 'object' AND
|
||||
json_extract(head_json, '$.schema') =
|
||||
'qinglong/model-price-catalog-head@v1' AND
|
||||
json_extract(head_json, '$.provider') = provider AND
|
||||
json_extract(head_json, '$.model') = model AND
|
||||
json_extract(head_json, '$.generation') = generation AND
|
||||
((previous_head_digest IS NULL AND
|
||||
json_type(head_json, '$.previousHeadDigest') = 'null') OR
|
||||
json_extract(head_json, '$.previousHeadDigest') =
|
||||
previous_head_digest) AND
|
||||
((active_price_revision IS NULL AND
|
||||
json_type(head_json, '$.activePriceRevision') = 'null') OR
|
||||
json_extract(head_json, '$.activePriceRevision') =
|
||||
active_price_revision) AND
|
||||
((active_catalog_digest IS NULL AND
|
||||
json_type(head_json, '$.activeCatalogDigest') = 'null') OR
|
||||
json_extract(head_json, '$.activeCatalogDigest') =
|
||||
active_catalog_digest) AND
|
||||
((revoked_price_revision IS NULL AND
|
||||
json_type(head_json, '$.revokedPriceRevision') = 'null') OR
|
||||
json_extract(head_json, '$.revokedPriceRevision') =
|
||||
revoked_price_revision) AND
|
||||
((revoked_catalog_digest IS NULL AND
|
||||
json_type(head_json, '$.revokedCatalogDigest') = 'null') OR
|
||||
json_extract(head_json, '$.revokedCatalogDigest') =
|
||||
revoked_catalog_digest) AND
|
||||
json_extract(head_json, '$.action') = action AND
|
||||
json_extract(head_json, '$.mutationId') = mutation_id AND
|
||||
json_extract(head_json, '$.changedByUserId') =
|
||||
changed_by_user_id AND
|
||||
json_extract(head_json, '$.changedAtMs') = changed_at_ms AND
|
||||
json_extract(head_json, '$.commandDigest') = command_digest AND
|
||||
json_extract(head_json, '$.headDigest') = head_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const localModelPriceCatalogMigration =
|
||||
defineSqlMigration<LocalMigrationContext>(
|
||||
LOCAL_MODEL_PRICE_CATALOG_MIGRATION_ID,
|
||||
[
|
||||
LOCAL_MODEL_PRICE_CATALOG_PUBLICATION_TABLE_SQL,
|
||||
`CREATE UNIQUE INDEX ql3_model_price_catalog_publication_identity_digest_uidx
|
||||
ON "ModelPriceCatalogPublications"
|
||||
(provider, model, price_revision, catalog_digest)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_price_catalog_publication_digest_uidx
|
||||
ON "ModelPriceCatalogPublications" (publication_digest)`,
|
||||
LOCAL_MODEL_PRICE_CATALOG_HEAD_TABLE_SQL,
|
||||
`CREATE INDEX ql3_model_price_catalog_head_current_idx
|
||||
ON "ModelPriceCatalogHeads" (provider, model, generation DESC)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_price_catalog_revoked_revision_uidx
|
||||
ON "ModelPriceCatalogHeads"
|
||||
(provider, model, revoked_price_revision)
|
||||
WHERE revoked_price_revision IS NOT NULL`,
|
||||
],
|
||||
(context, statement) => context.client.exec(statement),
|
||||
);
|
||||
|
||||
const LOCAL_MODEL_PRICE_CATALOG_AUTHORIZATION_TABLE_SQL = `
|
||||
CREATE TABLE "ModelPriceCatalogAuthorizations" (
|
||||
authorization_id TEXT PRIMARY KEY,
|
||||
request_id TEXT NOT NULL,
|
||||
operation TEXT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
price_revision TEXT,
|
||||
catalog_command_digest TEXT NOT NULL UNIQUE,
|
||||
publication_digest TEXT UNIQUE,
|
||||
head_digest TEXT UNIQUE,
|
||||
result_digest TEXT NOT NULL UNIQUE,
|
||||
user_id TEXT NOT NULL,
|
||||
authentication_id TEXT NOT NULL,
|
||||
assurance TEXT NOT NULL,
|
||||
authenticated_at_ms INTEGER NOT NULL,
|
||||
expires_at_ms INTEGER NOT NULL,
|
||||
policy_revision TEXT NOT NULL,
|
||||
policy_decision_digest TEXT NOT NULL,
|
||||
decision_mode TEXT NOT NULL,
|
||||
command_digest TEXT NOT NULL UNIQUE,
|
||||
committed_at_ms INTEGER NOT NULL,
|
||||
authorization_digest TEXT NOT NULL UNIQUE,
|
||||
reasons_json TEXT NOT NULL,
|
||||
authorization_json TEXT NOT NULL,
|
||||
FOREIGN KEY (publication_digest)
|
||||
REFERENCES "ModelPriceCatalogPublications" (publication_digest)
|
||||
ON DELETE RESTRICT,
|
||||
FOREIGN KEY (head_digest)
|
||||
REFERENCES "ModelPriceCatalogHeads" (head_digest)
|
||||
ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_price_catalog_authorization_identity_check CHECK (
|
||||
length(authorization_id) BETWEEN 1 AND 128 AND
|
||||
length(request_id) BETWEEN 1 AND 128 AND
|
||||
length(provider) BETWEEN 1 AND 128 AND
|
||||
length(model) BETWEEN 1 AND 128 AND
|
||||
(price_revision IS NULL OR length(price_revision) BETWEEN 1 AND 128) AND
|
||||
length(user_id) BETWEEN 1 AND 128 AND
|
||||
length(authentication_id) BETWEEN 1 AND 128 AND
|
||||
length(policy_revision) BETWEEN 1 AND 128
|
||||
),
|
||||
CONSTRAINT ql3_model_price_catalog_authorization_value_check CHECK (
|
||||
operation IN ('publish', 'activate', 'deactivate', 'revoke') AND
|
||||
((operation = 'deactivate' AND price_revision IS NULL) OR
|
||||
(operation <> 'deactivate' AND price_revision IS NOT NULL)) AND
|
||||
((operation = 'publish' AND publication_digest IS NOT NULL AND
|
||||
head_digest IS NULL AND result_digest = publication_digest) OR
|
||||
(operation <> 'publish' AND publication_digest IS NULL AND
|
||||
head_digest IS NOT NULL AND result_digest = head_digest)) AND
|
||||
assurance IN ('multi_factor', 'hardware', 'local_console') AND
|
||||
decision_mode IN ('human_confirmation', 'separation_of_duty') AND
|
||||
authenticated_at_ms >= 0 AND
|
||||
expires_at_ms > authenticated_at_ms AND
|
||||
committed_at_ms >= authenticated_at_ms AND
|
||||
committed_at_ms < expires_at_ms AND
|
||||
committed_at_ms - authenticated_at_ms <= 300000 AND
|
||||
length(catalog_command_digest) = 64 AND
|
||||
catalog_command_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(result_digest) = 64 AND
|
||||
result_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(policy_decision_digest) = 64 AND
|
||||
policy_decision_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(command_digest) = 64 AND
|
||||
command_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(authorization_digest) = 64 AND
|
||||
authorization_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_model_price_catalog_authorization_reasons_check CHECK (
|
||||
length(CAST(reasons_json AS BLOB)) BETWEEN 5 AND 2048 AND
|
||||
json_valid(reasons_json) AND json_type(reasons_json) = 'array' AND
|
||||
json_array_length(reasons_json) BETWEEN 1 AND 8
|
||||
),
|
||||
CONSTRAINT ql3_model_price_catalog_authorization_json_check CHECK (
|
||||
length(CAST(authorization_json AS BLOB)) BETWEEN 2 AND 32768 AND
|
||||
json_valid(authorization_json) AND
|
||||
json_type(authorization_json) = 'object' AND
|
||||
json_extract(authorization_json, '$.schema') =
|
||||
'qinglong/model-price-catalog-authorization@v1' AND
|
||||
json_extract(authorization_json, '$.authorizationId') =
|
||||
authorization_id AND
|
||||
json_extract(authorization_json, '$.requestId') = request_id AND
|
||||
json_extract(authorization_json, '$.operation') = operation AND
|
||||
json_extract(authorization_json, '$.provider') = provider AND
|
||||
json_extract(authorization_json, '$.model') = model AND
|
||||
((price_revision IS NULL AND
|
||||
json_type(authorization_json, '$.priceRevision') = 'null') OR
|
||||
json_extract(authorization_json, '$.priceRevision') =
|
||||
price_revision) AND
|
||||
json_extract(authorization_json, '$.catalogCommandDigest') =
|
||||
catalog_command_digest AND
|
||||
json_extract(authorization_json, '$.resultDigest') = result_digest AND
|
||||
json_extract(authorization_json, '$.principal.subject.type') = 'user' AND
|
||||
json_extract(authorization_json, '$.principal.subject.id') = user_id AND
|
||||
json_extract(authorization_json, '$.principal.authenticationId') =
|
||||
authentication_id AND
|
||||
json_extract(authorization_json, '$.principal.assurance') = assurance AND
|
||||
json_extract(authorization_json, '$.principal.authenticatedAtMs') =
|
||||
authenticated_at_ms AND
|
||||
json_extract(authorization_json, '$.principal.expiresAtMs') =
|
||||
expires_at_ms AND
|
||||
json_extract(authorization_json, '$.policy.revision') =
|
||||
policy_revision AND
|
||||
json_extract(authorization_json, '$.policy.decisionDigest') =
|
||||
policy_decision_digest AND
|
||||
json_extract(authorization_json, '$.decisionMode') = decision_mode AND
|
||||
json_extract(authorization_json, '$.commandDigest') = command_digest AND
|
||||
json_extract(authorization_json, '$.committedAtMs') = committed_at_ms AND
|
||||
json_extract(authorization_json, '$.authorizationDigest') =
|
||||
authorization_digest AND
|
||||
json_extract(authorization_json, '$.policy.reasons') =
|
||||
json(reasons_json)
|
||||
)
|
||||
)`;
|
||||
|
||||
const localModelPriceCatalogAuthorizationMigration =
|
||||
defineSqlMigration<LocalMigrationContext>(
|
||||
LOCAL_MODEL_PRICE_CATALOG_AUTHORIZATION_MIGRATION_ID,
|
||||
[
|
||||
LOCAL_MODEL_PRICE_CATALOG_AUTHORIZATION_TABLE_SQL,
|
||||
`CREATE INDEX ql3_model_price_catalog_authorization_target_idx
|
||||
ON "ModelPriceCatalogAuthorizations"
|
||||
(provider, model, operation, committed_at_ms DESC)`,
|
||||
],
|
||||
(context, statement) => context.client.exec(statement),
|
||||
);
|
||||
|
||||
const LOCAL_MODEL_INVOCATION_FEATURE_TRANSITION_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationFeatureTransitions" (
|
||||
feature_id TEXT NOT NULL
|
||||
CONSTRAINT ql3_ai_feature_transition_feature_check
|
||||
CHECK (feature_id = 'model-invocation'),
|
||||
generation INTEGER NOT NULL
|
||||
CONSTRAINT ql3_ai_feature_transition_generation_check
|
||||
CHECK (generation BETWEEN 1 AND 2147483647),
|
||||
previous_generation INTEGER,
|
||||
state TEXT NOT NULL
|
||||
CONSTRAINT ql3_ai_feature_transition_state_check
|
||||
CHECK (state IN ('active', 'inactive')),
|
||||
mutation_id TEXT NOT NULL UNIQUE,
|
||||
request_id TEXT NOT NULL,
|
||||
expected_migration_digest TEXT NOT NULL,
|
||||
safety_mode TEXT NOT NULL
|
||||
CONSTRAINT ql3_ai_feature_transition_safety_check
|
||||
CHECK (safety_mode IN (
|
||||
'fresh_database',
|
||||
'backup_verified',
|
||||
'preserve_existing'
|
||||
)),
|
||||
backup_evidence_digest TEXT,
|
||||
changed_by_user_id TEXT NOT NULL,
|
||||
authentication_id TEXT NOT NULL,
|
||||
assurance TEXT NOT NULL
|
||||
CONSTRAINT ql3_ai_feature_transition_assurance_check
|
||||
CHECK (assurance = 'local_console'),
|
||||
command_digest TEXT NOT NULL,
|
||||
transition_digest TEXT NOT NULL UNIQUE,
|
||||
committed_at_ms INTEGER NOT NULL
|
||||
CONSTRAINT ql3_ai_feature_transition_time_check
|
||||
CHECK (committed_at_ms >= 0),
|
||||
transition_json TEXT NOT NULL,
|
||||
PRIMARY KEY (feature_id, generation),
|
||||
UNIQUE (feature_id, generation, transition_digest),
|
||||
CONSTRAINT ql3_ai_feature_transition_previous_fk
|
||||
FOREIGN KEY (feature_id, previous_generation)
|
||||
REFERENCES "ModelInvocationFeatureTransitions" (feature_id, generation)
|
||||
ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_feature_transition_previous_check CHECK (
|
||||
(
|
||||
generation = 1 AND
|
||||
previous_generation IS NULL AND
|
||||
state = 'active'
|
||||
) OR (
|
||||
generation > 1 AND
|
||||
previous_generation = generation - 1
|
||||
)
|
||||
),
|
||||
CONSTRAINT ql3_ai_feature_transition_identity_check CHECK (
|
||||
length(mutation_id) BETWEEN 1 AND 128 AND
|
||||
length(request_id) BETWEEN 1 AND 128 AND
|
||||
length(changed_by_user_id) BETWEEN 1 AND 255 AND
|
||||
length(authentication_id) BETWEEN 1 AND 128
|
||||
),
|
||||
CONSTRAINT ql3_ai_feature_transition_digest_check CHECK (
|
||||
length(expected_migration_digest) = 64 AND
|
||||
expected_migration_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
(
|
||||
backup_evidence_digest IS NULL OR
|
||||
length(backup_evidence_digest) = 64 AND
|
||||
backup_evidence_digest NOT GLOB '*[^0-9a-f]*'
|
||||
) AND
|
||||
length(command_digest) = 64 AND
|
||||
command_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(transition_digest) = 64 AND
|
||||
transition_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_ai_feature_transition_safety_shape_check CHECK (
|
||||
(
|
||||
state = 'active' AND
|
||||
(
|
||||
(
|
||||
safety_mode = 'fresh_database' AND
|
||||
backup_evidence_digest IS NULL
|
||||
) OR (
|
||||
safety_mode = 'backup_verified' AND
|
||||
backup_evidence_digest IS NOT NULL
|
||||
)
|
||||
)
|
||||
) OR (
|
||||
state = 'inactive' AND
|
||||
safety_mode = 'preserve_existing' AND
|
||||
backup_evidence_digest IS NULL
|
||||
)
|
||||
),
|
||||
CONSTRAINT ql3_ai_feature_transition_json_check CHECK (
|
||||
json_valid(transition_json) AND
|
||||
json_type(transition_json) = 'object' AND
|
||||
json_extract(transition_json, '$.schema') =
|
||||
'qinglong/model-invocation-feature-transition@v1' AND
|
||||
json_extract(transition_json, '$.featureId') = feature_id AND
|
||||
json_extract(transition_json, '$.generation') = generation AND
|
||||
(
|
||||
(
|
||||
previous_generation IS NULL AND
|
||||
json_type(transition_json, '$.previousGeneration') = 'null'
|
||||
) OR
|
||||
json_extract(transition_json, '$.previousGeneration') =
|
||||
previous_generation
|
||||
) AND
|
||||
json_extract(transition_json, '$.state') = state AND
|
||||
json_extract(transition_json, '$.mutationId') = mutation_id AND
|
||||
json_extract(transition_json, '$.requestId') = request_id AND
|
||||
json_extract(transition_json, '$.expectedMigrationDigest') =
|
||||
expected_migration_digest AND
|
||||
json_extract(transition_json, '$.safety.mode') = safety_mode AND
|
||||
(
|
||||
(
|
||||
backup_evidence_digest IS NULL AND
|
||||
json_type(transition_json, '$.safety.backupEvidenceDigest') = 'null'
|
||||
) OR
|
||||
json_extract(transition_json, '$.safety.backupEvidenceDigest') =
|
||||
backup_evidence_digest
|
||||
) AND
|
||||
json_extract(transition_json, '$.changedByUserId') =
|
||||
changed_by_user_id AND
|
||||
json_extract(transition_json, '$.authenticationId') =
|
||||
authentication_id AND
|
||||
json_extract(transition_json, '$.assurance') = assurance AND
|
||||
json_extract(transition_json, '$.commandDigest') = command_digest AND
|
||||
json_extract(transition_json, '$.committedAtMs') = committed_at_ms AND
|
||||
json_extract(transition_json, '$.transitionDigest') =
|
||||
transition_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const LOCAL_MODEL_INVOCATION_FEATURE_HEAD_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationFeatureHead" (
|
||||
feature_id TEXT PRIMARY KEY NOT NULL
|
||||
CONSTRAINT ql3_ai_feature_head_feature_check
|
||||
CHECK (feature_id = 'model-invocation'),
|
||||
generation INTEGER NOT NULL
|
||||
CONSTRAINT ql3_ai_feature_head_generation_check
|
||||
CHECK (generation BETWEEN 1 AND 2147483647),
|
||||
state TEXT NOT NULL
|
||||
CONSTRAINT ql3_ai_feature_head_state_check
|
||||
CHECK (state IN ('active', 'inactive')),
|
||||
transition_digest TEXT NOT NULL UNIQUE,
|
||||
updated_at_ms INTEGER NOT NULL
|
||||
CONSTRAINT ql3_ai_feature_head_time_check
|
||||
CHECK (updated_at_ms >= 0),
|
||||
CONSTRAINT ql3_ai_feature_head_transition_fk
|
||||
FOREIGN KEY (feature_id, generation, transition_digest)
|
||||
REFERENCES "ModelInvocationFeatureTransitions" (
|
||||
feature_id,
|
||||
generation,
|
||||
transition_digest
|
||||
) ON DELETE RESTRICT
|
||||
)`;
|
||||
|
||||
const localModelInvocationFeatureActivationMigration =
|
||||
defineSqlMigration<LocalMigrationContext>(
|
||||
LOCAL_MODEL_INVOCATION_FEATURE_ACTIVATION_MIGRATION_ID,
|
||||
[
|
||||
LOCAL_MODEL_INVOCATION_FEATURE_TRANSITION_TABLE_SQL,
|
||||
`CREATE INDEX ql3_ai_feature_transition_state_idx
|
||||
ON "ModelInvocationFeatureTransitions"
|
||||
(feature_id, state, generation DESC)`,
|
||||
LOCAL_MODEL_INVOCATION_FEATURE_HEAD_TABLE_SQL,
|
||||
],
|
||||
(context, statement) => context.client.exec(statement),
|
||||
);
|
||||
|
||||
export const sqliteCatalogMigrations = Object.freeze([
|
||||
localModelPriceCatalogMigration,
|
||||
localModelPriceCatalogAuthorizationMigration,
|
||||
localModelInvocationFeatureActivationMigration,
|
||||
]);
|
||||
@@ -0,0 +1,5 @@
|
||||
import type { DatabaseSync } from 'node:sqlite';
|
||||
|
||||
export interface LocalMigrationContext {
|
||||
readonly client: DatabaseSync;
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
import type { LocalMigrationContext } from './context';
|
||||
|
||||
import {
|
||||
LOCAL_MODEL_INVOCATION_MIGRATION_ID,
|
||||
} from '../identities';
|
||||
|
||||
import { defineSqlMigration } from '../shared';
|
||||
|
||||
const LOCAL_START_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationStarts" (
|
||||
invocation_id TEXT PRIMARY KEY NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
run_id TEXT NOT NULL,
|
||||
step_run_id TEXT NOT NULL,
|
||||
trace_id TEXT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
policy_revision TEXT NOT NULL,
|
||||
request_digest TEXT NOT NULL,
|
||||
input_bytes INTEGER NOT NULL,
|
||||
max_output_tokens INTEGER NOT NULL,
|
||||
deadline_at_ms INTEGER NOT NULL,
|
||||
admitted_at_ms INTEGER NOT NULL,
|
||||
mutation_id TEXT NOT NULL,
|
||||
mutation_digest TEXT NOT NULL,
|
||||
run_event_id TEXT NOT NULL,
|
||||
start_digest TEXT NOT NULL,
|
||||
record_json TEXT NOT NULL,
|
||||
FOREIGN KEY (run_id, step_run_id)
|
||||
REFERENCES "StepRuns" (run_id, id) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (mutation_id)
|
||||
REFERENCES "StepRunMutations" (mutation_id) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (run_event_id)
|
||||
REFERENCES "RunEvents" (id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_invocation_starts_identity_check CHECK (
|
||||
length(invocation_id) BETWEEN 1 AND 128 AND
|
||||
length(project_id) BETWEEN 1 AND 128 AND
|
||||
length(run_id) BETWEEN 1 AND 128 AND
|
||||
length(step_run_id) BETWEEN 1 AND 128 AND
|
||||
length(trace_id) BETWEEN 1 AND 128 AND
|
||||
length(provider) BETWEEN 1 AND 128 AND
|
||||
length(model) BETWEEN 1 AND 128 AND
|
||||
length(policy_revision) BETWEEN 1 AND 128 AND
|
||||
length(mutation_id) BETWEEN 1 AND 128 AND
|
||||
length(run_event_id) BETWEEN 1 AND 36
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_starts_digest_check CHECK (
|
||||
length(request_digest) = 71 AND
|
||||
substr(request_digest, 1, 7) = 'sha256:' AND
|
||||
substr(request_digest, 8) NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(mutation_digest) = 64 AND
|
||||
mutation_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(start_digest) = 64 AND
|
||||
start_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_starts_budget_check CHECK (
|
||||
input_bytes BETWEEN 1 AND 262144 AND
|
||||
max_output_tokens BETWEEN 1 AND 32768 AND
|
||||
admitted_at_ms >= 0 AND deadline_at_ms > admitted_at_ms
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_starts_json_check CHECK (
|
||||
length(CAST(record_json AS BLOB)) BETWEEN 2 AND 24576 AND
|
||||
json_valid(record_json) AND json_type(record_json) = 'object' AND
|
||||
json_extract(record_json, '$.schema') =
|
||||
'qinglong/model-invocation-start@v1' AND
|
||||
json_extract(record_json, '$.invocationId') = invocation_id AND
|
||||
json_extract(record_json, '$.projectId') = project_id AND
|
||||
json_extract(record_json, '$.runId') = run_id AND
|
||||
json_extract(record_json, '$.stepRunId') = step_run_id AND
|
||||
json_extract(record_json, '$.traceId') = trace_id AND
|
||||
json_extract(record_json, '$.provider') = provider AND
|
||||
json_extract(record_json, '$.model') = model AND
|
||||
json_extract(record_json, '$.policyRevision') = policy_revision AND
|
||||
json_extract(record_json, '$.requestDigest') = request_digest AND
|
||||
json_extract(record_json, '$.inputBytes') = input_bytes AND
|
||||
json_extract(record_json, '$.maxOutputTokens') = max_output_tokens AND
|
||||
json_extract(record_json, '$.deadlineAtMs') = deadline_at_ms AND
|
||||
json_extract(record_json, '$.admittedAtMs') = admitted_at_ms AND
|
||||
json_extract(record_json, '$.stepRunMutationId') = mutation_id AND
|
||||
json_extract(record_json, '$.stepRunMutationDigest') = mutation_digest AND
|
||||
json_extract(record_json, '$.runEventId') = run_event_id AND
|
||||
json_extract(record_json, '$.startDigest') = start_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const LOCAL_COMPLETION_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationCompletions" (
|
||||
invocation_id TEXT PRIMARY KEY NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
run_id TEXT NOT NULL,
|
||||
step_run_id TEXT NOT NULL,
|
||||
trace_id TEXT NOT NULL,
|
||||
start_digest TEXT NOT NULL,
|
||||
outcome TEXT NOT NULL,
|
||||
output_bytes INTEGER NOT NULL,
|
||||
error_code TEXT,
|
||||
completed_at_ms INTEGER NOT NULL,
|
||||
mutation_id TEXT NOT NULL,
|
||||
mutation_digest TEXT NOT NULL,
|
||||
run_event_id TEXT NOT NULL,
|
||||
completion_digest TEXT NOT NULL,
|
||||
record_json TEXT NOT NULL,
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "ModelInvocationStarts" (invocation_id) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (run_id, step_run_id)
|
||||
REFERENCES "StepRuns" (run_id, id) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (mutation_id)
|
||||
REFERENCES "StepRunMutations" (mutation_id) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (run_event_id)
|
||||
REFERENCES "RunEvents" (id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_invocation_completions_outcome_check CHECK (
|
||||
outcome IN ('succeeded', 'failed', 'timed_out', 'outcome_unknown')
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_completions_value_check CHECK (
|
||||
output_bytes BETWEEN 0 AND 1048576 AND completed_at_ms >= 0 AND
|
||||
((outcome = 'succeeded' AND error_code IS NULL) OR
|
||||
(outcome <> 'succeeded' AND
|
||||
length(error_code) BETWEEN 1 AND 64 AND
|
||||
substr(error_code, 1, 1) GLOB '[A-Z]' AND
|
||||
error_code NOT GLOB '*[^A-Z0-9_]*')) AND
|
||||
length(start_digest) = 64 AND
|
||||
start_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(mutation_digest) = 64 AND
|
||||
mutation_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(completion_digest) = 64 AND
|
||||
completion_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_completions_json_check CHECK (
|
||||
length(CAST(record_json AS BLOB)) BETWEEN 2 AND 24576 AND
|
||||
json_valid(record_json) AND json_type(record_json) = 'object' AND
|
||||
json_extract(record_json, '$.schema') =
|
||||
'qinglong/model-invocation-completion@v1' AND
|
||||
json_extract(record_json, '$.invocationId') = invocation_id AND
|
||||
json_extract(record_json, '$.projectId') = project_id AND
|
||||
json_extract(record_json, '$.runId') = run_id AND
|
||||
json_extract(record_json, '$.stepRunId') = step_run_id AND
|
||||
json_extract(record_json, '$.traceId') = trace_id AND
|
||||
json_extract(record_json, '$.startDigest') = start_digest AND
|
||||
json_extract(record_json, '$.outcome') = outcome AND
|
||||
json_extract(record_json, '$.outputBytes') = output_bytes AND
|
||||
json_extract(record_json, '$.errorCode') IS error_code AND
|
||||
json_extract(record_json, '$.completedAtMs') = completed_at_ms AND
|
||||
json_extract(record_json, '$.stepRunMutationId') = mutation_id AND
|
||||
json_extract(record_json, '$.stepRunMutationDigest') = mutation_digest AND
|
||||
json_extract(record_json, '$.runEventId') = run_event_id AND
|
||||
json_extract(record_json, '$.completionDigest') = completion_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const LOCAL_RESOLUTION_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationResolutions" (
|
||||
resolution_id TEXT PRIMARY KEY NOT NULL,
|
||||
invocation_id TEXT NOT NULL UNIQUE,
|
||||
project_id TEXT NOT NULL,
|
||||
run_id TEXT NOT NULL,
|
||||
step_run_id TEXT NOT NULL,
|
||||
trace_id TEXT NOT NULL,
|
||||
completion_digest TEXT NOT NULL,
|
||||
decision TEXT NOT NULL,
|
||||
resolved_by_user_id TEXT NOT NULL,
|
||||
resolved_at_ms INTEGER NOT NULL,
|
||||
mutation_id TEXT NOT NULL UNIQUE,
|
||||
mutation_digest TEXT NOT NULL,
|
||||
run_event_id TEXT NOT NULL UNIQUE,
|
||||
resolution_digest TEXT NOT NULL UNIQUE,
|
||||
record_json TEXT NOT NULL,
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "ModelInvocationCompletions" (invocation_id) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (run_id, step_run_id)
|
||||
REFERENCES "StepRuns" (run_id, id) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (mutation_id)
|
||||
REFERENCES "StepRunMutations" (mutation_id) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (run_event_id)
|
||||
REFERENCES "RunEvents" (id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_invocation_resolutions_value_check CHECK (
|
||||
decision IN ('retry', 'fail', 'cancel') AND
|
||||
resolved_at_ms >= 0 AND
|
||||
length(resolution_id) BETWEEN 1 AND 128 AND
|
||||
length(invocation_id) BETWEEN 1 AND 128 AND
|
||||
length(project_id) BETWEEN 1 AND 128 AND
|
||||
length(run_id) BETWEEN 1 AND 128 AND
|
||||
length(step_run_id) BETWEEN 1 AND 128 AND
|
||||
length(trace_id) BETWEEN 1 AND 128 AND
|
||||
length(resolved_by_user_id) BETWEEN 1 AND 128 AND
|
||||
length(mutation_id) BETWEEN 1 AND 128 AND
|
||||
length(run_event_id) BETWEEN 1 AND 36 AND
|
||||
length(completion_digest) = 64 AND
|
||||
completion_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(mutation_digest) = 64 AND
|
||||
mutation_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(resolution_digest) = 64 AND
|
||||
resolution_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_resolutions_json_check CHECK (
|
||||
length(CAST(record_json AS BLOB)) BETWEEN 2 AND 24576 AND
|
||||
json_valid(record_json) AND json_type(record_json) = 'object' AND
|
||||
json_extract(record_json, '$.schema') =
|
||||
'qinglong/model-invocation-resolution@v1' AND
|
||||
json_extract(record_json, '$.resolutionId') = resolution_id AND
|
||||
json_extract(record_json, '$.invocationId') = invocation_id AND
|
||||
json_extract(record_json, '$.projectId') = project_id AND
|
||||
json_extract(record_json, '$.runId') = run_id AND
|
||||
json_extract(record_json, '$.stepRunId') = step_run_id AND
|
||||
json_extract(record_json, '$.traceId') = trace_id AND
|
||||
json_extract(record_json, '$.completionDigest') = completion_digest AND
|
||||
json_extract(record_json, '$.decision') = decision AND
|
||||
json_extract(record_json, '$.resolvedByUserId') = resolved_by_user_id AND
|
||||
json_extract(record_json, '$.resolvedAtMs') = resolved_at_ms AND
|
||||
json_extract(record_json, '$.stepRunMutationId') = mutation_id AND
|
||||
json_extract(record_json, '$.stepRunMutationDigest') = mutation_digest AND
|
||||
json_extract(record_json, '$.runEventId') = run_event_id AND
|
||||
json_extract(record_json, '$.resolutionDigest') = resolution_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const localMigration = defineSqlMigration<LocalMigrationContext>(
|
||||
LOCAL_MODEL_INVOCATION_MIGRATION_ID,
|
||||
[
|
||||
LOCAL_START_TABLE_SQL,
|
||||
`CREATE INDEX ql3_model_invocation_starts_step_history_idx
|
||||
ON "ModelInvocationStarts"
|
||||
(run_id, step_run_id, admitted_at_ms, invocation_id)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_starts_mutation_uidx
|
||||
ON "ModelInvocationStarts" (mutation_id)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_starts_event_uidx
|
||||
ON "ModelInvocationStarts" (run_event_id)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_starts_digest_uidx
|
||||
ON "ModelInvocationStarts" (start_digest)`,
|
||||
LOCAL_COMPLETION_TABLE_SQL,
|
||||
`CREATE INDEX ql3_model_invocation_completions_step_history_idx
|
||||
ON "ModelInvocationCompletions"
|
||||
(run_id, step_run_id, completed_at_ms, invocation_id)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_completions_mutation_uidx
|
||||
ON "ModelInvocationCompletions" (mutation_id)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_completions_event_uidx
|
||||
ON "ModelInvocationCompletions" (run_event_id)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_completions_digest_uidx
|
||||
ON "ModelInvocationCompletions" (completion_digest)`,
|
||||
LOCAL_RESOLUTION_TABLE_SQL,
|
||||
`CREATE INDEX ql3_model_invocation_resolutions_step_history_idx
|
||||
ON "ModelInvocationResolutions"
|
||||
(run_id, step_run_id, resolved_at_ms, resolution_id)`,
|
||||
],
|
||||
(context, statement) => context.client.exec(statement),
|
||||
);
|
||||
|
||||
export const sqliteCoreMigrations = Object.freeze([
|
||||
localMigration,
|
||||
]);
|
||||
@@ -0,0 +1,171 @@
|
||||
import type { LocalMigrationContext } from './context';
|
||||
|
||||
import {
|
||||
LOCAL_MODEL_PROVIDER_CREDENTIAL_CATALOG_MIGRATION_ID,
|
||||
} from '../identities';
|
||||
|
||||
import { defineSqlMigration } from '../shared';
|
||||
|
||||
const LOCAL_MODEL_PROVIDER_CREDENTIAL_BINDING_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationProviderCredentialBindings" (
|
||||
project_id TEXT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
revision TEXT NOT NULL,
|
||||
binding_digest TEXT NOT NULL,
|
||||
binding_json TEXT NOT NULL,
|
||||
PRIMARY KEY (project_id, provider, revision),
|
||||
CONSTRAINT ql3_ai_local_provider_binding_identity_check CHECK (
|
||||
length(project_id) BETWEEN 1 AND 128 AND
|
||||
length(provider) BETWEEN 1 AND 128 AND
|
||||
length(revision) BETWEEN 1 AND 128 AND
|
||||
project_id NOT GLOB '*[^A-Za-z0-9._:/-]*' AND
|
||||
provider NOT GLOB '*[^A-Za-z0-9._:/-]*' AND
|
||||
revision NOT GLOB '*[^A-Za-z0-9._:/-]*'
|
||||
),
|
||||
CONSTRAINT ql3_ai_local_provider_binding_digest_check CHECK (
|
||||
length(binding_digest) = 64 AND
|
||||
binding_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_ai_local_provider_binding_json_check CHECK (
|
||||
length(CAST(binding_json AS BLOB)) BETWEEN 2 AND 4096 AND
|
||||
json_valid(binding_json) AND json_type(binding_json) = 'object' AND
|
||||
json_extract(binding_json, '$.schema') =
|
||||
'qinglong/model-provider-credential-binding@v1' AND
|
||||
json_extract(binding_json, '$.projectId') = project_id AND
|
||||
json_extract(binding_json, '$.provider') = provider AND
|
||||
json_extract(binding_json, '$.revision') = revision AND
|
||||
json_extract(binding_json, '$.scheme') = 'bearer'
|
||||
)
|
||||
)`;
|
||||
|
||||
const LOCAL_MODEL_PROVIDER_CREDENTIAL_TRANSITION_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationProviderCredentialTransitions" (
|
||||
mutation_id TEXT PRIMARY KEY NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
generation INTEGER NOT NULL,
|
||||
action TEXT NOT NULL,
|
||||
active_binding_revision TEXT,
|
||||
active_binding_digest TEXT,
|
||||
previous_transition_digest TEXT,
|
||||
changed_by_type TEXT NOT NULL,
|
||||
changed_by_id TEXT NOT NULL,
|
||||
changed_at_ms INTEGER NOT NULL,
|
||||
command_digest TEXT NOT NULL,
|
||||
transition_digest TEXT NOT NULL,
|
||||
command_json TEXT NOT NULL,
|
||||
transition_json TEXT NOT NULL,
|
||||
UNIQUE (project_id, provider, generation),
|
||||
UNIQUE (transition_digest),
|
||||
FOREIGN KEY (project_id, provider, active_binding_revision)
|
||||
REFERENCES "ModelInvocationProviderCredentialBindings"
|
||||
(project_id, provider, revision) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_local_provider_transition_identity_check CHECK (
|
||||
length(mutation_id) BETWEEN 1 AND 128 AND
|
||||
length(project_id) BETWEEN 1 AND 128 AND
|
||||
length(provider) BETWEEN 1 AND 128 AND
|
||||
length(changed_by_id) BETWEEN 1 AND 128 AND
|
||||
project_id NOT GLOB '*[^A-Za-z0-9._:/-]*' AND
|
||||
provider NOT GLOB '*[^A-Za-z0-9._:/-]*'
|
||||
),
|
||||
CONSTRAINT ql3_ai_local_provider_transition_value_check CHECK (
|
||||
generation BETWEEN 1 AND 2147483647 AND
|
||||
action IN ('bind', 'revoke') AND
|
||||
changed_by_type = 'user' AND
|
||||
changed_at_ms >= 0 AND
|
||||
((action = 'bind' AND active_binding_revision IS NOT NULL AND
|
||||
active_binding_digest IS NOT NULL) OR
|
||||
(action = 'revoke' AND active_binding_revision IS NULL AND
|
||||
active_binding_digest IS NULL))
|
||||
),
|
||||
CONSTRAINT ql3_ai_local_provider_transition_digest_check CHECK (
|
||||
length(command_digest) = 64 AND command_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(transition_digest) = 64 AND
|
||||
transition_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
(previous_transition_digest IS NULL OR
|
||||
(length(previous_transition_digest) = 64 AND
|
||||
previous_transition_digest NOT GLOB '*[^0-9a-f]*')) AND
|
||||
(active_binding_digest IS NULL OR
|
||||
(length(active_binding_digest) = 64 AND
|
||||
active_binding_digest NOT GLOB '*[^0-9a-f]*'))
|
||||
),
|
||||
CONSTRAINT ql3_ai_local_provider_transition_json_check CHECK (
|
||||
length(CAST(command_json AS BLOB)) BETWEEN 2 AND 8192 AND
|
||||
json_valid(command_json) AND json_type(command_json) = 'object' AND
|
||||
json_extract(command_json, '$.mutationId') = mutation_id AND
|
||||
json_extract(command_json, '$.projectId') = project_id AND
|
||||
json_extract(command_json, '$.provider') = provider AND
|
||||
json_extract(command_json, '$.action') = action AND
|
||||
length(CAST(transition_json AS BLOB)) BETWEEN 2 AND 8192 AND
|
||||
json_valid(transition_json) AND json_type(transition_json) = 'object' AND
|
||||
json_extract(transition_json, '$.schema') =
|
||||
'qinglong/model-provider-credential-transition@v1' AND
|
||||
json_extract(transition_json, '$.mutationId') = mutation_id AND
|
||||
json_extract(transition_json, '$.projectId') = project_id AND
|
||||
json_extract(transition_json, '$.provider') = provider AND
|
||||
json_extract(transition_json, '$.generation') = generation AND
|
||||
json_extract(transition_json, '$.action') = action AND
|
||||
json_extract(transition_json, '$.transitionDigest') = transition_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const LOCAL_MODEL_PROVIDER_CREDENTIAL_AUDIT_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationProviderCredentialAudits" (
|
||||
operation TEXT NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
request_id TEXT NOT NULL,
|
||||
binding_revision TEXT NOT NULL,
|
||||
binding_digest TEXT NOT NULL,
|
||||
occurred_at_ms INTEGER NOT NULL,
|
||||
audit_json TEXT NOT NULL,
|
||||
PRIMARY KEY (operation, project_id, provider, request_id),
|
||||
CONSTRAINT ql3_ai_local_provider_audit_identity_check CHECK (
|
||||
operation IN ('list_models', 'generate', 'stream') AND
|
||||
length(project_id) BETWEEN 1 AND 128 AND
|
||||
length(provider) BETWEEN 1 AND 128 AND
|
||||
length(request_id) BETWEEN 1 AND 128 AND
|
||||
length(binding_revision) BETWEEN 1 AND 128
|
||||
),
|
||||
CONSTRAINT ql3_ai_local_provider_audit_digest_check CHECK (
|
||||
length(binding_digest) = 71 AND
|
||||
binding_digest GLOB 'sha256:[0-9a-f]*' AND
|
||||
occurred_at_ms >= 0
|
||||
),
|
||||
CONSTRAINT ql3_ai_local_provider_audit_json_check CHECK (
|
||||
length(CAST(audit_json AS BLOB)) BETWEEN 2 AND 4096 AND
|
||||
json_valid(audit_json) AND json_type(audit_json) = 'object' AND
|
||||
json_extract(audit_json, '$.schema') =
|
||||
'qinglong/model-provider-credential-audit@v1' AND
|
||||
json_extract(audit_json, '$.operation') = operation AND
|
||||
json_extract(audit_json, '$.projectId') = project_id AND
|
||||
json_extract(audit_json, '$.provider') = provider AND
|
||||
json_extract(audit_json, '$.requestId') = request_id AND
|
||||
json_extract(audit_json, '$.bindingRevision') = binding_revision AND
|
||||
json_extract(audit_json, '$.bindingDigest') = binding_digest AND
|
||||
json_extract(audit_json, '$.occurredAtMs') = occurred_at_ms
|
||||
)
|
||||
)`;
|
||||
|
||||
const localModelProviderCredentialCatalogMigration =
|
||||
defineSqlMigration<LocalMigrationContext>(
|
||||
LOCAL_MODEL_PROVIDER_CREDENTIAL_CATALOG_MIGRATION_ID,
|
||||
[
|
||||
LOCAL_MODEL_PROVIDER_CREDENTIAL_BINDING_TABLE_SQL,
|
||||
`CREATE UNIQUE INDEX ql3_ai_local_provider_binding_digest_uidx
|
||||
ON "ModelInvocationProviderCredentialBindings" (binding_digest)`,
|
||||
LOCAL_MODEL_PROVIDER_CREDENTIAL_TRANSITION_TABLE_SQL,
|
||||
`CREATE INDEX ql3_ai_local_provider_transition_current_idx
|
||||
ON "ModelInvocationProviderCredentialTransitions"
|
||||
(project_id, provider, generation DESC)`,
|
||||
LOCAL_MODEL_PROVIDER_CREDENTIAL_AUDIT_TABLE_SQL,
|
||||
`CREATE INDEX ql3_ai_local_provider_audit_time_idx
|
||||
ON "ModelInvocationProviderCredentialAudits"
|
||||
(project_id, occurred_at_ms DESC, provider, request_id)`,
|
||||
],
|
||||
(context, statement) => context.client.exec(statement),
|
||||
);
|
||||
|
||||
export const sqliteCredentialMigrations = Object.freeze([
|
||||
localModelProviderCredentialCatalogMigration,
|
||||
]);
|
||||
@@ -0,0 +1,539 @@
|
||||
import type { LocalMigrationContext } from './context';
|
||||
|
||||
import {
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_ADMISSION_MIGRATION_ID,
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_FINALIZATION_MIGRATION_ID,
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_ARTIFACT_MIGRATION_ID,
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_TOMBSTONE_MIGRATION_ID,
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_MIGRATION_ID,
|
||||
} from '../identities';
|
||||
|
||||
import { defineSqlMigration } from '../shared';
|
||||
|
||||
const LOCAL_PLUGIN_PACKAGE_PROMPT_ADMISSION_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationPromptAdmissions" (
|
||||
request_id TEXT PRIMARY KEY NOT NULL,
|
||||
invocation_id TEXT NOT NULL UNIQUE,
|
||||
plan_digest TEXT NOT NULL UNIQUE,
|
||||
run_id TEXT NOT NULL UNIQUE,
|
||||
step_run_id TEXT NOT NULL UNIQUE,
|
||||
project_id TEXT NOT NULL,
|
||||
package_name TEXT NOT NULL,
|
||||
installation_id TEXT NOT NULL,
|
||||
lock_digest TEXT NOT NULL,
|
||||
generation INTEGER NOT NULL,
|
||||
generation_digest TEXT NOT NULL,
|
||||
materialized_revision_digest TEXT NOT NULL,
|
||||
publication_digest TEXT NOT NULL,
|
||||
prompt_id TEXT NOT NULL,
|
||||
prompt_definition_digest TEXT NOT NULL,
|
||||
parameter_digest TEXT NOT NULL,
|
||||
model_request_digest TEXT NOT NULL,
|
||||
admitted_at_ms INTEGER NOT NULL,
|
||||
receipt_digest TEXT NOT NULL UNIQUE,
|
||||
plan_json TEXT NOT NULL,
|
||||
receipt_json TEXT NOT NULL,
|
||||
CONSTRAINT ql3_ai_prompt_admission_run_fk
|
||||
FOREIGN KEY (run_id) REFERENCES "Runs" (id)
|
||||
ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_admission_step_fk
|
||||
FOREIGN KEY (run_id, step_run_id) REFERENCES "StepRuns" (run_id, id)
|
||||
ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_admission_publication_fk
|
||||
FOREIGN KEY (publication_digest)
|
||||
REFERENCES "QingLong3PluginPackageAutomationPublications" (
|
||||
publication_digest
|
||||
) ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_admission_identity_check CHECK (
|
||||
length(request_id) BETWEEN 1 AND 128 AND
|
||||
length(invocation_id) BETWEEN 1 AND 128 AND
|
||||
length(run_id) BETWEEN 1 AND 128 AND
|
||||
length(step_run_id) BETWEEN 1 AND 128 AND
|
||||
length(project_id) BETWEEN 1 AND 128 AND
|
||||
length(package_name) BETWEEN 1 AND 63 AND
|
||||
length(installation_id) BETWEEN 1 AND 128 AND
|
||||
length(prompt_id) BETWEEN 1 AND 128 AND
|
||||
generation BETWEEN 1 AND 2147483647 AND
|
||||
admitted_at_ms >= 0
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_admission_digest_check CHECK (
|
||||
length(plan_digest) = 64 AND
|
||||
plan_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(lock_digest) = 64 AND
|
||||
lock_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(generation_digest) = 64 AND
|
||||
generation_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(materialized_revision_digest) = 64 AND
|
||||
materialized_revision_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(publication_digest) = 64 AND
|
||||
publication_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(prompt_definition_digest) = 64 AND
|
||||
prompt_definition_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(parameter_digest) = 64 AND
|
||||
parameter_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
model_request_digest GLOB 'sha256:[0-9a-f]*' AND
|
||||
length(model_request_digest) = 71 AND
|
||||
substr(model_request_digest, 8) NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(receipt_digest) = 64 AND
|
||||
receipt_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_admission_plan_json_check CHECK (
|
||||
length(CAST(plan_json AS BLOB)) BETWEEN 2 AND 32768 AND
|
||||
json_valid(plan_json) AND json_type(plan_json) = 'object' AND
|
||||
json_extract(plan_json, '$.schema') =
|
||||
'qinglong/plugin-package-prompt-execution-plan@v1' AND
|
||||
json_extract(plan_json, '$.requestId') = request_id AND
|
||||
json_extract(plan_json, '$.invocationId') = invocation_id AND
|
||||
json_extract(plan_json, '$.planDigest') = plan_digest AND
|
||||
json_extract(plan_json, '$.runId') = run_id AND
|
||||
json_extract(plan_json, '$.stepRunId') = step_run_id AND
|
||||
json_extract(plan_json, '$.target.projectId') = project_id AND
|
||||
json_extract(plan_json, '$.target.packageName') = package_name AND
|
||||
json_extract(plan_json, '$.target.installationId') = installation_id AND
|
||||
json_extract(plan_json, '$.target.lockDigest') = lock_digest AND
|
||||
json_extract(plan_json, '$.target.generation') = generation AND
|
||||
json_extract(plan_json, '$.target.generationDigest') =
|
||||
generation_digest AND
|
||||
json_extract(plan_json, '$.target.materializedRevisionDigest') =
|
||||
materialized_revision_digest AND
|
||||
json_extract(plan_json, '$.target.publicationDigest') =
|
||||
publication_digest AND
|
||||
json_extract(plan_json, '$.target.promptId') = prompt_id AND
|
||||
json_extract(plan_json, '$.target.promptDefinitionDigest') =
|
||||
prompt_definition_digest AND
|
||||
json_extract(plan_json, '$.parameterDigest') = parameter_digest AND
|
||||
json_extract(plan_json, '$.modelRequestDigest') = model_request_digest AND
|
||||
json_extract(plan_json, '$.plannedAtMs') = admitted_at_ms
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_admission_receipt_json_check CHECK (
|
||||
length(CAST(receipt_json AS BLOB)) BETWEEN 2 AND 16384 AND
|
||||
json_valid(receipt_json) AND json_type(receipt_json) = 'object' AND
|
||||
json_extract(receipt_json, '$.schema') =
|
||||
'qinglong/plugin-package-prompt-admission-receipt@v1' AND
|
||||
json_extract(receipt_json, '$.requestId') = request_id AND
|
||||
json_extract(receipt_json, '$.invocationId') = invocation_id AND
|
||||
json_extract(receipt_json, '$.planDigest') = plan_digest AND
|
||||
json_extract(receipt_json, '$.runId') = run_id AND
|
||||
json_extract(receipt_json, '$.stepRunId') = step_run_id AND
|
||||
json_extract(receipt_json, '$.publicationDigest') =
|
||||
publication_digest AND
|
||||
json_extract(receipt_json, '$.promptId') = prompt_id AND
|
||||
json_extract(receipt_json, '$.admittedAtMs') = admitted_at_ms AND
|
||||
json_extract(receipt_json, '$.receiptDigest') = receipt_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const localPluginPackagePromptAdmissionMigration =
|
||||
defineSqlMigration<LocalMigrationContext>(
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_ADMISSION_MIGRATION_ID,
|
||||
[
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_ADMISSION_TABLE_SQL,
|
||||
`CREATE UNIQUE INDEX ql3_ai_prompt_admission_run_step_uidx
|
||||
ON "ModelInvocationPromptAdmissions" (run_id, step_run_id)`,
|
||||
`CREATE INDEX ql3_ai_prompt_admission_target_idx
|
||||
ON "ModelInvocationPromptAdmissions"
|
||||
(project_id, package_name, admitted_at_ms, request_id)`,
|
||||
],
|
||||
(context, statement) => context.client.exec(statement),
|
||||
);
|
||||
|
||||
const LOCAL_PLUGIN_PACKAGE_PROMPT_FINALIZATION_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationPromptFinalizations" (
|
||||
request_id TEXT PRIMARY KEY NOT NULL,
|
||||
invocation_id TEXT NOT NULL UNIQUE,
|
||||
plan_digest TEXT NOT NULL UNIQUE,
|
||||
run_id TEXT NOT NULL UNIQUE,
|
||||
step_run_id TEXT NOT NULL UNIQUE,
|
||||
terminal_evidence_kind TEXT NOT NULL,
|
||||
terminal_evidence_digest TEXT NOT NULL UNIQUE,
|
||||
final_step_run_digest TEXT NOT NULL,
|
||||
run_status TEXT NOT NULL,
|
||||
event_id TEXT NOT NULL UNIQUE,
|
||||
final_run_version INTEGER NOT NULL,
|
||||
final_run_event_sequence INTEGER NOT NULL,
|
||||
finalized_at_ms INTEGER NOT NULL,
|
||||
receipt_digest TEXT NOT NULL UNIQUE,
|
||||
receipt_json TEXT NOT NULL,
|
||||
CONSTRAINT ql3_ai_prompt_finalization_admission_fk
|
||||
FOREIGN KEY (request_id) REFERENCES "ModelInvocationPromptAdmissions" (
|
||||
request_id
|
||||
) ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_finalization_invocation_fk
|
||||
FOREIGN KEY (invocation_id) REFERENCES "ModelInvocationPromptAdmissions" (
|
||||
invocation_id
|
||||
) ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_finalization_plan_fk
|
||||
FOREIGN KEY (plan_digest) REFERENCES "ModelInvocationPromptAdmissions" (
|
||||
plan_digest
|
||||
) ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_finalization_run_fk
|
||||
FOREIGN KEY (run_id) REFERENCES "Runs" (id)
|
||||
ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_finalization_step_fk
|
||||
FOREIGN KEY (run_id, step_run_id) REFERENCES "StepRuns" (run_id, id)
|
||||
ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_finalization_event_fk
|
||||
FOREIGN KEY (event_id) REFERENCES "RunEvents" (id)
|
||||
ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_finalization_identity_check CHECK (
|
||||
length(request_id) BETWEEN 1 AND 128 AND
|
||||
length(invocation_id) BETWEEN 1 AND 128 AND
|
||||
length(run_id) BETWEEN 1 AND 128 AND
|
||||
length(step_run_id) BETWEEN 1 AND 128 AND
|
||||
length(event_id) BETWEEN 1 AND 128 AND
|
||||
terminal_evidence_kind IN ('completion', 'resolution') AND
|
||||
run_status IN ('succeeded', 'failed', 'cancelled', 'timed_out') AND
|
||||
final_run_version BETWEEN 3 AND 2147483647 AND
|
||||
final_run_event_sequence = final_run_version AND
|
||||
finalized_at_ms >= 0
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_finalization_digest_check CHECK (
|
||||
length(plan_digest) = 64 AND
|
||||
plan_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(terminal_evidence_digest) = 64 AND
|
||||
terminal_evidence_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(final_step_run_digest) = 64 AND
|
||||
final_step_run_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(receipt_digest) = 64 AND
|
||||
receipt_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_finalization_receipt_json_check CHECK (
|
||||
length(CAST(receipt_json AS BLOB)) BETWEEN 2 AND 16384 AND
|
||||
json_valid(receipt_json) AND json_type(receipt_json) = 'object' AND
|
||||
json_extract(receipt_json, '$.schema') =
|
||||
'qinglong/plugin-package-prompt-finalization-receipt@v1' AND
|
||||
json_extract(receipt_json, '$.requestId') = request_id AND
|
||||
json_extract(receipt_json, '$.invocationId') = invocation_id AND
|
||||
json_extract(receipt_json, '$.planDigest') = plan_digest AND
|
||||
json_extract(receipt_json, '$.runId') = run_id AND
|
||||
json_extract(receipt_json, '$.stepRunId') = step_run_id AND
|
||||
json_extract(receipt_json, '$.terminalEvidenceKind') =
|
||||
terminal_evidence_kind AND
|
||||
json_extract(receipt_json, '$.terminalEvidenceDigest') =
|
||||
terminal_evidence_digest AND
|
||||
json_extract(receipt_json, '$.finalStepRunDigest') =
|
||||
final_step_run_digest AND
|
||||
json_extract(receipt_json, '$.runStatus') = run_status AND
|
||||
json_extract(receipt_json, '$.eventId') = event_id AND
|
||||
json_extract(receipt_json, '$.finalRunVersion') = final_run_version AND
|
||||
json_extract(receipt_json, '$.finalRunEventSequence') =
|
||||
final_run_event_sequence AND
|
||||
json_extract(receipt_json, '$.finalizedAtMs') = finalized_at_ms AND
|
||||
json_extract(receipt_json, '$.receiptDigest') = receipt_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const localPluginPackagePromptFinalizationMigration =
|
||||
defineSqlMigration<LocalMigrationContext>(
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_FINALIZATION_MIGRATION_ID,
|
||||
[
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_FINALIZATION_TABLE_SQL,
|
||||
`CREATE UNIQUE INDEX ql3_ai_prompt_finalization_run_step_uidx
|
||||
ON "ModelInvocationPromptFinalizations" (run_id, step_run_id)`,
|
||||
`CREATE INDEX ql3_ai_prompt_finalization_status_idx
|
||||
ON "ModelInvocationPromptFinalizations"
|
||||
(run_status, finalized_at_ms, request_id)`,
|
||||
],
|
||||
(context, statement) => context.client.exec(statement),
|
||||
);
|
||||
|
||||
const LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_ARTIFACT_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationPromptOutputArtifacts" (
|
||||
artifact_id TEXT PRIMARY KEY NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
run_id TEXT NOT NULL,
|
||||
step_run_id TEXT NOT NULL,
|
||||
invocation_id TEXT NOT NULL UNIQUE,
|
||||
requested_by_type TEXT NOT NULL,
|
||||
requested_by_id TEXT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
content_digest TEXT NOT NULL,
|
||||
output_bytes INTEGER NOT NULL,
|
||||
retention_policy_revision TEXT NOT NULL,
|
||||
retention_ms INTEGER NOT NULL,
|
||||
retention_policy_digest TEXT NOT NULL,
|
||||
retention_eligible_at_ms INTEGER NOT NULL,
|
||||
key_id TEXT NOT NULL,
|
||||
algorithm TEXT NOT NULL,
|
||||
plaintext_bytes INTEGER NOT NULL,
|
||||
sealed_at_ms INTEGER NOT NULL,
|
||||
artifact_digest TEXT NOT NULL UNIQUE,
|
||||
artifact_json TEXT NOT NULL,
|
||||
CONSTRAINT ql3_ai_prompt_output_artifact_admission_fk
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "ModelInvocationPromptAdmissions" (invocation_id)
|
||||
ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_output_artifact_start_fk
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "ModelInvocationStarts" (invocation_id)
|
||||
ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_output_artifact_step_fk
|
||||
FOREIGN KEY (run_id, step_run_id) REFERENCES "StepRuns" (run_id, id)
|
||||
ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_output_artifact_identity_check CHECK (
|
||||
length(artifact_id) BETWEEN 1 AND 128 AND
|
||||
length(project_id) BETWEEN 1 AND 128 AND
|
||||
length(run_id) BETWEEN 1 AND 128 AND
|
||||
length(step_run_id) BETWEEN 1 AND 128 AND
|
||||
length(invocation_id) BETWEEN 1 AND 128 AND
|
||||
length(requested_by_type) BETWEEN 1 AND 32 AND
|
||||
length(requested_by_id) BETWEEN 1 AND 128 AND
|
||||
length(provider) BETWEEN 1 AND 256 AND
|
||||
length(model) BETWEEN 1 AND 256 AND
|
||||
length(retention_policy_revision) BETWEEN 1 AND 128 AND
|
||||
length(key_id) BETWEEN 1 AND 128 AND
|
||||
algorithm = 'aes-256-gcm'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_output_artifact_value_check CHECK (
|
||||
output_bytes BETWEEN 0 AND 1048576 AND
|
||||
plaintext_bytes BETWEEN 1 AND 1052672 AND
|
||||
retention_ms BETWEEN 3600000 AND 31536000000 AND
|
||||
sealed_at_ms >= 0 AND
|
||||
retention_eligible_at_ms = sealed_at_ms + retention_ms AND
|
||||
length(content_digest) = 64 AND
|
||||
content_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(retention_policy_digest) = 64 AND
|
||||
retention_policy_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(artifact_digest) = 64 AND
|
||||
artifact_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_output_artifact_json_check CHECK (
|
||||
length(CAST(artifact_json AS BLOB)) BETWEEN 2 AND 1572864 AND
|
||||
json_valid(artifact_json) AND json_type(artifact_json) = 'object' AND
|
||||
json_extract(artifact_json, '$.schema') =
|
||||
'qinglong/plugin-package-prompt-output-artifact@v1' AND
|
||||
json_extract(artifact_json, '$.artifactId') = artifact_id AND
|
||||
json_extract(artifact_json, '$.projectId') = project_id AND
|
||||
json_extract(artifact_json, '$.runId') = run_id AND
|
||||
json_extract(artifact_json, '$.stepRunId') = step_run_id AND
|
||||
json_extract(artifact_json, '$.invocationId') = invocation_id AND
|
||||
json_extract(artifact_json, '$.requestedBy.type') = requested_by_type AND
|
||||
json_extract(artifact_json, '$.requestedBy.id') = requested_by_id AND
|
||||
json_extract(artifact_json, '$.provider') = provider AND
|
||||
json_extract(artifact_json, '$.model') = model AND
|
||||
json_extract(artifact_json, '$.contentDigest') = content_digest AND
|
||||
json_extract(artifact_json, '$.outputBytes') = output_bytes AND
|
||||
json_extract(artifact_json, '$.retentionPolicy.revision') =
|
||||
retention_policy_revision AND
|
||||
json_extract(artifact_json, '$.retentionPolicy.retentionMs') =
|
||||
retention_ms AND
|
||||
json_extract(artifact_json, '$.retentionPolicyDigest') =
|
||||
retention_policy_digest AND
|
||||
json_extract(artifact_json, '$.retentionEligibleAtMs') =
|
||||
retention_eligible_at_ms AND
|
||||
json_extract(artifact_json, '$.keyId') = key_id AND
|
||||
json_extract(artifact_json, '$.algorithm') = algorithm AND
|
||||
json_extract(artifact_json, '$.plaintextBytes') = plaintext_bytes AND
|
||||
json_extract(artifact_json, '$.sealedAtMs') = sealed_at_ms AND
|
||||
json_extract(artifact_json, '$.artifactDigest') = artifact_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const localPluginPackagePromptOutputArtifactMigration =
|
||||
defineSqlMigration<LocalMigrationContext>(
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_ARTIFACT_MIGRATION_ID,
|
||||
[
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_ARTIFACT_TABLE_SQL,
|
||||
`CREATE INDEX ql3_ai_prompt_output_artifact_retention_idx
|
||||
ON "ModelInvocationPromptOutputArtifacts"
|
||||
(retention_eligible_at_ms, artifact_id)`,
|
||||
`CREATE INDEX ql3_ai_prompt_output_artifact_project_run_idx
|
||||
ON "ModelInvocationPromptOutputArtifacts"
|
||||
(project_id, run_id, artifact_id)`,
|
||||
],
|
||||
(context, statement) => context.client.exec(statement),
|
||||
);
|
||||
|
||||
const LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_TOMBSTONE_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationPromptOutputArtifactTombstones" (
|
||||
artifact_id TEXT PRIMARY KEY NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
run_id TEXT NOT NULL,
|
||||
step_run_id TEXT NOT NULL,
|
||||
invocation_id TEXT NOT NULL UNIQUE,
|
||||
artifact_digest TEXT NOT NULL UNIQUE,
|
||||
retention_policy_digest TEXT NOT NULL,
|
||||
retention_eligible_at_ms INTEGER NOT NULL,
|
||||
key_id TEXT NOT NULL,
|
||||
tombstoned_at_ms INTEGER NOT NULL,
|
||||
tombstone_digest TEXT NOT NULL UNIQUE,
|
||||
tombstone_json TEXT NOT NULL,
|
||||
CONSTRAINT ql3_ai_prompt_output_tombstone_admission_fk
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "ModelInvocationPromptAdmissions" (invocation_id)
|
||||
ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_output_tombstone_start_fk
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "ModelInvocationStarts" (invocation_id)
|
||||
ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_output_tombstone_step_fk
|
||||
FOREIGN KEY (run_id, step_run_id) REFERENCES "StepRuns" (run_id, id)
|
||||
ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_output_tombstone_identity_check CHECK (
|
||||
length(artifact_id) BETWEEN 1 AND 128 AND
|
||||
length(project_id) BETWEEN 1 AND 128 AND
|
||||
length(run_id) BETWEEN 1 AND 128 AND
|
||||
length(step_run_id) BETWEEN 1 AND 128 AND
|
||||
length(invocation_id) BETWEEN 1 AND 128 AND
|
||||
length(key_id) BETWEEN 1 AND 128
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_output_tombstone_value_check CHECK (
|
||||
retention_eligible_at_ms >= 0 AND
|
||||
tombstoned_at_ms >= retention_eligible_at_ms AND
|
||||
length(artifact_digest) = 64 AND
|
||||
artifact_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(retention_policy_digest) = 64 AND
|
||||
retention_policy_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(tombstone_digest) = 64 AND
|
||||
tombstone_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_output_tombstone_json_check CHECK (
|
||||
length(CAST(tombstone_json AS BLOB)) BETWEEN 2 AND 8192 AND
|
||||
json_valid(tombstone_json) AND json_type(tombstone_json) = 'object' AND
|
||||
json_extract(tombstone_json, '$.schema') =
|
||||
'qinglong/plugin-package-prompt-output-artifact-tombstone@v1' AND
|
||||
json_extract(tombstone_json, '$.reference.artifactId') = artifact_id AND
|
||||
json_extract(tombstone_json, '$.reference.projectId') = project_id AND
|
||||
json_extract(tombstone_json, '$.reference.runId') = run_id AND
|
||||
json_extract(tombstone_json, '$.reference.stepRunId') = step_run_id AND
|
||||
json_extract(tombstone_json, '$.reference.invocationId') = invocation_id AND
|
||||
json_extract(tombstone_json, '$.reference.artifactDigest') = artifact_digest AND
|
||||
json_extract(tombstone_json, '$.reference.retentionPolicyDigest') =
|
||||
retention_policy_digest AND
|
||||
json_extract(tombstone_json, '$.reference.retentionEligibleAtMs') =
|
||||
retention_eligible_at_ms AND
|
||||
json_extract(tombstone_json, '$.reference.keyId') = key_id AND
|
||||
json_extract(tombstone_json, '$.tombstonedAtMs') = tombstoned_at_ms AND
|
||||
json_extract(tombstone_json, '$.tombstoneDigest') = tombstone_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const localPluginPackagePromptOutputTombstoneMigration =
|
||||
defineSqlMigration<LocalMigrationContext>(
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_TOMBSTONE_MIGRATION_ID,
|
||||
[
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_TOMBSTONE_TABLE_SQL,
|
||||
`CREATE INDEX ql3_ai_prompt_output_tombstone_time_idx
|
||||
ON "ModelInvocationPromptOutputArtifactTombstones"
|
||||
(tombstoned_at_ms, artifact_id)`,
|
||||
`CREATE INDEX ql3_ai_prompt_output_tombstone_project_run_idx
|
||||
ON "ModelInvocationPromptOutputArtifactTombstones"
|
||||
(project_id, run_id, artifact_id)`,
|
||||
],
|
||||
(context, statement) => context.client.exec(statement),
|
||||
);
|
||||
|
||||
const LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_PREPARATION_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationPromptOutputKeyRetirementPreparations" (
|
||||
key_id TEXT PRIMARY KEY NOT NULL,
|
||||
retirement_id TEXT NOT NULL UNIQUE,
|
||||
request_id TEXT NOT NULL UNIQUE,
|
||||
mutation_id TEXT NOT NULL UNIQUE,
|
||||
catalog_digest TEXT NOT NULL,
|
||||
material_proof TEXT NOT NULL,
|
||||
prepared_at_ms INTEGER NOT NULL,
|
||||
preparation_digest TEXT NOT NULL UNIQUE,
|
||||
preparation_json TEXT NOT NULL,
|
||||
CONSTRAINT ql3_ai_prompt_output_key_retirement_preparation_identity_check CHECK (
|
||||
length(key_id) BETWEEN 1 AND 128 AND
|
||||
length(retirement_id) BETWEEN 1 AND 128 AND
|
||||
length(request_id) BETWEEN 1 AND 128 AND
|
||||
length(mutation_id) BETWEEN 1 AND 128
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_output_key_retirement_preparation_value_check CHECK (
|
||||
prepared_at_ms >= 0 AND
|
||||
length(catalog_digest) = 64 AND catalog_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(material_proof) = 64 AND material_proof NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(preparation_digest) = 64 AND
|
||||
preparation_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_output_key_retirement_preparation_json_check CHECK (
|
||||
length(CAST(preparation_json AS BLOB)) BETWEEN 2 AND 8192 AND
|
||||
json_valid(preparation_json) AND json_type(preparation_json) = 'object' AND
|
||||
json_extract(preparation_json, '$.schema') =
|
||||
'qinglong/plugin-package-prompt-output-key-retirement-preparation@v1' AND
|
||||
json_extract(preparation_json, '$.keyId') = key_id AND
|
||||
json_extract(preparation_json, '$.retirementId') = retirement_id AND
|
||||
json_extract(preparation_json, '$.requestId') = request_id AND
|
||||
json_extract(preparation_json, '$.mutationId') = mutation_id AND
|
||||
json_extract(preparation_json, '$.catalogDigest') = catalog_digest AND
|
||||
json_extract(preparation_json, '$.materialProof') = material_proof AND
|
||||
json_extract(preparation_json, '$.preparedAtMs') = prepared_at_ms AND
|
||||
json_extract(preparation_json, '$.preparationDigest') = preparation_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_COMPLETION_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationPromptOutputKeyRetirementCompletions" (
|
||||
key_id TEXT PRIMARY KEY NOT NULL,
|
||||
retirement_id TEXT NOT NULL UNIQUE,
|
||||
request_id TEXT NOT NULL UNIQUE,
|
||||
mutation_id TEXT NOT NULL UNIQUE,
|
||||
preparation_digest TEXT NOT NULL UNIQUE,
|
||||
retired_catalog_digest TEXT NOT NULL,
|
||||
absence_proof TEXT NOT NULL,
|
||||
completed_at_ms INTEGER NOT NULL,
|
||||
completion_digest TEXT NOT NULL UNIQUE,
|
||||
completion_json TEXT NOT NULL,
|
||||
CONSTRAINT ql3_ai_prompt_output_key_retirement_completion_preparation_fk
|
||||
FOREIGN KEY (key_id, preparation_digest)
|
||||
REFERENCES "ModelInvocationPromptOutputKeyRetirementPreparations"
|
||||
(key_id, preparation_digest)
|
||||
ON DELETE RESTRICT ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_ai_prompt_output_key_retirement_completion_identity_check CHECK (
|
||||
length(key_id) BETWEEN 1 AND 128 AND
|
||||
length(retirement_id) BETWEEN 1 AND 128 AND
|
||||
length(request_id) BETWEEN 1 AND 128 AND
|
||||
length(mutation_id) BETWEEN 1 AND 128
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_output_key_retirement_completion_value_check CHECK (
|
||||
completed_at_ms >= 0 AND
|
||||
length(preparation_digest) = 64 AND
|
||||
preparation_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(retired_catalog_digest) = 64 AND
|
||||
retired_catalog_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(absence_proof) = 64 AND absence_proof NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(completion_digest) = 64 AND completion_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_ai_prompt_output_key_retirement_completion_json_check CHECK (
|
||||
length(CAST(completion_json AS BLOB)) BETWEEN 2 AND 8192 AND
|
||||
json_valid(completion_json) AND json_type(completion_json) = 'object' AND
|
||||
json_extract(completion_json, '$.schema') =
|
||||
'qinglong/plugin-package-prompt-output-key-retirement-completion@v1' AND
|
||||
json_extract(completion_json, '$.keyId') = key_id AND
|
||||
json_extract(completion_json, '$.retirementId') = retirement_id AND
|
||||
json_extract(completion_json, '$.requestId') = request_id AND
|
||||
json_extract(completion_json, '$.mutationId') = mutation_id AND
|
||||
json_extract(completion_json, '$.preparationDigest') = preparation_digest AND
|
||||
json_extract(completion_json, '$.retiredCatalogDigest') =
|
||||
retired_catalog_digest AND
|
||||
json_extract(completion_json, '$.absenceProof') = absence_proof AND
|
||||
json_extract(completion_json, '$.completedAtMs') = completed_at_ms AND
|
||||
json_extract(completion_json, '$.completionDigest') = completion_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const localPluginPackagePromptOutputKeyRetirementMigration =
|
||||
defineSqlMigration<LocalMigrationContext>(
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_MIGRATION_ID,
|
||||
[
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_PREPARATION_TABLE_SQL,
|
||||
`CREATE UNIQUE INDEX ql3_ai_prompt_output_key_retirement_preparation_fk_uidx
|
||||
ON "ModelInvocationPromptOutputKeyRetirementPreparations"
|
||||
(key_id, preparation_digest)`,
|
||||
LOCAL_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_RETIREMENT_COMPLETION_TABLE_SQL,
|
||||
`CREATE INDEX ql3_ai_prompt_output_key_retirement_completion_time_idx
|
||||
ON "ModelInvocationPromptOutputKeyRetirementCompletions"
|
||||
(completed_at_ms, key_id)`,
|
||||
],
|
||||
(context, statement) => context.client.exec(statement),
|
||||
);
|
||||
|
||||
export const sqlitePromptMigrations = Object.freeze([
|
||||
localPluginPackagePromptAdmissionMigration,
|
||||
localPluginPackagePromptFinalizationMigration,
|
||||
localPluginPackagePromptOutputArtifactMigration,
|
||||
localPluginPackagePromptOutputTombstoneMigration,
|
||||
localPluginPackagePromptOutputKeyRetirementMigration,
|
||||
]);
|
||||
@@ -0,0 +1,374 @@
|
||||
import type { LocalMigrationContext } from './context';
|
||||
|
||||
import {
|
||||
LOCAL_MODEL_INVOCATION_USAGE_MIGRATION_ID,
|
||||
LOCAL_MODEL_INVOCATION_QUOTA_MIGRATION_ID,
|
||||
LOCAL_MODEL_INVOCATION_PRICING_MIGRATION_ID,
|
||||
} from '../identities';
|
||||
|
||||
import { defineSqlMigration } from '../shared';
|
||||
|
||||
const LOCAL_USAGE_LEDGER_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationUsageLedger" (
|
||||
invocation_id TEXT PRIMARY KEY NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
run_id TEXT NOT NULL,
|
||||
step_run_id TEXT NOT NULL,
|
||||
trace_id TEXT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
policy_revision TEXT NOT NULL,
|
||||
completion_digest TEXT NOT NULL,
|
||||
outcome TEXT NOT NULL,
|
||||
settled_at_ms INTEGER NOT NULL,
|
||||
input_bytes INTEGER NOT NULL,
|
||||
output_bytes INTEGER NOT NULL,
|
||||
input_tokens INTEGER NOT NULL,
|
||||
output_tokens INTEGER NOT NULL,
|
||||
total_tokens INTEGER NOT NULL,
|
||||
cost_micros INTEGER,
|
||||
ledger_digest TEXT NOT NULL,
|
||||
record_json TEXT NOT NULL,
|
||||
FOREIGN KEY (invocation_id, completion_digest)
|
||||
REFERENCES "ModelInvocationCompletions"
|
||||
(invocation_id, completion_digest) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_invocation_usage_identity_check CHECK (
|
||||
length(invocation_id) BETWEEN 1 AND 128 AND
|
||||
length(project_id) BETWEEN 1 AND 128 AND
|
||||
length(run_id) BETWEEN 1 AND 128 AND
|
||||
length(step_run_id) BETWEEN 1 AND 128 AND
|
||||
length(trace_id) BETWEEN 1 AND 128 AND
|
||||
length(provider) BETWEEN 1 AND 128 AND
|
||||
length(model) BETWEEN 1 AND 128 AND
|
||||
length(policy_revision) BETWEEN 1 AND 128
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_usage_value_check CHECK (
|
||||
outcome IN ('succeeded', 'failed', 'timed_out', 'outcome_unknown') AND
|
||||
settled_at_ms >= 0 AND
|
||||
input_bytes BETWEEN 1 AND 262144 AND
|
||||
output_bytes BETWEEN 0 AND 1048576 AND
|
||||
input_tokens >= 0 AND output_tokens >= 0 AND
|
||||
total_tokens = input_tokens + output_tokens AND
|
||||
(cost_micros IS NULL OR cost_micros >= 0) AND
|
||||
length(completion_digest) = 64 AND
|
||||
completion_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(ledger_digest) = 64 AND
|
||||
ledger_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_usage_json_check CHECK (
|
||||
length(CAST(record_json AS BLOB)) BETWEEN 2 AND 24576 AND
|
||||
json_valid(record_json) AND json_type(record_json) = 'object' AND
|
||||
json_extract(record_json, '$.schema') =
|
||||
'qinglong/model-invocation-usage-ledger@v1' AND
|
||||
json_extract(record_json, '$.invocationId') = invocation_id AND
|
||||
json_extract(record_json, '$.projectId') = project_id AND
|
||||
json_extract(record_json, '$.runId') = run_id AND
|
||||
json_extract(record_json, '$.stepRunId') = step_run_id AND
|
||||
json_extract(record_json, '$.traceId') = trace_id AND
|
||||
json_extract(record_json, '$.provider') = provider AND
|
||||
json_extract(record_json, '$.model') = model AND
|
||||
json_extract(record_json, '$.policyRevision') = policy_revision AND
|
||||
json_extract(record_json, '$.completionDigest') = completion_digest AND
|
||||
json_extract(record_json, '$.outcome') = outcome AND
|
||||
json_extract(record_json, '$.settledAtMs') = settled_at_ms AND
|
||||
json_extract(record_json, '$.inputBytes') = input_bytes AND
|
||||
json_extract(record_json, '$.outputBytes') = output_bytes AND
|
||||
json_extract(record_json, '$.inputTokens') = input_tokens AND
|
||||
json_extract(record_json, '$.outputTokens') = output_tokens AND
|
||||
json_extract(record_json, '$.totalTokens') = total_tokens AND
|
||||
((cost_micros IS NULL AND
|
||||
json_type(record_json, '$.costMicros') = 'null') OR
|
||||
json_extract(record_json, '$.costMicros') = cost_micros) AND
|
||||
json_extract(record_json, '$.ledgerDigest') = ledger_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const localUsageMigration = defineSqlMigration<LocalMigrationContext>(
|
||||
LOCAL_MODEL_INVOCATION_USAGE_MIGRATION_ID,
|
||||
[
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_completions_identity_digest_uidx
|
||||
ON "ModelInvocationCompletions" (invocation_id, completion_digest)`,
|
||||
LOCAL_USAGE_LEDGER_TABLE_SQL,
|
||||
`CREATE INDEX ql3_model_invocation_usage_project_time_idx
|
||||
ON "ModelInvocationUsageLedger"
|
||||
(project_id, settled_at_ms, invocation_id)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_usage_digest_uidx
|
||||
ON "ModelInvocationUsageLedger" (ledger_digest)`,
|
||||
],
|
||||
(context, statement) => context.client.exec(statement),
|
||||
);
|
||||
|
||||
const LOCAL_QUOTA_RESERVATION_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationQuotaReservations" (
|
||||
invocation_id TEXT PRIMARY KEY NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
model_policy_revision TEXT NOT NULL,
|
||||
quota_policy_revision TEXT NOT NULL,
|
||||
window_ms INTEGER NOT NULL,
|
||||
window_start_ms INTEGER NOT NULL,
|
||||
window_end_ms INTEGER NOT NULL,
|
||||
max_invocations INTEGER NOT NULL,
|
||||
max_tokens INTEGER NOT NULL,
|
||||
max_cost_micros INTEGER,
|
||||
reserved_tokens INTEGER NOT NULL,
|
||||
reserved_cost_micros INTEGER,
|
||||
reserved_at_ms INTEGER NOT NULL,
|
||||
admission_digest TEXT NOT NULL,
|
||||
reservation_digest TEXT NOT NULL,
|
||||
record_json TEXT NOT NULL,
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "ModelInvocationStarts" (invocation_id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_invocation_quota_reservation_value_check CHECK (
|
||||
window_ms IN (60000, 3600000, 86400000) AND
|
||||
window_start_ms >= 0 AND window_end_ms = window_start_ms + window_ms AND
|
||||
reserved_at_ms >= window_start_ms AND reserved_at_ms < window_end_ms AND
|
||||
max_invocations BETWEEN 1 AND 100000 AND
|
||||
max_tokens BETWEEN 1 AND 1000000000000 AND
|
||||
(max_cost_micros IS NULL OR
|
||||
max_cost_micros BETWEEN 0 AND 1000000000000000) AND
|
||||
reserved_tokens BETWEEN 1 AND 1000000000000 AND
|
||||
(reserved_cost_micros IS NULL OR
|
||||
reserved_cost_micros BETWEEN 0 AND 1000000000000000) AND
|
||||
(max_cost_micros IS NULL OR reserved_cost_micros IS NOT NULL) AND
|
||||
length(admission_digest) = 64 AND
|
||||
admission_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(reservation_digest) = 64 AND
|
||||
reservation_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_quota_reservation_json_check CHECK (
|
||||
length(CAST(record_json AS BLOB)) BETWEEN 2 AND 24576 AND
|
||||
json_valid(record_json) AND json_type(record_json) = 'object' AND
|
||||
json_extract(record_json, '$.schema') =
|
||||
'qinglong/model-invocation-quota-reservation@v1' AND
|
||||
json_extract(record_json, '$.invocationId') = invocation_id AND
|
||||
json_extract(record_json, '$.projectId') = project_id AND
|
||||
json_extract(record_json, '$.modelPolicyRevision') =
|
||||
model_policy_revision AND
|
||||
json_extract(record_json, '$.quotaPolicyRevision') =
|
||||
quota_policy_revision AND
|
||||
json_extract(record_json, '$.windowMs') = window_ms AND
|
||||
json_extract(record_json, '$.windowStartMs') = window_start_ms AND
|
||||
json_extract(record_json, '$.windowEndMs') = window_end_ms AND
|
||||
json_extract(record_json, '$.maxInvocations') = max_invocations AND
|
||||
json_extract(record_json, '$.maxTokens') = max_tokens AND
|
||||
((max_cost_micros IS NULL AND
|
||||
json_type(record_json, '$.maxCostMicros') = 'null') OR
|
||||
json_extract(record_json, '$.maxCostMicros') = max_cost_micros) AND
|
||||
json_extract(record_json, '$.reservedTokens') = reserved_tokens AND
|
||||
((reserved_cost_micros IS NULL AND
|
||||
json_type(record_json, '$.reservedCostMicros') = 'null') OR
|
||||
json_extract(record_json, '$.reservedCostMicros') =
|
||||
reserved_cost_micros) AND
|
||||
json_extract(record_json, '$.reservedAtMs') = reserved_at_ms AND
|
||||
json_extract(record_json, '$.admissionDigest') = admission_digest AND
|
||||
json_extract(record_json, '$.reservationDigest') = reservation_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const LOCAL_QUOTA_SETTLEMENT_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationQuotaSettlements" (
|
||||
invocation_id TEXT PRIMARY KEY NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
reservation_digest TEXT NOT NULL,
|
||||
completion_digest TEXT NOT NULL,
|
||||
effective_tokens INTEGER NOT NULL,
|
||||
effective_cost_micros INTEGER,
|
||||
retained_token_reservation INTEGER NOT NULL,
|
||||
retained_cost_reservation INTEGER NOT NULL,
|
||||
settled_at_ms INTEGER NOT NULL,
|
||||
settlement_digest TEXT NOT NULL,
|
||||
record_json TEXT NOT NULL,
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "ModelInvocationQuotaReservations" (invocation_id)
|
||||
ON DELETE RESTRICT,
|
||||
FOREIGN KEY (invocation_id, completion_digest)
|
||||
REFERENCES "ModelInvocationCompletions"
|
||||
(invocation_id, completion_digest) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_invocation_quota_settlement_value_check CHECK (
|
||||
effective_tokens BETWEEN 0 AND 1000000000000 AND
|
||||
(effective_cost_micros IS NULL OR
|
||||
effective_cost_micros BETWEEN 0 AND 1000000000000000) AND
|
||||
retained_token_reservation IN (0, 1) AND
|
||||
retained_cost_reservation IN (0, 1) AND
|
||||
settled_at_ms >= 0 AND
|
||||
length(reservation_digest) = 64 AND
|
||||
reservation_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(completion_digest) = 64 AND
|
||||
completion_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(settlement_digest) = 64 AND
|
||||
settlement_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_quota_settlement_json_check CHECK (
|
||||
length(CAST(record_json AS BLOB)) BETWEEN 2 AND 24576 AND
|
||||
json_valid(record_json) AND json_type(record_json) = 'object' AND
|
||||
json_extract(record_json, '$.schema') =
|
||||
'qinglong/model-invocation-quota-settlement@v1' AND
|
||||
json_extract(record_json, '$.invocationId') = invocation_id AND
|
||||
json_extract(record_json, '$.projectId') = project_id AND
|
||||
json_extract(record_json, '$.reservationDigest') =
|
||||
reservation_digest AND
|
||||
json_extract(record_json, '$.completionDigest') = completion_digest AND
|
||||
json_extract(record_json, '$.effectiveTokens') = effective_tokens AND
|
||||
((effective_cost_micros IS NULL AND
|
||||
json_type(record_json, '$.effectiveCostMicros') = 'null') OR
|
||||
json_extract(record_json, '$.effectiveCostMicros') =
|
||||
effective_cost_micros) AND
|
||||
json_extract(record_json, '$.retainedTokenReservation') =
|
||||
retained_token_reservation AND
|
||||
json_extract(record_json, '$.retainedCostReservation') =
|
||||
retained_cost_reservation AND
|
||||
json_extract(record_json, '$.settledAtMs') = settled_at_ms AND
|
||||
json_extract(record_json, '$.settlementDigest') = settlement_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const localQuotaMigration = defineSqlMigration<LocalMigrationContext>(
|
||||
LOCAL_MODEL_INVOCATION_QUOTA_MIGRATION_ID,
|
||||
[
|
||||
LOCAL_QUOTA_RESERVATION_TABLE_SQL,
|
||||
`CREATE INDEX ql3_model_invocation_quota_reservation_window_idx
|
||||
ON "ModelInvocationQuotaReservations"
|
||||
(project_id, window_start_ms, invocation_id)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_quota_reservation_digest_uidx
|
||||
ON "ModelInvocationQuotaReservations" (reservation_digest)`,
|
||||
LOCAL_QUOTA_SETTLEMENT_TABLE_SQL,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_quota_settlement_digest_uidx
|
||||
ON "ModelInvocationQuotaSettlements" (settlement_digest)`,
|
||||
],
|
||||
(context, statement) => context.client.exec(statement),
|
||||
);
|
||||
|
||||
const LOCAL_PRICE_QUOTE_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationPriceQuotes" (
|
||||
invocation_id TEXT PRIMARY KEY NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
model_policy_revision TEXT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
price_revision TEXT NOT NULL,
|
||||
currency TEXT NOT NULL,
|
||||
input_micros_per_million_tokens INTEGER NOT NULL,
|
||||
output_micros_per_million_tokens INTEGER NOT NULL,
|
||||
max_total_tokens INTEGER NOT NULL,
|
||||
max_output_tokens INTEGER NOT NULL,
|
||||
reserved_cost_micros INTEGER NOT NULL,
|
||||
catalog_digest TEXT NOT NULL,
|
||||
quote_digest TEXT NOT NULL,
|
||||
record_json TEXT NOT NULL,
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "ModelInvocationStarts" (invocation_id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_invocation_price_quote_identity_check CHECK (
|
||||
length(project_id) BETWEEN 1 AND 128 AND
|
||||
length(model_policy_revision) BETWEEN 1 AND 128 AND
|
||||
length(provider) BETWEEN 1 AND 128 AND
|
||||
length(model) BETWEEN 1 AND 128 AND
|
||||
length(price_revision) BETWEEN 1 AND 128
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_price_quote_value_check CHECK (
|
||||
currency = 'USD' AND
|
||||
input_micros_per_million_tokens BETWEEN 0 AND 1000000000000 AND
|
||||
output_micros_per_million_tokens BETWEEN 0 AND 1000000000000 AND
|
||||
max_total_tokens BETWEEN 1 AND 1000000000000 AND
|
||||
max_output_tokens BETWEEN 1 AND max_total_tokens AND
|
||||
reserved_cost_micros BETWEEN 0 AND 1000000000000000 AND
|
||||
length(catalog_digest) = 64 AND
|
||||
catalog_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(quote_digest) = 64 AND
|
||||
quote_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_price_quote_json_check CHECK (
|
||||
length(CAST(record_json AS BLOB)) BETWEEN 2 AND 24576 AND
|
||||
json_valid(record_json) AND json_type(record_json) = 'object' AND
|
||||
json_extract(record_json, '$.schema') =
|
||||
'qinglong/model-invocation-price-quote@v1' AND
|
||||
json_extract(record_json, '$.invocationId') = invocation_id AND
|
||||
json_extract(record_json, '$.projectId') = project_id AND
|
||||
json_extract(record_json, '$.modelPolicyRevision') =
|
||||
model_policy_revision AND
|
||||
json_extract(record_json, '$.provider') = provider AND
|
||||
json_extract(record_json, '$.model') = model AND
|
||||
json_extract(record_json, '$.priceRevision') = price_revision AND
|
||||
json_extract(record_json, '$.currency') = currency AND
|
||||
json_extract(record_json, '$.inputMicrosPerMillionTokens') =
|
||||
input_micros_per_million_tokens AND
|
||||
json_extract(record_json, '$.outputMicrosPerMillionTokens') =
|
||||
output_micros_per_million_tokens AND
|
||||
json_extract(record_json, '$.maxTotalTokens') = max_total_tokens AND
|
||||
json_extract(record_json, '$.maxOutputTokens') = max_output_tokens AND
|
||||
json_extract(record_json, '$.reservedCostMicros') =
|
||||
reserved_cost_micros AND
|
||||
json_extract(record_json, '$.catalogDigest') = catalog_digest AND
|
||||
json_extract(record_json, '$.quoteDigest') = quote_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const LOCAL_PRICE_SETTLEMENT_TABLE_SQL = `
|
||||
CREATE TABLE "ModelInvocationPriceSettlements" (
|
||||
invocation_id TEXT PRIMARY KEY NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
quote_digest TEXT NOT NULL,
|
||||
completion_digest TEXT NOT NULL,
|
||||
currency TEXT NOT NULL,
|
||||
input_tokens INTEGER NOT NULL,
|
||||
output_tokens INTEGER NOT NULL,
|
||||
cost_micros INTEGER NOT NULL,
|
||||
settled_at_ms INTEGER NOT NULL,
|
||||
settlement_digest TEXT NOT NULL,
|
||||
record_json TEXT NOT NULL,
|
||||
FOREIGN KEY (invocation_id, quote_digest)
|
||||
REFERENCES "ModelInvocationPriceQuotes"
|
||||
(invocation_id, quote_digest) ON DELETE RESTRICT,
|
||||
FOREIGN KEY (invocation_id, completion_digest)
|
||||
REFERENCES "ModelInvocationCompletions"
|
||||
(invocation_id, completion_digest) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_model_invocation_price_settlement_value_check CHECK (
|
||||
currency = 'USD' AND
|
||||
input_tokens >= 0 AND output_tokens >= 0 AND
|
||||
cost_micros BETWEEN 0 AND 1000000000000000 AND
|
||||
settled_at_ms >= 0 AND
|
||||
length(quote_digest) = 64 AND
|
||||
quote_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(completion_digest) = 64 AND
|
||||
completion_digest NOT GLOB '*[^0-9a-f]*' AND
|
||||
length(settlement_digest) = 64 AND
|
||||
settlement_digest NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
CONSTRAINT ql3_model_invocation_price_settlement_json_check CHECK (
|
||||
length(CAST(record_json AS BLOB)) BETWEEN 2 AND 24576 AND
|
||||
json_valid(record_json) AND json_type(record_json) = 'object' AND
|
||||
json_extract(record_json, '$.schema') =
|
||||
'qinglong/model-invocation-price-settlement@v1' AND
|
||||
json_extract(record_json, '$.invocationId') = invocation_id AND
|
||||
json_extract(record_json, '$.projectId') = project_id AND
|
||||
json_extract(record_json, '$.quoteDigest') = quote_digest AND
|
||||
json_extract(record_json, '$.completionDigest') =
|
||||
completion_digest AND
|
||||
json_extract(record_json, '$.currency') = currency AND
|
||||
json_extract(record_json, '$.inputTokens') = input_tokens AND
|
||||
json_extract(record_json, '$.outputTokens') = output_tokens AND
|
||||
json_extract(record_json, '$.costMicros') = cost_micros AND
|
||||
json_extract(record_json, '$.settledAtMs') = settled_at_ms AND
|
||||
json_extract(record_json, '$.settlementDigest') = settlement_digest
|
||||
)
|
||||
)`;
|
||||
|
||||
const localPricingMigration = defineSqlMigration<LocalMigrationContext>(
|
||||
LOCAL_MODEL_INVOCATION_PRICING_MIGRATION_ID,
|
||||
[
|
||||
LOCAL_PRICE_QUOTE_TABLE_SQL,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_price_quote_identity_digest_uidx
|
||||
ON "ModelInvocationPriceQuotes" (invocation_id, quote_digest)`,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_price_quote_digest_uidx
|
||||
ON "ModelInvocationPriceQuotes" (quote_digest)`,
|
||||
LOCAL_PRICE_SETTLEMENT_TABLE_SQL,
|
||||
`CREATE UNIQUE INDEX ql3_model_invocation_price_settlement_digest_uidx
|
||||
ON "ModelInvocationPriceSettlements" (settlement_digest)`,
|
||||
],
|
||||
(context, statement) => context.client.exec(statement),
|
||||
);
|
||||
|
||||
export const sqliteUsagePricingMigrations = Object.freeze([
|
||||
localUsageMigration,
|
||||
localQuotaMigration,
|
||||
localPricingMigration,
|
||||
]);
|
||||
@@ -0,0 +1,3 @@
|
||||
export * from './model-invocation/identities';
|
||||
export * from './model-invocation/postgres';
|
||||
export * from './model-invocation/sqlite';
|
||||
Reference in New Issue
Block a user