feat(ql3): expose bounded worker session observation

packages/ql3-cluster-postgres/src/migration/migrationManifest.ts

@@ -348,5 +348,10 @@ export const postgresqlMainMigrationManifest: MigrationStreamManifest =
         checksum:
           '2fcac38386581189db63faacff325356f11c4529a8db9cef6be1a1ca706aaf10',
       }),
+      Object.freeze({
+        id: 'pg-0069-worker-session-management-observation',
+        checksum:
+          '1191255575589abc2686b391827607abddb4edb78007245dbaaf45dc1c4e5e8b',
+      }),
     ]),
   });

packages/ql3-cluster-postgres/src/migrations/index.ts

@@ -71,6 +71,7 @@ import { pg0065ApprovedActionManualRecoveryMigration } from '../approved-action/
 import { pg0066CancellationDispatchMigration } from '../run/migrations/pg-0066-cancellation-dispatch';
 import { pg0067CancellationDispatchManagementMigration } from '../run-management/pg-0067-cancellation-dispatch-management';
 import { pg0068CancellationDispatchProjectKeysetMigration } from '../run-management/pg-0068-cancellation-dispatch-project-keyset';
+import { pg0069WorkerSessionManagementObservationMigration } from '../remote-execution/pg-0069-worker-session-management-observation';
 
 export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMigrationContext> =
   Object.freeze({
@@ -147,5 +148,6 @@ export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMi
       pg0066CancellationDispatchMigration,
       pg0067CancellationDispatchManagementMigration,
       pg0068CancellationDispatchProjectKeysetMigration,
+      pg0069WorkerSessionManagementObservationMigration,
     ]),
   });

packages/ql3-cluster-postgres/src/remote-execution/pg-0069-worker-session-management-observation.ts

@@ -0,0 +1,17 @@
+import { CAPABILITIES_V67 } from '../run-management/pg-0068-cancellation-dispatch-project-keyset';
+import { definePostgresSqlMigration } from '../migrations/sqlMigration';
+
+export const CAPABILITIES_V68 = CAPABILITIES_V67.replace(
+  '"worker_session":1}',
+  '"worker_session":1,"worker_session_observation":1}',
+);
+
+export const pg0069WorkerSessionManagementObservationMigration =
+  definePostgresSqlMigration({
+    id: 'pg-0069-worker-session-management-observation',
+    statements: [
+      `REVOKE ALL ON "ql3"."worker_sessions" FROM ql3_worker_credential_manager`,
+      `GRANT SELECT ON "ql3"."worker_sessions" TO ql3_worker_credential_manager`,
+      `DO $ql3$ BEGIN UPDATE "ql3"."schema_capabilities" SET contract_version = 68, migration_id = 'pg-0069-worker-session-management-observation', capabilities = '${CAPABILITIES_V68}'::jsonb, updated_at_ms = floor(extract(epoch FROM transaction_timestamp()) * 1000)::bigint WHERE contract_name = 'control-core' AND contract_version = 67 AND migration_id = 'pg-0068-cancellation-dispatch-project-keyset' AND capabilities = '${CAPABILITIES_V67}'::jsonb; IF NOT FOUND THEN RAISE EXCEPTION 'control-core capability is not at version 67' USING ERRCODE = 'check_violation'; END IF; END $ql3$`,
+    ],
+  });

packages/ql3-cluster-postgres/src/remote-execution/workerSessionObservationRepository.ts

@@ -0,0 +1,188 @@
+import type { PostgresPool } from '@qinglong/runtime-core';
+import {
+  WORKER_SESSION_STATUSES,
+  assertWorkerCapabilitiesSnapshot,
+  assertWorkerConcurrency,
+  assertWorkerId,
+  assertWorkerSessionId,
+  type WorkerSessionRecord,
+  type WorkerSessionStatus,
+} from '@qinglong/runtime-core/worker-session';
+import {
+  MAX_WORKER_SESSION_OBSERVATION_PAGE_SIZE,
+  projectWorkerSessionObservation,
+  summarizeWorkerSessionObservation,
+  type WorkerSessionInspection,
+  type WorkerSessionObservationPage,
+} from '@qinglong/runtime-core/worker-session-observation';
+
+type Row = Record<string, unknown>;
+
+const SELECT_COLUMNS = `
+  worker_id AS "workerId",
+  session_id AS "sessionId",
+  generation AS "generation",
+  status AS "status",
+  version AS "version",
+  capabilities_json AS "capabilitiesJson",
+  capabilities_hash AS "capabilitiesHash",
+  max_concurrent_runs AS "maxConcurrentRuns",
+  available_slots AS "availableSlots",
+  registered_at_ms AS "registeredAtMs",
+  last_heartbeat_at_ms AS "lastHeartbeatAtMs",
+  lease_expires_at_ms AS "leaseExpiresAtMs",
+  updated_at_ms AS "updatedAtMs"
+`.trim();
+
+function string(row: Row, key: string): string {
+  const value = row[key];
+  if (typeof value !== 'string' || value.length === 0) {
+    throw new TypeError(`PostgreSQL Worker observation ${key} is invalid`);
+  }
+  return value;
+}
+
+function integer(row: Row, key: string): number {
+  const value = row[key];
+  const normalized =
+    typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value)
+      ? Number(value)
+      : value;
+  if (!Number.isSafeInteger(normalized) || (normalized as number) < 0) {
+    throw new TypeError(`PostgreSQL Worker observation ${key} is invalid`);
+  }
+  return normalized as number;
+}
+
+function record(row: Row): Readonly<WorkerSessionRecord> {
+  const status = string(row, 'status');
+  if (!WORKER_SESSION_STATUSES.includes(status as WorkerSessionStatus)) {
+    throw new TypeError('PostgreSQL Worker observation status is invalid');
+  }
+  const worker: WorkerSessionRecord = Object.freeze({
+    workerId: string(row, 'workerId'),
+    sessionId: string(row, 'sessionId'),
+    generation: integer(row, 'generation'),
+    status: status as WorkerSessionStatus,
+    version: integer(row, 'version'),
+    capabilitiesJson: string(row, 'capabilitiesJson'),
+    capabilitiesHash: string(row, 'capabilitiesHash'),
+    maxConcurrentRuns: integer(row, 'maxConcurrentRuns'),
+    availableSlots: integer(row, 'availableSlots'),
+    registeredAtMs: integer(row, 'registeredAtMs'),
+    lastHeartbeatAtMs: integer(row, 'lastHeartbeatAtMs'),
+    leaseExpiresAtMs: integer(row, 'leaseExpiresAtMs'),
+    updatedAtMs: integer(row, 'updatedAtMs'),
+  });
+  assertWorkerId(worker.workerId);
+  assertWorkerSessionId(worker.sessionId);
+  assertWorkerCapabilitiesSnapshot(
+    worker.capabilitiesJson,
+    worker.capabilitiesHash,
+  );
+  assertWorkerConcurrency(worker.maxConcurrentRuns, worker.availableSlots);
+  return worker;
+}
+
+function observedAtMs(row: Row): number {
+  return integer(row, 'observedAtMs');
+}
+
+export class PostgresWorkerSessionObservationRepository {
+  constructor(private readonly pool: PostgresPool) {
+    if (!pool || typeof pool.query !== 'function') {
+      throw new TypeError('PostgreSQL Worker observation Pool is invalid');
+    }
+  }
+
+  async inspect(workerId: string): Promise<Readonly<WorkerSessionInspection>> {
+    assertWorkerId(workerId);
+    const result = await this.pool.query<Row>(
+      `WITH observation AS (
+         SELECT floor(extract(epoch FROM statement_timestamp()) * 1000)::bigint AS observed_at_ms
+       )
+       SELECT observation.observed_at_ms AS "observedAtMs", worker.*
+       FROM observation
+       LEFT JOIN LATERAL (
+         SELECT ${SELECT_COLUMNS}
+         FROM "ql3"."worker_sessions"
+         WHERE worker_id = $1
+         LIMIT 2
+       ) AS worker ON TRUE`,
+      [workerId],
+    );
+    if (result.rows.length !== 1) {
+      throw new TypeError('PostgreSQL Worker inspection violated its bound');
+    }
+    const observed = observedAtMs(result.rows[0]!);
+    if (
+      result.rows[0]!.workerId !== null &&
+      string(result.rows[0]!, 'workerId') !== workerId
+    ) {
+      throw new TypeError('PostgreSQL Worker inspection identity drifted');
+    }
+    return Object.freeze({
+      observedAtMs: observed,
+      worker:
+        result.rows[0]!.workerId === null
+          ? null
+          : projectWorkerSessionObservation(record(result.rows[0]!), observed),
+    });
+  }
+
+  async list(
+    afterWorkerId: string | null,
+  ): Promise<Readonly<WorkerSessionObservationPage>> {
+    if (afterWorkerId !== null) assertWorkerId(afterWorkerId);
+    const result = await this.pool.query<Row>(
+      `WITH observation AS (
+         SELECT floor(extract(epoch FROM statement_timestamp()) * 1000)::bigint AS observed_at_ms
+       ), page AS (
+         SELECT ${SELECT_COLUMNS}
+         FROM "ql3"."worker_sessions"
+         WHERE ($1::varchar IS NULL OR worker_id > $1)
+         ORDER BY worker_id
+         LIMIT $2
+       )
+       SELECT observation.observed_at_ms AS "observedAtMs", page.*
+       FROM observation LEFT JOIN page ON TRUE
+       ORDER BY page."workerId"`,
+      [afterWorkerId, MAX_WORKER_SESSION_OBSERVATION_PAGE_SIZE + 1],
+    );
+    if (
+      result.rows.length < 1 ||
+      result.rows.length > MAX_WORKER_SESSION_OBSERVATION_PAGE_SIZE + 1
+    ) {
+      throw new TypeError('PostgreSQL Worker observation page violated its bound');
+    }
+    const observed = observedAtMs(result.rows[0]!);
+    if (result.rows.some((row) => observedAtMs(row) !== observed)) {
+      throw new TypeError('PostgreSQL Worker observation clock drifted');
+    }
+    const observations =
+      result.rows[0]!.workerId === null
+        ? []
+        : result.rows.map((row) =>
+            projectWorkerSessionObservation(record(row), observed),
+          );
+    for (let index = 0; index < observations.length; index += 1) {
+      const workerId = observations[index]!.workerId;
+      const previous = observations[index - 1]?.workerId ?? afterWorkerId;
+      if (previous !== null && workerId <= previous) {
+        throw new TypeError('PostgreSQL Worker observation page is unordered');
+      }
+    }
+    const page = observations.slice(
+      0,
+      MAX_WORKER_SESSION_OBSERVATION_PAGE_SIZE,
+    );
+    return Object.freeze({
+      observedAtMs: observed,
+      workers: Object.freeze(page.map(summarizeWorkerSessionObservation)),
+      nextCursor:
+        observations.length > MAX_WORKER_SESSION_OBSERVATION_PAGE_SIZE
+          ? page.at(-1)!.workerId
+          : null,
+    });
+  }
+}

packages/ql3-cluster-postgres/src/schema/schemaContract.ts

@@ -21,8 +21,8 @@ export interface PostgresSchemaContractTrigger {
 export interface PostgresSchemaContract {
   readonly schema: 'ql3';
   readonly contractName: 'control-core';
-  readonly contractVersion: 67;
-  readonly migrationId: 'pg-0068-cancellation-dispatch-project-keyset';
+  readonly contractVersion: 68;
+  readonly migrationId: 'pg-0069-worker-session-management-observation';
   readonly minimumServerMajor: 16;
   readonly maximumServerMajor: 18;
   readonly capabilities: Readonly<{
@@ -92,6 +92,7 @@ export interface PostgresSchemaContract {
     tool_invocation_artifact: 1;
     trigger_definition: 1;
     worker_session: 1;
+    worker_session_observation: 1;
     worker_credential: 1;
     worker_credential_delivery: 1;
     worker_credential_execution_receipt: 1;
@@ -121,8 +122,8 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
   Object.freeze({
     schema: 'ql3',
     contractName: 'control-core',
-    contractVersion: 67,
-    migrationId: 'pg-0068-cancellation-dispatch-project-keyset',
+    contractVersion: 68,
+    migrationId: 'pg-0069-worker-session-management-observation',
     minimumServerMajor: 16,
     maximumServerMajor: 18,
     capabilities: Object.freeze({
@@ -192,6 +193,7 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
       tool_invocation_artifact: 1,
       trigger_definition: 1,
       worker_session: 1,
+      worker_session_observation: 1,
       worker_credential: 1,
       worker_credential_delivery: 1,
       worker_credential_execution_receipt: 1,

packages/ql3-cluster-postgres/src/schema/schemaReadiness.ts

@@ -1509,7 +1509,8 @@ const REQUIRED_WORKER_CREDENTIAL_MANAGER_PRIVILEGES: RequiredPrivileges =
           name === 'schema_migrations' ||
             name === 'schema_capabilities' ||
             name === 'projects' ||
-            name === 'project_role_bindings'
+            name === 'project_role_bindings' ||
+            name === 'worker_sessions'
             ? { ...NO_TABLE_PRIVILEGES, select: true }
             : name === 'worker_credential_management_plans' ||
               name === 'security_audit_events'

packages/ql3-cluster-postgres/src/worker-credential/workerCredentialManagementQuotaRepository.ts

@@ -10,6 +10,7 @@ const OPERATIONS = [
   'worker-credential.propose',
   'worker-credential.decide',
   'worker-credential.inspect',
+  'worker-session.observe',
 ] as const;
 type Operation = (typeof OPERATIONS)[number];
 type Row = Record<string, unknown>;
@@ -31,6 +32,7 @@ const DEFAULT_LIMITS: Readonly<Record<Operation, number>> = Object.freeze({
   'worker-credential.propose': 30,
   'worker-credential.decide': 60,
   'worker-credential.inspect': 600,
+  'worker-session.observe': 600,
 });
 const ID = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,254}$/;
 const PROJECT = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;

packages/ql3-cluster-postgres/src/worker-credential/workerCredentialManager.ts

@@ -44,6 +44,7 @@ export { postgresqlMainMigrationManifest } from '../migration/migrationManifest'
 export { PostgresApprovalRequestRepository } from '../approved-action/approvalRequestRepository';
 export { PostgresProjectPolicyRepository } from '../security/projectPolicyRepository';
 export { PostgresWorkerCredentialManagementPlanRepository } from './workerCredentialManagementPlanRepository';
+export { PostgresWorkerSessionObservationRepository } from '../remote-execution/workerSessionObservationRepository';
 export {
   PostgresPluginPackageIdentityKeysetLedgerRepository as PostgresWorkerCredentialManagementIdentityKeysetLedgerRepository,
   type ClusterManagementIdentityAuthority,