mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-22 02:27:44 +08:00
feat(ql3): expose bounded worker session observation
This commit is contained in:
whyour
+97
-1
@@ -3,6 +3,7 @@ import {
|
||||
PostgresApprovalRequestRepository,
|
||||
PostgresProjectPolicyRepository,
|
||||
PostgresWorkerCredentialManagementPlanRepository,
|
||||
PostgresWorkerSessionObservationRepository,
|
||||
} from '@qinglong/cluster-postgres/worker-credential-manager';
|
||||
import type { PostgresPool } from '@qinglong/runtime-core';
|
||||
import {
|
||||
@@ -20,6 +21,10 @@ import {
|
||||
type SecuritySubject,
|
||||
} from '@qinglong/runtime-core/security';
|
||||
import type { SecurityAuditRecord } from '@qinglong/runtime-core/security-audit';
|
||||
import type {
|
||||
WorkerSessionInspection,
|
||||
WorkerSessionObservationPage,
|
||||
} from '@qinglong/runtime-core/worker-session-observation';
|
||||
import {
|
||||
InvalidWorkerCredentialManagementPlanError,
|
||||
MAX_WORKER_CREDENTIAL_MANAGEMENT_PLAN_LIFETIME_MS,
|
||||
@@ -93,7 +98,8 @@ export type WorkerCredentialManagementQuotaOperation =
|
||||
| 'worker-credential.plan'
|
||||
| 'worker-credential.propose'
|
||||
| 'worker-credential.decide'
|
||||
| 'worker-credential.inspect';
|
||||
| 'worker-credential.inspect'
|
||||
| 'worker-session.observe';
|
||||
|
||||
export interface WorkerCredentialManagementQuotaPort {
|
||||
consume(
|
||||
@@ -166,6 +172,20 @@ export interface InspectClusterWorkerCredentialResult {
|
||||
readonly stale: boolean;
|
||||
}
|
||||
|
||||
export interface InspectClusterWorkerSessionRequest {
|
||||
readonly authorityProjectId: string;
|
||||
readonly workerId: string;
|
||||
readonly inspectionId: string;
|
||||
readonly principal: SecurityPrincipal;
|
||||
}
|
||||
|
||||
export interface ListClusterWorkerSessionsRequest {
|
||||
readonly authorityProjectId: string;
|
||||
readonly afterWorkerId: string | null;
|
||||
readonly inspectionId: string;
|
||||
readonly principal: SecurityPrincipal;
|
||||
}
|
||||
|
||||
export interface ClusterWorkerCredentialManagementService {
|
||||
plan(
|
||||
request: PlanClusterWorkerCredentialRequest,
|
||||
@@ -179,6 +199,12 @@ export interface ClusterWorkerCredentialManagementService {
|
||||
inspectAuthorized(
|
||||
request: InspectClusterWorkerCredentialRequest,
|
||||
): Promise<Readonly<InspectClusterWorkerCredentialResult>>;
|
||||
inspectSession(
|
||||
request: InspectClusterWorkerSessionRequest,
|
||||
): Promise<Readonly<WorkerSessionInspection>>;
|
||||
listSessions(
|
||||
request: ListClusterWorkerSessionsRequest,
|
||||
): Promise<Readonly<WorkerSessionObservationPage>>;
|
||||
}
|
||||
|
||||
export interface ClusterWorkerCredentialManagementOptions {
|
||||
@@ -324,6 +350,7 @@ export function createClusterWorkerCredentialManagementService(
|
||||
const plans = new PostgresWorkerCredentialManagementPlanRepository(
|
||||
options.pool,
|
||||
);
|
||||
const sessions = new PostgresWorkerSessionObservationRepository(options.pool);
|
||||
const approvals = new PostgresApprovalRequestRepository(options.pool);
|
||||
const policy = new ProjectPolicyEngine(
|
||||
new PostgresProjectPolicyRepository(options.pool),
|
||||
@@ -796,5 +823,74 @@ export function createClusterWorkerCredentialManagementService(
|
||||
observedAtMs > plan.expiresAtMs,
|
||||
});
|
||||
},
|
||||
|
||||
async inspectSession(request: InspectClusterWorkerSessionRequest) {
|
||||
exact(
|
||||
request,
|
||||
['authorityProjectId', 'inspectionId', 'principal', 'workerId'],
|
||||
'Session inspection request',
|
||||
);
|
||||
const projectId = identifier(
|
||||
request.authorityProjectId,
|
||||
'authorityProjectId',
|
||||
);
|
||||
const inspectionId = identifier(request.inspectionId, 'inspectionId');
|
||||
const workerId = identifier(request.workerId, 'workerId');
|
||||
const authorization = await authorize(
|
||||
request.principal,
|
||||
projectId,
|
||||
'worker.manage',
|
||||
currentTime(now),
|
||||
);
|
||||
await consumeQuota(
|
||||
projectId,
|
||||
authorization.principal,
|
||||
'worker-session.observe',
|
||||
`session-inspect:${inspectionId}`,
|
||||
);
|
||||
try {
|
||||
return await sessions.inspect(workerId);
|
||||
} catch (error) {
|
||||
throw new WorkerCredentialManagementUnavailableError({
|
||||
cause: error instanceof Error ? error : undefined,
|
||||
});
|
||||
}
|
||||
},
|
||||
|
||||
async listSessions(request: ListClusterWorkerSessionsRequest) {
|
||||
exact(
|
||||
request,
|
||||
['afterWorkerId', 'authorityProjectId', 'inspectionId', 'principal'],
|
||||
'Session list request',
|
||||
);
|
||||
const projectId = identifier(
|
||||
request.authorityProjectId,
|
||||
'authorityProjectId',
|
||||
);
|
||||
const inspectionId = identifier(request.inspectionId, 'inspectionId');
|
||||
const afterWorkerId =
|
||||
request.afterWorkerId === null
|
||||
? null
|
||||
: identifier(request.afterWorkerId, 'afterWorkerId');
|
||||
const authorization = await authorize(
|
||||
request.principal,
|
||||
projectId,
|
||||
'worker.manage',
|
||||
currentTime(now),
|
||||
);
|
||||
await consumeQuota(
|
||||
projectId,
|
||||
authorization.principal,
|
||||
'worker-session.observe',
|
||||
`session-list:${inspectionId}`,
|
||||
);
|
||||
try {
|
||||
return await sessions.list(afterWorkerId);
|
||||
} catch (error) {
|
||||
throw new WorkerCredentialManagementUnavailableError({
|
||||
cause: error instanceof Error ? error : undefined,
|
||||
});
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
+81
-1
@@ -74,11 +74,33 @@ export interface InspectClusterWorkerCredentialCommand {
|
||||
};
|
||||
}
|
||||
|
||||
export interface InspectClusterWorkerSessionCommand {
|
||||
readonly schemaVersion: 1;
|
||||
readonly operation: 'worker-session.inspect';
|
||||
readonly request: {
|
||||
readonly authorityProjectId: string;
|
||||
readonly workerId: string;
|
||||
readonly inspectionId: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ListClusterWorkerSessionsCommand {
|
||||
readonly schemaVersion: 1;
|
||||
readonly operation: 'worker-session.list';
|
||||
readonly request: {
|
||||
readonly authorityProjectId: string;
|
||||
readonly afterWorkerId: string | null;
|
||||
readonly inspectionId: string;
|
||||
};
|
||||
}
|
||||
|
||||
export type ClusterWorkerCredentialManagementCommand =
|
||||
| PlanClusterWorkerCredentialCommand
|
||||
| ProposeClusterWorkerCredentialCommand
|
||||
| DecideClusterWorkerCredentialCommand
|
||||
| InspectClusterWorkerCredentialCommand;
|
||||
| InspectClusterWorkerCredentialCommand
|
||||
| InspectClusterWorkerSessionCommand
|
||||
| ListClusterWorkerSessionsCommand;
|
||||
|
||||
type PlanSummary = ReturnType<typeof planSummary>;
|
||||
type ApprovalSummary = ReturnType<typeof approvalSummary>;
|
||||
@@ -109,6 +131,23 @@ export type ClusterWorkerCredentialManagementTransportResult =
|
||||
plan: PlanSummary | null;
|
||||
approval: ApprovalSummary | null;
|
||||
stale: boolean;
|
||||
}>
|
||||
| Readonly<{
|
||||
schemaVersion: 1;
|
||||
operation: 'worker-session.inspect';
|
||||
observedAtMs: number;
|
||||
worker: Awaited<
|
||||
ReturnType<ClusterWorkerCredentialManagementService['inspectSession']>
|
||||
>['worker'];
|
||||
}>
|
||||
| Readonly<{
|
||||
schemaVersion: 1;
|
||||
operation: 'worker-session.list';
|
||||
observedAtMs: number;
|
||||
workers: Awaited<
|
||||
ReturnType<ClusterWorkerCredentialManagementService['listSessions']>
|
||||
>['workers'];
|
||||
nextCursor: string | null;
|
||||
}>;
|
||||
|
||||
export interface ClusterWorkerCredentialManagementTransport {
|
||||
@@ -254,6 +293,20 @@ export function normalizeClusterWorkerCredentialManagementCommand(
|
||||
'inspection request',
|
||||
);
|
||||
break;
|
||||
case 'worker-session.inspect':
|
||||
exactObject(
|
||||
value.request,
|
||||
['authorityProjectId', 'inspectionId', 'workerId'],
|
||||
'Session inspection request',
|
||||
);
|
||||
break;
|
||||
case 'worker-session.list':
|
||||
exactObject(
|
||||
value.request,
|
||||
['afterWorkerId', 'authorityProjectId', 'inspectionId'],
|
||||
'Session list request',
|
||||
);
|
||||
break;
|
||||
default:
|
||||
throw new ClusterWorkerCredentialManagementTransportRequestError(
|
||||
'operation is not publicly available',
|
||||
@@ -317,6 +370,8 @@ export function createClusterWorkerCredentialManagementTransport(
|
||||
typeof options.service.propose !== 'function' ||
|
||||
typeof options.service.decide !== 'function' ||
|
||||
typeof options.service.inspectAuthorized !== 'function' ||
|
||||
typeof options.service.inspectSession !== 'function' ||
|
||||
typeof options.service.listSessions !== 'function' ||
|
||||
(options.now !== undefined && typeof options.now !== 'function')
|
||||
) {
|
||||
throw new ClusterWorkerCredentialManagementTransportConfigurationError(
|
||||
@@ -420,6 +475,31 @@ export function createClusterWorkerCredentialManagementTransport(
|
||||
stale: result.stale,
|
||||
});
|
||||
}
|
||||
case 'worker-session.inspect': {
|
||||
const result = await options.service.inspectSession({
|
||||
...command.request,
|
||||
principal,
|
||||
});
|
||||
return Object.freeze({
|
||||
schemaVersion: 1 as const,
|
||||
operation: command.operation,
|
||||
observedAtMs: result.observedAtMs,
|
||||
worker: result.worker,
|
||||
});
|
||||
}
|
||||
case 'worker-session.list': {
|
||||
const result = await options.service.listSessions({
|
||||
...command.request,
|
||||
principal,
|
||||
});
|
||||
return Object.freeze({
|
||||
schemaVersion: 1 as const,
|
||||
operation: command.operation,
|
||||
observedAtMs: result.observedAtMs,
|
||||
workers: result.workers,
|
||||
nextCursor: result.nextCursor,
|
||||
});
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
+164
-2
@@ -57,6 +57,23 @@ function boundedScalar(value: unknown): void {
|
||||
}
|
||||
}
|
||||
|
||||
function boundedText(value: unknown, nullable = false): void {
|
||||
if (value === null && nullable) return;
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length === 0 ||
|
||||
value.length > 2_048 ||
|
||||
CONTROL_PATTERN.test(value)
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
}
|
||||
|
||||
function nonNegativeInteger(value: unknown, nullable = false): void {
|
||||
if (value === null && nullable) return;
|
||||
if (!Number.isSafeInteger(value) || (value as number) < 0) invalid();
|
||||
}
|
||||
|
||||
function subject(value: unknown): void {
|
||||
const record = exactRecord(value, ['type', 'id']);
|
||||
if (record.type !== 'user') invalid();
|
||||
@@ -131,6 +148,106 @@ function approval(value: unknown): void {
|
||||
}
|
||||
}
|
||||
|
||||
const WORKER_OBSERVATION_KEYS = [
|
||||
'architecture',
|
||||
'availableSlots',
|
||||
'compatibility',
|
||||
'generation',
|
||||
'lastHeartbeatAtMs',
|
||||
'leaseExpiresAtMs',
|
||||
'lifecycle',
|
||||
'maxConcurrentRuns',
|
||||
'observedAtMs',
|
||||
'operatingSystem',
|
||||
'protocolVersion',
|
||||
'registeredAtMs',
|
||||
'sessionId',
|
||||
'sessionVersion',
|
||||
'supportTier',
|
||||
'updatedAtMs',
|
||||
'workerId',
|
||||
] as const;
|
||||
|
||||
function workerObservation(value: unknown, detailed: boolean): void {
|
||||
const record = exactRecord(
|
||||
value,
|
||||
detailed
|
||||
? [...WORKER_OBSERVATION_KEYS, 'declaredCapacity', 'runtimes']
|
||||
: WORKER_OBSERVATION_KEYS,
|
||||
);
|
||||
boundedText(record.workerId);
|
||||
boundedText(record.sessionId);
|
||||
boundedText(record.protocolVersion);
|
||||
boundedText(record.operatingSystem, true);
|
||||
for (const key of [
|
||||
'generation',
|
||||
'sessionVersion',
|
||||
'maxConcurrentRuns',
|
||||
'availableSlots',
|
||||
'registeredAtMs',
|
||||
'lastHeartbeatAtMs',
|
||||
'leaseExpiresAtMs',
|
||||
'updatedAtMs',
|
||||
'observedAtMs',
|
||||
] as const) {
|
||||
nonNegativeInteger(record[key]);
|
||||
}
|
||||
if (
|
||||
!['online', 'draining', 'offline', 'lease_expired'].includes(
|
||||
String(record.lifecycle),
|
||||
) ||
|
||||
![
|
||||
'default_placement',
|
||||
'explicit_placement_required',
|
||||
'protocol_incompatible',
|
||||
].includes(String(record.compatibility)) ||
|
||||
!['tier1', 'candidate', 'experimental', 'legacy-only'].includes(
|
||||
String(record.supportTier),
|
||||
) ||
|
||||
!['amd64', 'arm64', 'ppc64le', 's390x', 'arm/v7', 'arm/v6', '386'].includes(
|
||||
String(record.architecture),
|
||||
)
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
if (
|
||||
(record.generation as number) < 1 ||
|
||||
(record.maxConcurrentRuns as number) < 1 ||
|
||||
(record.availableSlots as number) > (record.maxConcurrentRuns as number) ||
|
||||
(record.lastHeartbeatAtMs as number) < (record.registeredAtMs as number) ||
|
||||
(record.updatedAtMs as number) < (record.lastHeartbeatAtMs as number) ||
|
||||
(record.observedAtMs as number) < (record.updatedAtMs as number) ||
|
||||
(record.lifecycle !== 'offline' &&
|
||||
(record.leaseExpiresAtMs as number) <=
|
||||
(record.lastHeartbeatAtMs as number)) ||
|
||||
(['online', 'draining'].includes(String(record.lifecycle)) &&
|
||||
(record.leaseExpiresAtMs as number) <= (record.observedAtMs as number)) ||
|
||||
(record.lifecycle === 'lease_expired' &&
|
||||
(record.leaseExpiresAtMs as number) > (record.observedAtMs as number)) ||
|
||||
(['draining', 'offline'].includes(String(record.lifecycle)) &&
|
||||
(record.availableSlots as number) !== 0)
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
if (!detailed) return;
|
||||
if (!Array.isArray(record.runtimes) || record.runtimes.length > 32) invalid();
|
||||
for (const runtime of record.runtimes) {
|
||||
const item = exactRecord(runtime, ['name', 'version']);
|
||||
boundedText(item.name);
|
||||
boundedText(item.version);
|
||||
}
|
||||
const capacity = exactRecord(record.declaredCapacity, [
|
||||
'cpuCores',
|
||||
'diskBytes',
|
||||
'gpuCount',
|
||||
'memoryBytes',
|
||||
]);
|
||||
nonNegativeInteger(capacity.cpuCores, true);
|
||||
nonNegativeInteger(capacity.memoryBytes, true);
|
||||
nonNegativeInteger(capacity.diskBytes, true);
|
||||
nonNegativeInteger(capacity.gpuCount);
|
||||
}
|
||||
|
||||
export function validateClusterWorkerCredentialManagementClientResult(
|
||||
value: unknown,
|
||||
command: Readonly<ClusterWorkerCredentialManagementCommand>,
|
||||
@@ -143,7 +260,17 @@ export function validateClusterWorkerCredentialManagementClientResult(
|
||||
? ['schemaVersion', 'operation', 'approvalStatus', 'plan', 'approval']
|
||||
: operation === 'worker-credential.decide'
|
||||
? ['schemaVersion', 'operation', 'status', 'approval']
|
||||
: ['schemaVersion', 'operation', 'plan', 'approval', 'stale'];
|
||||
: operation === 'worker-credential.inspect'
|
||||
? ['schemaVersion', 'operation', 'plan', 'approval', 'stale']
|
||||
: operation === 'worker-session.inspect'
|
||||
? ['schemaVersion', 'operation', 'observedAtMs', 'worker']
|
||||
: [
|
||||
'schemaVersion',
|
||||
'operation',
|
||||
'observedAtMs',
|
||||
'workers',
|
||||
'nextCursor',
|
||||
];
|
||||
const record = exactRecord(value, keys);
|
||||
if (record.schemaVersion !== 1 || record.operation !== operation) invalid();
|
||||
if (operation === 'worker-credential.plan') {
|
||||
@@ -157,10 +284,45 @@ export function validateClusterWorkerCredentialManagementClientResult(
|
||||
} else if (operation === 'worker-credential.decide') {
|
||||
if (!['decided', 'existing'].includes(String(record.status))) invalid();
|
||||
approval(record.approval);
|
||||
} else {
|
||||
} else if (operation === 'worker-credential.inspect') {
|
||||
if (typeof record.stale !== 'boolean') invalid();
|
||||
if (record.plan !== null) plan(record.plan);
|
||||
if (record.approval !== null) approval(record.approval);
|
||||
} else if (operation === 'worker-session.inspect') {
|
||||
nonNegativeInteger(record.observedAtMs);
|
||||
if (record.worker !== null) {
|
||||
workerObservation(record.worker, true);
|
||||
if (
|
||||
(record.worker as Record<string, unknown>).observedAtMs !==
|
||||
record.observedAtMs
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
nonNegativeInteger(record.observedAtMs);
|
||||
boundedText(record.nextCursor, true);
|
||||
if (!Array.isArray(record.workers) || record.workers.length > 16) invalid();
|
||||
let previous = '';
|
||||
for (const worker of record.workers) {
|
||||
workerObservation(worker, false);
|
||||
const item = worker as Record<string, unknown>;
|
||||
if (
|
||||
item.observedAtMs !== record.observedAtMs ||
|
||||
typeof item.workerId !== 'string' ||
|
||||
item.workerId <= previous
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
previous = item.workerId;
|
||||
}
|
||||
if (
|
||||
(record.nextCursor !== null &&
|
||||
(record.workers.length === 0 || record.nextCursor !== previous)) ||
|
||||
(record.nextCursor === null && record.workers.length > 16)
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
}
|
||||
return Object.freeze(
|
||||
record as unknown as ClusterWorkerCredentialManagementTransportResult,
|
||||
|
||||
@@ -4,6 +4,9 @@ const { test } = require('node:test');
|
||||
const {
|
||||
createClusterWorkerCredentialManagementService,
|
||||
} = require('@qinglong/cluster-admin/worker-credential-management');
|
||||
const {
|
||||
canonicalRemoteWorkerCapabilities,
|
||||
} = require('@qinglong/runtime-core/remote-dispatch');
|
||||
|
||||
const REQUESTER = Object.freeze({
|
||||
subject: Object.freeze({ type: 'user', id: 'operator-a' }),
|
||||
@@ -20,6 +23,32 @@ const REVIEWER = Object.freeze({
|
||||
assurance: 'hardware',
|
||||
});
|
||||
|
||||
function sessionRow() {
|
||||
const capabilities = canonicalRemoteWorkerCapabilities({
|
||||
architecture: 'arm64',
|
||||
executors: ['remote-worker'],
|
||||
protocolVersion: '1.0.0',
|
||||
supportTier: 'tier1',
|
||||
runtimes: [{ name: 'node', version: '24.18.0' }],
|
||||
});
|
||||
return {
|
||||
observedAtMs: 2_000,
|
||||
workerId: 'worker-a',
|
||||
sessionId: '018f0f5d-7b6a-7a11-8f4d-2f7b4f477001',
|
||||
generation: 2,
|
||||
status: 'online',
|
||||
version: 5,
|
||||
capabilitiesJson: capabilities.json,
|
||||
capabilitiesHash: capabilities.hash,
|
||||
maxConcurrentRuns: 2,
|
||||
availableSlots: 1,
|
||||
registeredAtMs: 1_000,
|
||||
lastHeartbeatAtMs: 1_900,
|
||||
leaseExpiresAtMs: 3_000,
|
||||
updatedAtMs: 1_900,
|
||||
};
|
||||
}
|
||||
|
||||
function approvalFixture() {
|
||||
const plans = new Map();
|
||||
const approvals = new Map();
|
||||
@@ -76,6 +105,9 @@ function approvalFixture() {
|
||||
const stored = audits.get(values[0]);
|
||||
return { rows: stored ? [stored] : [], rowCount: stored ? 1 : 0 };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."worker_sessions"')) {
|
||||
return { rows: [sessionRow()], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('"ql3"."lock_approval_policy_fence"')) {
|
||||
return { rows: [{ matches: true }], rowCount: 1 };
|
||||
}
|
||||
@@ -277,3 +309,50 @@ test('authorizes and consumes durable quota before reading management state', as
|
||||
assert.equal(quotaCalls[1].operation, 'worker-credential.propose');
|
||||
assert.equal(quotaCalls[1].planReads, readsBefore);
|
||||
});
|
||||
|
||||
test('observes one Session and one bounded page only after worker.manage quota', async () => {
|
||||
const state = approvalFixture();
|
||||
const quotaCalls = [];
|
||||
const service = createClusterWorkerCredentialManagementService({
|
||||
pool: state.pool,
|
||||
now: () => 2_000,
|
||||
quota: {
|
||||
async consume(command) {
|
||||
quotaCalls.push(command);
|
||||
return { admitted: true, retryAfterMs: null };
|
||||
},
|
||||
},
|
||||
});
|
||||
const inspection = await service.inspectSession({
|
||||
authorityProjectId: 'cluster-authority',
|
||||
workerId: 'worker-a',
|
||||
inspectionId: 'worker-session-inspection-a',
|
||||
principal: REQUESTER,
|
||||
});
|
||||
const page = await service.listSessions({
|
||||
authorityProjectId: 'cluster-authority',
|
||||
afterWorkerId: null,
|
||||
inspectionId: 'worker-session-list-a',
|
||||
principal: REQUESTER,
|
||||
});
|
||||
assert.equal(inspection.worker.workerId, 'worker-a');
|
||||
assert.equal(inspection.worker.compatibility, 'default_placement');
|
||||
assert.equal(page.workers.length, 1);
|
||||
assert.equal(page.workers[0].workerId, 'worker-a');
|
||||
assert.deepEqual(
|
||||
quotaCalls.map(({ operation, idempotencyKey }) => ({
|
||||
operation,
|
||||
idempotencyKey,
|
||||
})),
|
||||
[
|
||||
{
|
||||
operation: 'worker-session.observe',
|
||||
idempotencyKey: 'session-inspect:worker-session-inspection-a',
|
||||
},
|
||||
{
|
||||
operation: 'worker-session.observe',
|
||||
idempotencyKey: 'session-list:worker-session-list-a',
|
||||
},
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
@@ -82,6 +82,39 @@ const approval = Object.freeze({
|
||||
actionDigest: 'd'.repeat(64),
|
||||
previewDigest: plan.previewDigest,
|
||||
});
|
||||
const session = Object.freeze({
|
||||
workerId: 'worker-1',
|
||||
sessionId: '018f0f5d-7b6a-7a11-8f4d-2f7b4f477001',
|
||||
generation: 2,
|
||||
sessionVersion: 5,
|
||||
lifecycle: 'online',
|
||||
compatibility: 'default_placement',
|
||||
architecture: 'arm64',
|
||||
supportTier: 'tier1',
|
||||
protocolVersion: '1.0.0',
|
||||
operatingSystem: 'linux',
|
||||
maxConcurrentRuns: 2,
|
||||
availableSlots: 1,
|
||||
registeredAtMs: 900,
|
||||
lastHeartbeatAtMs: 1_050,
|
||||
leaseExpiresAtMs: 2_000,
|
||||
updatedAtMs: 1_050,
|
||||
observedAtMs: 1_100,
|
||||
runtimes: Object.freeze([{ name: 'node', version: '24.18.0' }]),
|
||||
declaredCapacity: Object.freeze({
|
||||
cpuCores: 1,
|
||||
memoryBytes: 268_435_456,
|
||||
diskBytes: 1_073_741_824,
|
||||
gpuCount: 0,
|
||||
}),
|
||||
});
|
||||
const sessionSummary = Object.freeze(
|
||||
Object.fromEntries(
|
||||
Object.entries(session).filter(
|
||||
([key]) => key !== 'runtimes' && key !== 'declaredCapacity',
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
function command(operation) {
|
||||
return {
|
||||
@@ -152,7 +185,7 @@ afterEach(() => {
|
||||
}
|
||||
});
|
||||
|
||||
test('validates all four low-sensitive Worker management results', () => {
|
||||
test('validates all six low-sensitive Worker management results', () => {
|
||||
const fixtures = [
|
||||
[
|
||||
'worker-credential.plan',
|
||||
@@ -187,6 +220,25 @@ test('validates all four low-sensitive Worker management results', () => {
|
||||
stale: false,
|
||||
},
|
||||
],
|
||||
[
|
||||
'worker-session.inspect',
|
||||
{
|
||||
schemaVersion: 1,
|
||||
operation: 'worker-session.inspect',
|
||||
observedAtMs: 1_100,
|
||||
worker: session,
|
||||
},
|
||||
],
|
||||
[
|
||||
'worker-session.list',
|
||||
{
|
||||
schemaVersion: 1,
|
||||
operation: 'worker-session.list',
|
||||
observedAtMs: 1_100,
|
||||
workers: [sessionSummary],
|
||||
nextCursor: null,
|
||||
},
|
||||
],
|
||||
];
|
||||
for (const [operation, result] of fixtures) {
|
||||
assert.equal(
|
||||
@@ -223,6 +275,51 @@ test('rejects widened and secret-bearing response shapes', () => {
|
||||
command('worker-credential.inspect'),
|
||||
),
|
||||
);
|
||||
assert.throws(() =>
|
||||
validateClusterWorkerCredentialManagementClientResult(
|
||||
{
|
||||
schemaVersion: 1,
|
||||
operation: 'worker-session.inspect',
|
||||
observedAtMs: 1_100,
|
||||
worker: { ...session, labels: { secret: 'value' } },
|
||||
},
|
||||
command('worker-session.inspect'),
|
||||
),
|
||||
);
|
||||
assert.throws(() =>
|
||||
validateClusterWorkerCredentialManagementClientResult(
|
||||
{
|
||||
schemaVersion: 1,
|
||||
operation: 'worker-session.list',
|
||||
observedAtMs: 1_100,
|
||||
workers: [
|
||||
{ ...sessionSummary, workerId: 'worker-b' },
|
||||
{ ...sessionSummary, workerId: 'worker-a' },
|
||||
],
|
||||
nextCursor: null,
|
||||
},
|
||||
command('worker-session.list'),
|
||||
),
|
||||
);
|
||||
for (const worker of [
|
||||
{ ...session, workerId: null },
|
||||
{ ...session, generation: -1 },
|
||||
{ ...session, availableSlots: 3 },
|
||||
{ ...session, observedAtMs: 1_000 },
|
||||
{ ...session, declaredCapacity: { ...session.declaredCapacity, gpuCount: -1 } },
|
||||
]) {
|
||||
assert.throws(() =>
|
||||
validateClusterWorkerCredentialManagementClientResult(
|
||||
{
|
||||
schemaVersion: 1,
|
||||
operation: 'worker-session.inspect',
|
||||
observedAtMs: worker.observedAtMs,
|
||||
worker,
|
||||
},
|
||||
command('worker-session.inspect'),
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('requires one matching private client certificate identity before connect', async () => {
|
||||
|
||||
@@ -69,6 +69,41 @@ function approval(planValue) {
|
||||
});
|
||||
}
|
||||
|
||||
const SESSION_OBSERVATION = Object.freeze({
|
||||
workerId: 'worker-a',
|
||||
sessionId: '018f0f5d-7b6a-7a11-8f4d-2f7b4f477001',
|
||||
generation: 2,
|
||||
sessionVersion: 5,
|
||||
lifecycle: 'online',
|
||||
compatibility: 'default_placement',
|
||||
architecture: 'arm64',
|
||||
supportTier: 'tier1',
|
||||
protocolVersion: '1.0.0',
|
||||
operatingSystem: 'linux',
|
||||
maxConcurrentRuns: 2,
|
||||
availableSlots: 1,
|
||||
registeredAtMs: 900,
|
||||
lastHeartbeatAtMs: 1_050,
|
||||
leaseExpiresAtMs: 2_000,
|
||||
updatedAtMs: 1_050,
|
||||
observedAtMs: 1_100,
|
||||
runtimes: Object.freeze([{ name: 'node', version: '24.18.0' }]),
|
||||
declaredCapacity: Object.freeze({
|
||||
cpuCores: 1,
|
||||
memoryBytes: 268_435_456,
|
||||
diskBytes: 1_073_741_824,
|
||||
gpuCount: 0,
|
||||
}),
|
||||
});
|
||||
|
||||
const SESSION_SUMMARY = Object.freeze(
|
||||
Object.fromEntries(
|
||||
Object.entries(SESSION_OBSERVATION).filter(
|
||||
([key]) => key !== 'runtimes' && key !== 'declaredCapacity',
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
function commands() {
|
||||
return [
|
||||
{
|
||||
@@ -122,10 +157,28 @@ function commands() {
|
||||
inspectionId: 'inspection-worker-a-generation-2',
|
||||
},
|
||||
},
|
||||
{
|
||||
schemaVersion: 1,
|
||||
operation: 'worker-session.inspect',
|
||||
request: {
|
||||
authorityProjectId: 'cluster-authority',
|
||||
workerId: 'worker-a',
|
||||
inspectionId: 'inspection-worker-session-a',
|
||||
},
|
||||
},
|
||||
{
|
||||
schemaVersion: 1,
|
||||
operation: 'worker-session.list',
|
||||
request: {
|
||||
authorityProjectId: 'cluster-authority',
|
||||
afterWorkerId: null,
|
||||
inspectionId: 'inspection-worker-session-list-a',
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
test('routes the four public commands with strong User authority and low-sensitive results', async () => {
|
||||
test('routes six public commands with strong User authority and low-sensitive results', async () => {
|
||||
const planValue = plan();
|
||||
const approvalValue = approval(planValue);
|
||||
const calls = [];
|
||||
@@ -154,6 +207,18 @@ test('routes the four public commands with strong User authority and low-sensiti
|
||||
stale: false,
|
||||
};
|
||||
},
|
||||
async inspectSession(request) {
|
||||
calls.push(['inspectSession', request]);
|
||||
return { observedAtMs: 1_100, worker: SESSION_OBSERVATION };
|
||||
},
|
||||
async listSessions(request) {
|
||||
calls.push(['listSessions', request]);
|
||||
return {
|
||||
observedAtMs: 1_100,
|
||||
workers: [SESSION_SUMMARY],
|
||||
nextCursor: null,
|
||||
};
|
||||
},
|
||||
};
|
||||
const transport = createClusterWorkerCredentialManagementTransport({
|
||||
service,
|
||||
@@ -170,7 +235,7 @@ test('routes the four public commands with strong User authority and low-sensiti
|
||||
}
|
||||
assert.deepEqual(
|
||||
calls.map(([kind]) => kind),
|
||||
['plan', 'propose', 'decide', 'inspect'],
|
||||
['plan', 'propose', 'decide', 'inspect', 'inspectSession', 'listSessions'],
|
||||
);
|
||||
for (const [, request] of calls) {
|
||||
assert.deepEqual(request.principal, principal());
|
||||
@@ -182,11 +247,15 @@ test('routes the four public commands with strong User authority and low-sensiti
|
||||
'worker-credential.propose',
|
||||
'worker-credential.decide',
|
||||
'worker-credential.inspect',
|
||||
'worker-session.inspect',
|
||||
'worker-session.list',
|
||||
],
|
||||
);
|
||||
assert.equal(results[0].plan.planDigest, planValue.planDigest);
|
||||
assert.equal(results[1].approval.actionDigest, planValue.planDigest);
|
||||
assert.equal(results[3].stale, false);
|
||||
assert.equal(results[4].worker.compatibility, 'default_placement');
|
||||
assert.equal(results[5].workers[0].workerId, 'worker-a');
|
||||
const serialized = JSON.stringify(results);
|
||||
assert.doesNotMatch(serialized, /authenticationId|credential-token|secret/i);
|
||||
});
|
||||
@@ -194,7 +263,14 @@ test('routes the four public commands with strong User authority and low-sensiti
|
||||
test('rejects weak or unavailable identity before management authority', async () => {
|
||||
let calls = 0;
|
||||
const service = Object.fromEntries(
|
||||
['plan', 'propose', 'decide', 'inspectAuthorized'].map((name) => [
|
||||
[
|
||||
'plan',
|
||||
'propose',
|
||||
'decide',
|
||||
'inspectAuthorized',
|
||||
'inspectSession',
|
||||
'listSessions',
|
||||
].map((name) => [
|
||||
name,
|
||||
async () => {
|
||||
calls += 1;
|
||||
@@ -232,6 +308,8 @@ test('rejects widened and internal commands before authentication', async () =>
|
||||
async propose() {},
|
||||
async decide() {},
|
||||
async inspectAuthorized() {},
|
||||
async inspectSession() {},
|
||||
async listSessions() {},
|
||||
},
|
||||
});
|
||||
let authentications = 0;
|
||||
|
||||
@@ -348,5 +348,10 @@ export const postgresqlMainMigrationManifest: MigrationStreamManifest =
|
||||
checksum:
|
||||
'2fcac38386581189db63faacff325356f11c4529a8db9cef6be1a1ca706aaf10',
|
||||
}),
|
||||
Object.freeze({
|
||||
id: 'pg-0069-worker-session-management-observation',
|
||||
checksum:
|
||||
'1191255575589abc2686b391827607abddb4edb78007245dbaaf45dc1c4e5e8b',
|
||||
}),
|
||||
]),
|
||||
});
|
||||
|
||||
@@ -71,6 +71,7 @@ import { pg0065ApprovedActionManualRecoveryMigration } from '../approved-action/
|
||||
import { pg0066CancellationDispatchMigration } from '../run/migrations/pg-0066-cancellation-dispatch';
|
||||
import { pg0067CancellationDispatchManagementMigration } from '../run-management/pg-0067-cancellation-dispatch-management';
|
||||
import { pg0068CancellationDispatchProjectKeysetMigration } from '../run-management/pg-0068-cancellation-dispatch-project-keyset';
|
||||
import { pg0069WorkerSessionManagementObservationMigration } from '../remote-execution/pg-0069-worker-session-management-observation';
|
||||
|
||||
export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMigrationContext> =
|
||||
Object.freeze({
|
||||
@@ -147,5 +148,6 @@ export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMi
|
||||
pg0066CancellationDispatchMigration,
|
||||
pg0067CancellationDispatchManagementMigration,
|
||||
pg0068CancellationDispatchProjectKeysetMigration,
|
||||
pg0069WorkerSessionManagementObservationMigration,
|
||||
]),
|
||||
});
|
||||
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
import { CAPABILITIES_V67 } from '../run-management/pg-0068-cancellation-dispatch-project-keyset';
|
||||
import { definePostgresSqlMigration } from '../migrations/sqlMigration';
|
||||
|
||||
export const CAPABILITIES_V68 = CAPABILITIES_V67.replace(
|
||||
'"worker_session":1}',
|
||||
'"worker_session":1,"worker_session_observation":1}',
|
||||
);
|
||||
|
||||
export const pg0069WorkerSessionManagementObservationMigration =
|
||||
definePostgresSqlMigration({
|
||||
id: 'pg-0069-worker-session-management-observation',
|
||||
statements: [
|
||||
`REVOKE ALL ON "ql3"."worker_sessions" FROM ql3_worker_credential_manager`,
|
||||
`GRANT SELECT ON "ql3"."worker_sessions" TO ql3_worker_credential_manager`,
|
||||
`DO $ql3$ BEGIN UPDATE "ql3"."schema_capabilities" SET contract_version = 68, migration_id = 'pg-0069-worker-session-management-observation', capabilities = '${CAPABILITIES_V68}'::jsonb, updated_at_ms = floor(extract(epoch FROM transaction_timestamp()) * 1000)::bigint WHERE contract_name = 'control-core' AND contract_version = 67 AND migration_id = 'pg-0068-cancellation-dispatch-project-keyset' AND capabilities = '${CAPABILITIES_V67}'::jsonb; IF NOT FOUND THEN RAISE EXCEPTION 'control-core capability is not at version 67' USING ERRCODE = 'check_violation'; END IF; END $ql3$`,
|
||||
],
|
||||
});
|
||||
+188
@@ -0,0 +1,188 @@
|
||||
import type { PostgresPool } from '@qinglong/runtime-core';
|
||||
import {
|
||||
WORKER_SESSION_STATUSES,
|
||||
assertWorkerCapabilitiesSnapshot,
|
||||
assertWorkerConcurrency,
|
||||
assertWorkerId,
|
||||
assertWorkerSessionId,
|
||||
type WorkerSessionRecord,
|
||||
type WorkerSessionStatus,
|
||||
} from '@qinglong/runtime-core/worker-session';
|
||||
import {
|
||||
MAX_WORKER_SESSION_OBSERVATION_PAGE_SIZE,
|
||||
projectWorkerSessionObservation,
|
||||
summarizeWorkerSessionObservation,
|
||||
type WorkerSessionInspection,
|
||||
type WorkerSessionObservationPage,
|
||||
} from '@qinglong/runtime-core/worker-session-observation';
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
|
||||
const SELECT_COLUMNS = `
|
||||
worker_id AS "workerId",
|
||||
session_id AS "sessionId",
|
||||
generation AS "generation",
|
||||
status AS "status",
|
||||
version AS "version",
|
||||
capabilities_json AS "capabilitiesJson",
|
||||
capabilities_hash AS "capabilitiesHash",
|
||||
max_concurrent_runs AS "maxConcurrentRuns",
|
||||
available_slots AS "availableSlots",
|
||||
registered_at_ms AS "registeredAtMs",
|
||||
last_heartbeat_at_ms AS "lastHeartbeatAtMs",
|
||||
lease_expires_at_ms AS "leaseExpiresAtMs",
|
||||
updated_at_ms AS "updatedAtMs"
|
||||
`.trim();
|
||||
|
||||
function string(row: Row, key: string): string {
|
||||
const value = row[key];
|
||||
if (typeof value !== 'string' || value.length === 0) {
|
||||
throw new TypeError(`PostgreSQL Worker observation ${key} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function integer(row: Row, key: string): number {
|
||||
const value = row[key];
|
||||
const normalized =
|
||||
typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value)
|
||||
? Number(value)
|
||||
: value;
|
||||
if (!Number.isSafeInteger(normalized) || (normalized as number) < 0) {
|
||||
throw new TypeError(`PostgreSQL Worker observation ${key} is invalid`);
|
||||
}
|
||||
return normalized as number;
|
||||
}
|
||||
|
||||
function record(row: Row): Readonly<WorkerSessionRecord> {
|
||||
const status = string(row, 'status');
|
||||
if (!WORKER_SESSION_STATUSES.includes(status as WorkerSessionStatus)) {
|
||||
throw new TypeError('PostgreSQL Worker observation status is invalid');
|
||||
}
|
||||
const worker: WorkerSessionRecord = Object.freeze({
|
||||
workerId: string(row, 'workerId'),
|
||||
sessionId: string(row, 'sessionId'),
|
||||
generation: integer(row, 'generation'),
|
||||
status: status as WorkerSessionStatus,
|
||||
version: integer(row, 'version'),
|
||||
capabilitiesJson: string(row, 'capabilitiesJson'),
|
||||
capabilitiesHash: string(row, 'capabilitiesHash'),
|
||||
maxConcurrentRuns: integer(row, 'maxConcurrentRuns'),
|
||||
availableSlots: integer(row, 'availableSlots'),
|
||||
registeredAtMs: integer(row, 'registeredAtMs'),
|
||||
lastHeartbeatAtMs: integer(row, 'lastHeartbeatAtMs'),
|
||||
leaseExpiresAtMs: integer(row, 'leaseExpiresAtMs'),
|
||||
updatedAtMs: integer(row, 'updatedAtMs'),
|
||||
});
|
||||
assertWorkerId(worker.workerId);
|
||||
assertWorkerSessionId(worker.sessionId);
|
||||
assertWorkerCapabilitiesSnapshot(
|
||||
worker.capabilitiesJson,
|
||||
worker.capabilitiesHash,
|
||||
);
|
||||
assertWorkerConcurrency(worker.maxConcurrentRuns, worker.availableSlots);
|
||||
return worker;
|
||||
}
|
||||
|
||||
function observedAtMs(row: Row): number {
|
||||
return integer(row, 'observedAtMs');
|
||||
}
|
||||
|
||||
export class PostgresWorkerSessionObservationRepository {
|
||||
constructor(private readonly pool: PostgresPool) {
|
||||
if (!pool || typeof pool.query !== 'function') {
|
||||
throw new TypeError('PostgreSQL Worker observation Pool is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
async inspect(workerId: string): Promise<Readonly<WorkerSessionInspection>> {
|
||||
assertWorkerId(workerId);
|
||||
const result = await this.pool.query<Row>(
|
||||
`WITH observation AS (
|
||||
SELECT floor(extract(epoch FROM statement_timestamp()) * 1000)::bigint AS observed_at_ms
|
||||
)
|
||||
SELECT observation.observed_at_ms AS "observedAtMs", worker.*
|
||||
FROM observation
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT ${SELECT_COLUMNS}
|
||||
FROM "ql3"."worker_sessions"
|
||||
WHERE worker_id = $1
|
||||
LIMIT 2
|
||||
) AS worker ON TRUE`,
|
||||
[workerId],
|
||||
);
|
||||
if (result.rows.length !== 1) {
|
||||
throw new TypeError('PostgreSQL Worker inspection violated its bound');
|
||||
}
|
||||
const observed = observedAtMs(result.rows[0]!);
|
||||
if (
|
||||
result.rows[0]!.workerId !== null &&
|
||||
string(result.rows[0]!, 'workerId') !== workerId
|
||||
) {
|
||||
throw new TypeError('PostgreSQL Worker inspection identity drifted');
|
||||
}
|
||||
return Object.freeze({
|
||||
observedAtMs: observed,
|
||||
worker:
|
||||
result.rows[0]!.workerId === null
|
||||
? null
|
||||
: projectWorkerSessionObservation(record(result.rows[0]!), observed),
|
||||
});
|
||||
}
|
||||
|
||||
async list(
|
||||
afterWorkerId: string | null,
|
||||
): Promise<Readonly<WorkerSessionObservationPage>> {
|
||||
if (afterWorkerId !== null) assertWorkerId(afterWorkerId);
|
||||
const result = await this.pool.query<Row>(
|
||||
`WITH observation AS (
|
||||
SELECT floor(extract(epoch FROM statement_timestamp()) * 1000)::bigint AS observed_at_ms
|
||||
), page AS (
|
||||
SELECT ${SELECT_COLUMNS}
|
||||
FROM "ql3"."worker_sessions"
|
||||
WHERE ($1::varchar IS NULL OR worker_id > $1)
|
||||
ORDER BY worker_id
|
||||
LIMIT $2
|
||||
)
|
||||
SELECT observation.observed_at_ms AS "observedAtMs", page.*
|
||||
FROM observation LEFT JOIN page ON TRUE
|
||||
ORDER BY page."workerId"`,
|
||||
[afterWorkerId, MAX_WORKER_SESSION_OBSERVATION_PAGE_SIZE + 1],
|
||||
);
|
||||
if (
|
||||
result.rows.length < 1 ||
|
||||
result.rows.length > MAX_WORKER_SESSION_OBSERVATION_PAGE_SIZE + 1
|
||||
) {
|
||||
throw new TypeError('PostgreSQL Worker observation page violated its bound');
|
||||
}
|
||||
const observed = observedAtMs(result.rows[0]!);
|
||||
if (result.rows.some((row) => observedAtMs(row) !== observed)) {
|
||||
throw new TypeError('PostgreSQL Worker observation clock drifted');
|
||||
}
|
||||
const observations =
|
||||
result.rows[0]!.workerId === null
|
||||
? []
|
||||
: result.rows.map((row) =>
|
||||
projectWorkerSessionObservation(record(row), observed),
|
||||
);
|
||||
for (let index = 0; index < observations.length; index += 1) {
|
||||
const workerId = observations[index]!.workerId;
|
||||
const previous = observations[index - 1]?.workerId ?? afterWorkerId;
|
||||
if (previous !== null && workerId <= previous) {
|
||||
throw new TypeError('PostgreSQL Worker observation page is unordered');
|
||||
}
|
||||
}
|
||||
const page = observations.slice(
|
||||
0,
|
||||
MAX_WORKER_SESSION_OBSERVATION_PAGE_SIZE,
|
||||
);
|
||||
return Object.freeze({
|
||||
observedAtMs: observed,
|
||||
workers: Object.freeze(page.map(summarizeWorkerSessionObservation)),
|
||||
nextCursor:
|
||||
observations.length > MAX_WORKER_SESSION_OBSERVATION_PAGE_SIZE
|
||||
? page.at(-1)!.workerId
|
||||
: null,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -21,8 +21,8 @@ export interface PostgresSchemaContractTrigger {
|
||||
export interface PostgresSchemaContract {
|
||||
readonly schema: 'ql3';
|
||||
readonly contractName: 'control-core';
|
||||
readonly contractVersion: 67;
|
||||
readonly migrationId: 'pg-0068-cancellation-dispatch-project-keyset';
|
||||
readonly contractVersion: 68;
|
||||
readonly migrationId: 'pg-0069-worker-session-management-observation';
|
||||
readonly minimumServerMajor: 16;
|
||||
readonly maximumServerMajor: 18;
|
||||
readonly capabilities: Readonly<{
|
||||
@@ -92,6 +92,7 @@ export interface PostgresSchemaContract {
|
||||
tool_invocation_artifact: 1;
|
||||
trigger_definition: 1;
|
||||
worker_session: 1;
|
||||
worker_session_observation: 1;
|
||||
worker_credential: 1;
|
||||
worker_credential_delivery: 1;
|
||||
worker_credential_execution_receipt: 1;
|
||||
@@ -121,8 +122,8 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
Object.freeze({
|
||||
schema: 'ql3',
|
||||
contractName: 'control-core',
|
||||
contractVersion: 67,
|
||||
migrationId: 'pg-0068-cancellation-dispatch-project-keyset',
|
||||
contractVersion: 68,
|
||||
migrationId: 'pg-0069-worker-session-management-observation',
|
||||
minimumServerMajor: 16,
|
||||
maximumServerMajor: 18,
|
||||
capabilities: Object.freeze({
|
||||
@@ -192,6 +193,7 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
tool_invocation_artifact: 1,
|
||||
trigger_definition: 1,
|
||||
worker_session: 1,
|
||||
worker_session_observation: 1,
|
||||
worker_credential: 1,
|
||||
worker_credential_delivery: 1,
|
||||
worker_credential_execution_receipt: 1,
|
||||
|
||||
@@ -1509,7 +1509,8 @@ const REQUIRED_WORKER_CREDENTIAL_MANAGER_PRIVILEGES: RequiredPrivileges =
|
||||
name === 'schema_migrations' ||
|
||||
name === 'schema_capabilities' ||
|
||||
name === 'projects' ||
|
||||
name === 'project_role_bindings'
|
||||
name === 'project_role_bindings' ||
|
||||
name === 'worker_sessions'
|
||||
? { ...NO_TABLE_PRIVILEGES, select: true }
|
||||
: name === 'worker_credential_management_plans' ||
|
||||
name === 'security_audit_events'
|
||||
|
||||
+2
@@ -10,6 +10,7 @@ const OPERATIONS = [
|
||||
'worker-credential.propose',
|
||||
'worker-credential.decide',
|
||||
'worker-credential.inspect',
|
||||
'worker-session.observe',
|
||||
] as const;
|
||||
type Operation = (typeof OPERATIONS)[number];
|
||||
type Row = Record<string, unknown>;
|
||||
@@ -31,6 +32,7 @@ const DEFAULT_LIMITS: Readonly<Record<Operation, number>> = Object.freeze({
|
||||
'worker-credential.propose': 30,
|
||||
'worker-credential.decide': 60,
|
||||
'worker-credential.inspect': 600,
|
||||
'worker-session.observe': 600,
|
||||
});
|
||||
const ID = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,254}$/;
|
||||
const PROJECT = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
|
||||
@@ -44,6 +44,7 @@ export { postgresqlMainMigrationManifest } from '../migration/migrationManifest'
|
||||
export { PostgresApprovalRequestRepository } from '../approved-action/approvalRequestRepository';
|
||||
export { PostgresProjectPolicyRepository } from '../security/projectPolicyRepository';
|
||||
export { PostgresWorkerCredentialManagementPlanRepository } from './workerCredentialManagementPlanRepository';
|
||||
export { PostgresWorkerSessionObservationRepository } from '../remote-execution/workerSessionObservationRepository';
|
||||
export {
|
||||
PostgresPluginPackageIdentityKeysetLedgerRepository as PostgresWorkerCredentialManagementIdentityKeysetLedgerRepository,
|
||||
type ClusterManagementIdentityAuthority,
|
||||
|
||||
@@ -119,6 +119,7 @@ test('defines the immutable PostgreSQL capability and Run core stream', async ()
|
||||
'pg-0066-cancellation-dispatch',
|
||||
'pg-0067-cancellation-dispatch-management',
|
||||
'pg-0068-cancellation-dispatch-project-keyset',
|
||||
'pg-0069-worker-session-management-observation',
|
||||
],
|
||||
);
|
||||
for (const migration of postgresqlMainMigrationStream.migrations) {
|
||||
@@ -597,6 +598,11 @@ test('freezes every published PostgreSQL migration checksum', () => {
|
||||
checksum:
|
||||
'2fcac38386581189db63faacff325356f11c4529a8db9cef6be1a1ca706aaf10',
|
||||
},
|
||||
{
|
||||
id: 'pg-0069-worker-session-management-observation',
|
||||
checksum:
|
||||
'1191255575589abc2686b391827607abddb4edb78007245dbaaf45dc1c4e5e8b',
|
||||
},
|
||||
];
|
||||
assert.deepEqual(
|
||||
postgresqlMainMigrationStream.migrations.map(({ id, checksum }) => ({
|
||||
@@ -2421,3 +2427,36 @@ test('advances capability v67 with a Project-scoped blocked keyset', async () =>
|
||||
/migration_id = 'pg-0067-cancellation-dispatch-management'/,
|
||||
);
|
||||
});
|
||||
|
||||
test('advances capability v68 with read-only Worker session observation', async () => {
|
||||
const migration = migrationById(
|
||||
'pg-0069-worker-session-management-observation',
|
||||
);
|
||||
const statements = [];
|
||||
await migration.up({
|
||||
async query(statement) {
|
||||
statements.push(statement);
|
||||
return { rows: [] };
|
||||
},
|
||||
});
|
||||
const sql = statements.join('\n');
|
||||
assert.match(
|
||||
sql,
|
||||
/REVOKE ALL ON "ql3"\."worker_sessions" FROM ql3_worker_credential_manager/,
|
||||
);
|
||||
assert.match(
|
||||
sql,
|
||||
/GRANT SELECT ON "ql3"\."worker_sessions" TO ql3_worker_credential_manager/,
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
sql,
|
||||
/GRANT (?:INSERT|UPDATE|DELETE|TRUNCATE)[^;]+worker_sessions[^;]+ql3_worker_credential_manager/,
|
||||
);
|
||||
assert.match(sql, /contract_version = 68/);
|
||||
assert.match(sql, /"worker_session_observation":1/);
|
||||
assert.match(sql, /contract_version = 67/);
|
||||
assert.match(
|
||||
sql,
|
||||
/migration_id = 'pg-0068-cancellation-dispatch-project-keyset'/,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -561,6 +561,7 @@ function workerCredentialPrivileges(kind) {
|
||||
? [
|
||||
'worker_credential_management_quota_buckets',
|
||||
'plugin_package_identity_keyset_ledger',
|
||||
'worker_sessions',
|
||||
]
|
||||
: []),
|
||||
...(manager
|
||||
@@ -835,7 +836,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
|
||||
serverMajor: 16,
|
||||
currentUser: 'ql3_runtime',
|
||||
contractName: 'control-core',
|
||||
contractVersion: 67,
|
||||
contractVersion: 68,
|
||||
migrationIds: [
|
||||
'pg-0001-schema-capability',
|
||||
'pg-0002-run-core',
|
||||
@@ -905,6 +906,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
|
||||
'pg-0066-cancellation-dispatch',
|
||||
'pg-0067-cancellation-dispatch-management',
|
||||
'pg-0068-cancellation-dispatch-project-keyset',
|
||||
'pg-0069-worker-session-management-observation',
|
||||
],
|
||||
});
|
||||
});
|
||||
@@ -935,10 +937,10 @@ test('accepts the exact schema and isolated least-privilege admin role', async (
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_admin');
|
||||
assert.equal(report.contractVersion, 67);
|
||||
assert.equal(report.contractVersion, 68);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0068-cancellation-dispatch-project-keyset',
|
||||
'pg-0069-worker-session-management-observation',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -951,10 +953,10 @@ test('accepts the isolated least-privilege automation manager role', async () =>
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_automation_manager');
|
||||
assert.equal(report.contractVersion, 67);
|
||||
assert.equal(report.contractVersion, 68);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0068-cancellation-dispatch-project-keyset',
|
||||
'pg-0069-worker-session-management-observation',
|
||||
);
|
||||
|
||||
const widened = automationManagerPrivileges();
|
||||
@@ -983,10 +985,10 @@ test('accepts the isolated least-privilege human Approval manager role', async (
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_approval_manager');
|
||||
assert.equal(report.contractVersion, 67);
|
||||
assert.equal(report.contractVersion, 68);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0068-cancellation-dispatch-project-keyset',
|
||||
'pg-0069-worker-session-management-observation',
|
||||
);
|
||||
|
||||
const widened = approvalManagerPrivileges();
|
||||
@@ -1017,10 +1019,10 @@ test('accepts the isolated least-privilege Run manager role', async () => {
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_run_manager');
|
||||
assert.equal(report.contractVersion, 67);
|
||||
assert.equal(report.contractVersion, 68);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0068-cancellation-dispatch-project-keyset',
|
||||
'pg-0069-worker-session-management-observation',
|
||||
);
|
||||
|
||||
const widened = runManagerPrivileges();
|
||||
@@ -1181,10 +1183,10 @@ test('accepts the exact schema and isolated Worker ingress role', async () => {
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_worker_ingress');
|
||||
assert.equal(report.contractVersion, 67);
|
||||
assert.equal(report.contractVersion, 68);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0068-cancellation-dispatch-project-keyset',
|
||||
'pg-0069-worker-session-management-observation',
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
canonicalRemoteWorkerCapabilities,
|
||||
} = require('@qinglong/runtime-core/remote-dispatch');
|
||||
const {
|
||||
PostgresWorkerSessionObservationRepository,
|
||||
} = require('@qinglong/cluster-postgres/worker-credential-manager');
|
||||
|
||||
function row(index, overrides = {}) {
|
||||
const canonical = canonicalRemoteWorkerCapabilities(
|
||||
overrides.capabilities ?? {
|
||||
architecture: index % 2 === 0 ? 'arm64' : 's390x',
|
||||
executors: ['remote-worker'],
|
||||
protocolVersion: '1.0.0',
|
||||
supportTier: index % 2 === 0 ? 'tier1' : 'candidate',
|
||||
runtimes: [{ name: 'node', version: '24.18.0' }],
|
||||
},
|
||||
);
|
||||
return {
|
||||
observedAtMs: 2_000,
|
||||
workerId: `worker-${String(index).padStart(2, '0')}`,
|
||||
sessionId: `018f0f5d-7b6a-7a11-8f4d-${String(index + 1).padStart(
|
||||
12,
|
||||
'0',
|
||||
)}`,
|
||||
generation: 1,
|
||||
status: 'online',
|
||||
version: 2,
|
||||
capabilitiesJson: canonical.json,
|
||||
capabilitiesHash: canonical.hash,
|
||||
maxConcurrentRuns: 4,
|
||||
availableSlots: 2,
|
||||
registeredAtMs: 1_000,
|
||||
lastHeartbeatAtMs: 1_900,
|
||||
leaseExpiresAtMs: 3_000,
|
||||
updatedAtMs: 1_900,
|
||||
...Object.fromEntries(
|
||||
Object.entries(overrides).filter(([key]) => key !== 'capabilities'),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
test('inspects one exact Worker without exposing raw capability content', async () => {
|
||||
const queries = [];
|
||||
const repository = new PostgresWorkerSessionObservationRepository({
|
||||
async query(text, values) {
|
||||
queries.push({ text, values });
|
||||
return { rows: [row(0)], rowCount: 1 };
|
||||
},
|
||||
});
|
||||
const result = await repository.inspect('worker-00');
|
||||
assert.equal(result.observedAtMs, 2_000);
|
||||
assert.equal(result.worker.workerId, 'worker-00');
|
||||
assert.equal(result.worker.compatibility, 'default_placement');
|
||||
assert.deepEqual(result.worker.runtimes, [
|
||||
{ name: 'node', version: '24.18.0' },
|
||||
]);
|
||||
assert.equal(Object.hasOwn(result.worker, 'capabilitiesJson'), false);
|
||||
assert.deepEqual(queries[0].values, ['worker-00']);
|
||||
assert.match(queries[0].text, /LIMIT 2/);
|
||||
});
|
||||
|
||||
test('returns a masked absent inspection and a fixed sixteen-item keyset page', async () => {
|
||||
let call = 0;
|
||||
const repository = new PostgresWorkerSessionObservationRepository({
|
||||
async query(_text, values) {
|
||||
call += 1;
|
||||
if (call === 1) {
|
||||
return {
|
||||
rows: [{ observedAtMs: 2_000, workerId: null }],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
assert.deepEqual(values, [null, 17]);
|
||||
return {
|
||||
rows: Array.from({ length: 17 }, (_, index) => row(index)),
|
||||
rowCount: 17,
|
||||
};
|
||||
},
|
||||
});
|
||||
assert.deepEqual(await repository.inspect('missing-worker'), {
|
||||
observedAtMs: 2_000,
|
||||
worker: null,
|
||||
});
|
||||
const page = await repository.list(null);
|
||||
assert.equal(page.workers.length, 16);
|
||||
assert.equal(page.nextCursor, 'worker-15');
|
||||
assert.equal(page.workers[1].compatibility, 'explicit_placement_required');
|
||||
assert.equal(Object.hasOwn(page.workers[0], 'runtimes'), false);
|
||||
assert.equal(Object.hasOwn(page.workers[0], 'declaredCapacity'), false);
|
||||
});
|
||||
|
||||
test('fails closed on identity, ordering and database-clock drift', async () => {
|
||||
const identity = new PostgresWorkerSessionObservationRepository({
|
||||
async query() {
|
||||
return { rows: [row(1)], rowCount: 1 };
|
||||
},
|
||||
});
|
||||
await assert.rejects(identity.inspect('worker-00'), /identity drifted/);
|
||||
|
||||
const unordered = new PostgresWorkerSessionObservationRepository({
|
||||
async query() {
|
||||
return { rows: [row(1), row(0)], rowCount: 2 };
|
||||
},
|
||||
});
|
||||
await assert.rejects(unordered.list(null), /page is unordered/);
|
||||
|
||||
const clock = new PostgresWorkerSessionObservationRepository({
|
||||
async query() {
|
||||
return {
|
||||
rows: [row(0), row(1, { observedAtMs: 2_001 })],
|
||||
rowCount: 2,
|
||||
};
|
||||
},
|
||||
});
|
||||
await assert.rejects(clock.list(null), /clock drifted/);
|
||||
});
|
||||
@@ -56,6 +56,9 @@
|
||||
"worker-session": [
|
||||
"dist/worker/workerSession.d.ts"
|
||||
],
|
||||
"worker-session-observation": [
|
||||
"dist/worker/workerSessionObservation.d.ts"
|
||||
],
|
||||
"worker-session-transport": [
|
||||
"dist/worker/workerSessionTransport.d.ts"
|
||||
],
|
||||
@@ -804,6 +807,11 @@
|
||||
"require": "./dist/worker/workerSession.js",
|
||||
"default": "./dist/worker/workerSession.js"
|
||||
},
|
||||
"./worker-session-observation": {
|
||||
"types": "./dist/worker/workerSessionObservation.d.ts",
|
||||
"require": "./dist/worker/workerSessionObservation.js",
|
||||
"default": "./dist/worker/workerSessionObservation.js"
|
||||
},
|
||||
"./worker-session-transport": {
|
||||
"types": "./dist/worker/workerSessionTransport.d.ts",
|
||||
"require": "./dist/worker/workerSessionTransport.js",
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
import {
|
||||
REMOTE_WORKER_PROTOCOL_RANGE,
|
||||
parseRemoteWorkerCapabilities,
|
||||
remoteWorkerProtocolIsCompatible,
|
||||
type RemoteWorkerArchitecture,
|
||||
type RemoteWorkerRuntimeCapability,
|
||||
type RemoteWorkerSupportTier,
|
||||
} from '../remote-execution/remoteWorkerPlacement';
|
||||
import {
|
||||
assertWorkerSessionRecord,
|
||||
type WorkerSessionRecord,
|
||||
} from './workerSession';
|
||||
|
||||
export const MAX_WORKER_SESSION_OBSERVATION_PAGE_SIZE = 16;
|
||||
|
||||
export type WorkerSessionObservedLifecycle =
|
||||
| 'online'
|
||||
| 'draining'
|
||||
| 'offline'
|
||||
| 'lease_expired';
|
||||
|
||||
export type WorkerSessionObservedCompatibility =
|
||||
| 'default_placement'
|
||||
| 'explicit_placement_required'
|
||||
| 'protocol_incompatible';
|
||||
|
||||
export interface WorkerSessionObservationSummary {
|
||||
readonly workerId: string;
|
||||
readonly sessionId: string;
|
||||
readonly generation: number;
|
||||
readonly sessionVersion: number;
|
||||
readonly lifecycle: WorkerSessionObservedLifecycle;
|
||||
readonly compatibility: WorkerSessionObservedCompatibility;
|
||||
readonly architecture: RemoteWorkerArchitecture;
|
||||
readonly supportTier: RemoteWorkerSupportTier;
|
||||
readonly protocolVersion: string;
|
||||
readonly operatingSystem: string | null;
|
||||
readonly maxConcurrentRuns: number;
|
||||
readonly availableSlots: number;
|
||||
readonly registeredAtMs: number;
|
||||
readonly lastHeartbeatAtMs: number;
|
||||
readonly leaseExpiresAtMs: number;
|
||||
readonly updatedAtMs: number;
|
||||
readonly observedAtMs: number;
|
||||
}
|
||||
|
||||
export interface WorkerSessionObservation
|
||||
extends WorkerSessionObservationSummary {
|
||||
readonly runtimes: readonly Readonly<RemoteWorkerRuntimeCapability>[];
|
||||
readonly declaredCapacity: Readonly<{
|
||||
readonly cpuCores: number | null;
|
||||
readonly memoryBytes: number | null;
|
||||
readonly diskBytes: number | null;
|
||||
readonly gpuCount: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface WorkerSessionObservationPage {
|
||||
readonly observedAtMs: number;
|
||||
readonly workers: readonly Readonly<WorkerSessionObservationSummary>[];
|
||||
readonly nextCursor: string | null;
|
||||
}
|
||||
|
||||
export interface WorkerSessionInspection {
|
||||
readonly observedAtMs: number;
|
||||
readonly worker: Readonly<WorkerSessionObservation> | null;
|
||||
}
|
||||
|
||||
function invalid(message: string): never {
|
||||
throw new TypeError(`Worker Session observation is invalid: ${message}`);
|
||||
}
|
||||
|
||||
function observedLifecycle(
|
||||
worker: Readonly<WorkerSessionRecord>,
|
||||
observedAtMs: number,
|
||||
): WorkerSessionObservedLifecycle {
|
||||
if (worker.status === 'offline') return 'offline';
|
||||
if (worker.leaseExpiresAtMs <= observedAtMs) return 'lease_expired';
|
||||
return worker.status;
|
||||
}
|
||||
|
||||
export function projectWorkerSessionObservation(
|
||||
worker: Readonly<WorkerSessionRecord>,
|
||||
observedAtMs: number,
|
||||
): Readonly<WorkerSessionObservation> {
|
||||
assertWorkerSessionRecord(worker);
|
||||
if (
|
||||
!Number.isSafeInteger(observedAtMs) ||
|
||||
observedAtMs < worker.updatedAtMs
|
||||
) {
|
||||
return invalid('observedAtMs is invalid');
|
||||
}
|
||||
const capabilities = parseRemoteWorkerCapabilities(worker);
|
||||
const protocolCompatible = remoteWorkerProtocolIsCompatible(
|
||||
capabilities.protocolVersion,
|
||||
REMOTE_WORKER_PROTOCOL_RANGE,
|
||||
);
|
||||
const compatibility: WorkerSessionObservedCompatibility = protocolCompatible
|
||||
? capabilities.supportTier === 'tier1'
|
||||
? 'default_placement'
|
||||
: 'explicit_placement_required'
|
||||
: 'protocol_incompatible';
|
||||
return Object.freeze({
|
||||
workerId: worker.workerId,
|
||||
sessionId: worker.sessionId,
|
||||
generation: worker.generation,
|
||||
sessionVersion: worker.version,
|
||||
lifecycle: observedLifecycle(worker, observedAtMs),
|
||||
compatibility,
|
||||
architecture: capabilities.architecture,
|
||||
supportTier: capabilities.supportTier,
|
||||
protocolVersion: capabilities.protocolVersion,
|
||||
operatingSystem: capabilities.operatingSystem ?? null,
|
||||
maxConcurrentRuns: worker.maxConcurrentRuns,
|
||||
availableSlots: worker.availableSlots,
|
||||
registeredAtMs: worker.registeredAtMs,
|
||||
lastHeartbeatAtMs: worker.lastHeartbeatAtMs,
|
||||
leaseExpiresAtMs: worker.leaseExpiresAtMs,
|
||||
updatedAtMs: worker.updatedAtMs,
|
||||
observedAtMs,
|
||||
runtimes: Object.freeze(
|
||||
(capabilities.runtimes ?? []).map((runtime) =>
|
||||
Object.freeze({ ...runtime }),
|
||||
),
|
||||
),
|
||||
declaredCapacity: Object.freeze({
|
||||
cpuCores: capabilities.capacity?.cpuCores ?? null,
|
||||
memoryBytes: capabilities.capacity?.memoryBytes ?? null,
|
||||
diskBytes: capabilities.capacity?.diskBytes ?? null,
|
||||
gpuCount: capabilities.capacity?.gpu?.length ?? 0,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export function summarizeWorkerSessionObservation(
|
||||
observation: Readonly<WorkerSessionObservation>,
|
||||
): Readonly<WorkerSessionObservationSummary> {
|
||||
const {
|
||||
runtimes: _runtimes,
|
||||
declaredCapacity: _declaredCapacity,
|
||||
...summary
|
||||
} = observation;
|
||||
return Object.freeze(summary);
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
canonicalRemoteWorkerCapabilities,
|
||||
} = require('@qinglong/runtime-core/remote-dispatch');
|
||||
const {
|
||||
projectWorkerSessionObservation,
|
||||
summarizeWorkerSessionObservation,
|
||||
} = require('@qinglong/runtime-core/worker-session-observation');
|
||||
|
||||
function worker(overrides = {}) {
|
||||
const canonical = canonicalRemoteWorkerCapabilities(
|
||||
overrides.capabilities ?? {
|
||||
architecture: 'arm64',
|
||||
executors: ['remote-worker'],
|
||||
protocolVersion: '1.0.0',
|
||||
supportTier: 'tier1',
|
||||
operatingSystem: 'linux',
|
||||
runtimes: [{ name: 'node', version: '24.18.0' }],
|
||||
labels: { private: 'must-not-project' },
|
||||
capacity: {
|
||||
cpuCores: 2,
|
||||
memoryBytes: 512 * 1024 * 1024,
|
||||
diskBytes: 4 * 1024 * 1024 * 1024,
|
||||
gpu: [{ vendor: 'example', model: 'must-not-project' }],
|
||||
},
|
||||
features: ['must-not-project'],
|
||||
},
|
||||
);
|
||||
return Object.freeze({
|
||||
workerId: 'worker-a',
|
||||
sessionId: '018f0f5d-7b6a-7a11-8f4d-2f7b4f477001',
|
||||
generation: 3,
|
||||
status: 'online',
|
||||
version: 9,
|
||||
capabilitiesJson: canonical.json,
|
||||
capabilitiesHash: canonical.hash,
|
||||
maxConcurrentRuns: 4,
|
||||
availableSlots: 2,
|
||||
registeredAtMs: 1_000,
|
||||
lastHeartbeatAtMs: 1_900,
|
||||
leaseExpiresAtMs: 3_000,
|
||||
updatedAtMs: 1_900,
|
||||
...Object.fromEntries(
|
||||
Object.entries(overrides).filter(([key]) => key !== 'capabilities'),
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
test('projects only bounded Worker compatibility, runtime and capacity facts', () => {
|
||||
const observation = projectWorkerSessionObservation(worker(), 2_000);
|
||||
assert.deepEqual(observation, {
|
||||
workerId: 'worker-a',
|
||||
sessionId: '018f0f5d-7b6a-7a11-8f4d-2f7b4f477001',
|
||||
generation: 3,
|
||||
sessionVersion: 9,
|
||||
lifecycle: 'online',
|
||||
compatibility: 'default_placement',
|
||||
architecture: 'arm64',
|
||||
supportTier: 'tier1',
|
||||
protocolVersion: '1.0.0',
|
||||
operatingSystem: 'linux',
|
||||
maxConcurrentRuns: 4,
|
||||
availableSlots: 2,
|
||||
registeredAtMs: 1_000,
|
||||
lastHeartbeatAtMs: 1_900,
|
||||
leaseExpiresAtMs: 3_000,
|
||||
updatedAtMs: 1_900,
|
||||
observedAtMs: 2_000,
|
||||
runtimes: [{ name: 'node', version: '24.18.0' }],
|
||||
declaredCapacity: {
|
||||
cpuCores: 2,
|
||||
memoryBytes: 512 * 1024 * 1024,
|
||||
diskBytes: 4 * 1024 * 1024 * 1024,
|
||||
gpuCount: 1,
|
||||
},
|
||||
});
|
||||
assert.doesNotMatch(
|
||||
JSON.stringify(observation),
|
||||
/must-not-project|labels|features|model/,
|
||||
);
|
||||
const summary = summarizeWorkerSessionObservation(observation);
|
||||
assert.equal(Object.hasOwn(summary, 'runtimes'), false);
|
||||
assert.equal(Object.hasOwn(summary, 'declaredCapacity'), false);
|
||||
});
|
||||
|
||||
test('distinguishes explicit Tier, incompatible protocol and lifecycle state', () => {
|
||||
const candidate = projectWorkerSessionObservation(
|
||||
worker({
|
||||
capabilities: {
|
||||
architecture: 's390x',
|
||||
executors: ['remote-worker'],
|
||||
protocolVersion: '1.0.0',
|
||||
supportTier: 'candidate',
|
||||
},
|
||||
}),
|
||||
2_000,
|
||||
);
|
||||
assert.equal(candidate.compatibility, 'explicit_placement_required');
|
||||
|
||||
const incompatible = projectWorkerSessionObservation(
|
||||
worker({
|
||||
capabilities: {
|
||||
architecture: 'amd64',
|
||||
executors: ['remote-worker'],
|
||||
protocolVersion: '2.0.0',
|
||||
supportTier: 'tier1',
|
||||
},
|
||||
}),
|
||||
2_000,
|
||||
);
|
||||
assert.equal(incompatible.compatibility, 'protocol_incompatible');
|
||||
assert.equal(
|
||||
projectWorkerSessionObservation(worker(), 3_000).lifecycle,
|
||||
'lease_expired',
|
||||
);
|
||||
assert.equal(
|
||||
projectWorkerSessionObservation(
|
||||
worker({ status: 'draining', availableSlots: 0 }),
|
||||
2_000,
|
||||
).lifecycle,
|
||||
'draining',
|
||||
);
|
||||
assert.equal(
|
||||
projectWorkerSessionObservation(
|
||||
worker({
|
||||
status: 'offline',
|
||||
availableSlots: 0,
|
||||
leaseExpiresAtMs: 1_900,
|
||||
}),
|
||||
2_000,
|
||||
).lifecycle,
|
||||
'offline',
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects an observation clock before the durable Session head', () => {
|
||||
assert.throws(
|
||||
() => projectWorkerSessionObservation(worker(), 1_899),
|
||||
/observedAtMs is invalid/,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user