open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-23 03:18:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): expose bounded worker session observation

Browse Source
This commit is contained in:
whyour
2026-08-20 12:55:44 +08:00
parent 6265e31dce
commit 4a4fa85f51
24 changed files with 1437 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-cluster-admin/src/worker-credential/management-server/workerCredentialManagement.ts
+97 -1
View File
@@ -3,6 +3,7 @@ import {
PostgresApprovalRequestRepository,
PostgresProjectPolicyRepository,
PostgresWorkerCredentialManagementPlanRepository,
PostgresWorkerSessionObservationRepository,
} from '@qinglong/cluster-postgres/worker-credential-manager';
import type { PostgresPool } from '@qinglong/runtime-core';
import {
@@ -20,6 +21,10 @@ import {
type SecuritySubject,
} from '@qinglong/runtime-core/security';
import type { SecurityAuditRecord } from '@qinglong/runtime-core/security-audit';
import type {
WorkerSessionInspection,
WorkerSessionObservationPage,
} from '@qinglong/runtime-core/worker-session-observation';
import {
InvalidWorkerCredentialManagementPlanError,
MAX_WORKER_CREDENTIAL_MANAGEMENT_PLAN_LIFETIME_MS,
@@ -93,7 +98,8 @@ export type WorkerCredentialManagementQuotaOperation =
| 'worker-credential.plan'
| 'worker-credential.propose'
| 'worker-credential.decide'
| 'worker-credential.inspect';
| 'worker-credential.inspect'
| 'worker-session.observe';
export interface WorkerCredentialManagementQuotaPort {
consume(
@@ -166,6 +172,20 @@ export interface InspectClusterWorkerCredentialResult {
readonly stale: boolean;
}
export interface InspectClusterWorkerSessionRequest {
readonly authorityProjectId: string;
readonly workerId: string;
readonly inspectionId: string;
readonly principal: SecurityPrincipal;
}
export interface ListClusterWorkerSessionsRequest {
readonly authorityProjectId: string;
readonly afterWorkerId: string | null;
readonly inspectionId: string;
readonly principal: SecurityPrincipal;
}
export interface ClusterWorkerCredentialManagementService {
plan(
request: PlanClusterWorkerCredentialRequest,
@@ -179,6 +199,12 @@ export interface ClusterWorkerCredentialManagementService {
inspectAuthorized(
request: InspectClusterWorkerCredentialRequest,
): Promise<Readonly<InspectClusterWorkerCredentialResult>>;
inspectSession(
request: InspectClusterWorkerSessionRequest,
): Promise<Readonly<WorkerSessionInspection>>;
listSessions(
request: ListClusterWorkerSessionsRequest,
): Promise<Readonly<WorkerSessionObservationPage>>;
}
export interface ClusterWorkerCredentialManagementOptions {
@@ -324,6 +350,7 @@ export function createClusterWorkerCredentialManagementService(
const plans = new PostgresWorkerCredentialManagementPlanRepository(
options.pool,
);
const sessions = new PostgresWorkerSessionObservationRepository(options.pool);
const approvals = new PostgresApprovalRequestRepository(options.pool);
const policy = new ProjectPolicyEngine(
new PostgresProjectPolicyRepository(options.pool),
@@ -796,5 +823,74 @@ export function createClusterWorkerCredentialManagementService(
observedAtMs > plan.expiresAtMs,
});
},
async inspectSession(request: InspectClusterWorkerSessionRequest) {
exact(
request,
['authorityProjectId', 'inspectionId', 'principal', 'workerId'],
'Session inspection request',
);
const projectId = identifier(
request.authorityProjectId,
'authorityProjectId',
);
const inspectionId = identifier(request.inspectionId, 'inspectionId');
const workerId = identifier(request.workerId, 'workerId');
const authorization = await authorize(
request.principal,
projectId,
'worker.manage',
currentTime(now),
);
await consumeQuota(
projectId,
authorization.principal,
'worker-session.observe',
`session-inspect:${inspectionId}`,
);
try {
return await sessions.inspect(workerId);
} catch (error) {
throw new WorkerCredentialManagementUnavailableError({
cause: error instanceof Error ? error : undefined,
});
}
},
async listSessions(request: ListClusterWorkerSessionsRequest) {
exact(
request,
['afterWorkerId', 'authorityProjectId', 'inspectionId', 'principal'],
'Session list request',
);
const projectId = identifier(
request.authorityProjectId,
'authorityProjectId',
);
const inspectionId = identifier(request.inspectionId, 'inspectionId');
const afterWorkerId =
request.afterWorkerId === null
? null
: identifier(request.afterWorkerId, 'afterWorkerId');
const authorization = await authorize(
request.principal,
projectId,
'worker.manage',
currentTime(now),
);
await consumeQuota(
projectId,
authorization.principal,
'worker-session.observe',
`session-list:${inspectionId}`,
);
try {
return await sessions.list(afterWorkerId);
} catch (error) {
throw new WorkerCredentialManagementUnavailableError({
cause: error instanceof Error ? error : undefined,
});
}
},
});
}
packages/ql3-cluster-admin/src/worker-credential/management-server/workerCredentialManagementTransport.ts
+81 -1
View File
@@ -74,11 +74,33 @@ export interface InspectClusterWorkerCredentialCommand {
};
}
export interface InspectClusterWorkerSessionCommand {
readonly schemaVersion: 1;
readonly operation: 'worker-session.inspect';
readonly request: {
readonly authorityProjectId: string;
readonly workerId: string;
readonly inspectionId: string;
};
}
export interface ListClusterWorkerSessionsCommand {
readonly schemaVersion: 1;
readonly operation: 'worker-session.list';
readonly request: {
readonly authorityProjectId: string;
readonly afterWorkerId: string | null;
readonly inspectionId: string;
};
}
export type ClusterWorkerCredentialManagementCommand =
| PlanClusterWorkerCredentialCommand
| ProposeClusterWorkerCredentialCommand
| DecideClusterWorkerCredentialCommand
| InspectClusterWorkerCredentialCommand;
| InspectClusterWorkerCredentialCommand
| InspectClusterWorkerSessionCommand
| ListClusterWorkerSessionsCommand;
type PlanSummary = ReturnType<typeof planSummary>;
type ApprovalSummary = ReturnType<typeof approvalSummary>;
@@ -109,6 +131,23 @@ export type ClusterWorkerCredentialManagementTransportResult =
plan: PlanSummary | null;
approval: ApprovalSummary | null;
stale: boolean;
}>
| Readonly<{
schemaVersion: 1;
operation: 'worker-session.inspect';
observedAtMs: number;
worker: Awaited<
ReturnType<ClusterWorkerCredentialManagementService['inspectSession']>
>['worker'];
}>
| Readonly<{
schemaVersion: 1;
operation: 'worker-session.list';
observedAtMs: number;
workers: Awaited<
ReturnType<ClusterWorkerCredentialManagementService['listSessions']>
>['workers'];
nextCursor: string | null;
}>;
export interface ClusterWorkerCredentialManagementTransport {
@@ -254,6 +293,20 @@ export function normalizeClusterWorkerCredentialManagementCommand(
'inspection request',
);
break;
case 'worker-session.inspect':
exactObject(
value.request,
['authorityProjectId', 'inspectionId', 'workerId'],
'Session inspection request',
);
break;
case 'worker-session.list':
exactObject(
value.request,
['afterWorkerId', 'authorityProjectId', 'inspectionId'],
'Session list request',
);
break;
default:
throw new ClusterWorkerCredentialManagementTransportRequestError(
'operation is not publicly available',
@@ -317,6 +370,8 @@ export function createClusterWorkerCredentialManagementTransport(
typeof options.service.propose !== 'function' ||
typeof options.service.decide !== 'function' ||
typeof options.service.inspectAuthorized !== 'function' ||
typeof options.service.inspectSession !== 'function' ||
typeof options.service.listSessions !== 'function' ||
(options.now !== undefined && typeof options.now !== 'function')
) {
throw new ClusterWorkerCredentialManagementTransportConfigurationError(
@@ -420,6 +475,31 @@ export function createClusterWorkerCredentialManagementTransport(
stale: result.stale,
});
}
case 'worker-session.inspect': {
const result = await options.service.inspectSession({
...command.request,
principal,
});
return Object.freeze({
schemaVersion: 1 as const,
operation: command.operation,
observedAtMs: result.observedAtMs,
worker: result.worker,
});
}
case 'worker-session.list': {
const result = await options.service.listSessions({
...command.request,
principal,
});
return Object.freeze({
schemaVersion: 1 as const,
operation: command.operation,
observedAtMs: result.observedAtMs,
workers: result.workers,
nextCursor: result.nextCursor,
});
}
}
},
});
packages/ql3-cluster-admin/src/worker-credential/workerCredentialManagementClient.ts
+164 -2
View File
@@ -57,6 +57,23 @@ function boundedScalar(value: unknown): void {
}
}
function boundedText(value: unknown, nullable = false): void {
if (value === null && nullable) return;
if (
typeof value !== 'string' ||
value.length === 0 ||
value.length > 2_048 ||
CONTROL_PATTERN.test(value)
) {
invalid();
}
}
function nonNegativeInteger(value: unknown, nullable = false): void {
if (value === null && nullable) return;
if (!Number.isSafeInteger(value) || (value as number) < 0) invalid();
}
function subject(value: unknown): void {
const record = exactRecord(value, ['type', 'id']);
if (record.type !== 'user') invalid();
@@ -131,6 +148,106 @@ function approval(value: unknown): void {
}
}
const WORKER_OBSERVATION_KEYS = [
'architecture',
'availableSlots',
'compatibility',
'generation',
'lastHeartbeatAtMs',
'leaseExpiresAtMs',
'lifecycle',
'maxConcurrentRuns',
'observedAtMs',
'operatingSystem',
'protocolVersion',
'registeredAtMs',
'sessionId',
'sessionVersion',
'supportTier',
'updatedAtMs',
'workerId',
] as const;
function workerObservation(value: unknown, detailed: boolean): void {
const record = exactRecord(
value,
detailed
? [...WORKER_OBSERVATION_KEYS, 'declaredCapacity', 'runtimes']
: WORKER_OBSERVATION_KEYS,
);
boundedText(record.workerId);
boundedText(record.sessionId);
boundedText(record.protocolVersion);
boundedText(record.operatingSystem, true);
for (const key of [
'generation',
'sessionVersion',
'maxConcurrentRuns',
'availableSlots',
'registeredAtMs',
'lastHeartbeatAtMs',
'leaseExpiresAtMs',
'updatedAtMs',
'observedAtMs',
] as const) {
nonNegativeInteger(record[key]);
}
if (
!['online', 'draining', 'offline', 'lease_expired'].includes(
String(record.lifecycle),
) ||
![
'default_placement',
'explicit_placement_required',
'protocol_incompatible',
].includes(String(record.compatibility)) ||
!['tier1', 'candidate', 'experimental', 'legacy-only'].includes(
String(record.supportTier),
) ||
!['amd64', 'arm64', 'ppc64le', 's390x', 'arm/v7', 'arm/v6', '386'].includes(
String(record.architecture),
)
) {
invalid();
}
if (
(record.generation as number) < 1 ||
(record.maxConcurrentRuns as number) < 1 ||
(record.availableSlots as number) > (record.maxConcurrentRuns as number) ||
(record.lastHeartbeatAtMs as number) < (record.registeredAtMs as number) ||
(record.updatedAtMs as number) < (record.lastHeartbeatAtMs as number) ||
(record.observedAtMs as number) < (record.updatedAtMs as number) ||
(record.lifecycle !== 'offline' &&
(record.leaseExpiresAtMs as number) <=
(record.lastHeartbeatAtMs as number)) ||
(['online', 'draining'].includes(String(record.lifecycle)) &&
(record.leaseExpiresAtMs as number) <= (record.observedAtMs as number)) ||
(record.lifecycle === 'lease_expired' &&
(record.leaseExpiresAtMs as number) > (record.observedAtMs as number)) ||
(['draining', 'offline'].includes(String(record.lifecycle)) &&
(record.availableSlots as number) !== 0)
) {
invalid();
}
if (!detailed) return;
if (!Array.isArray(record.runtimes) || record.runtimes.length > 32) invalid();
for (const runtime of record.runtimes) {
const item = exactRecord(runtime, ['name', 'version']);
boundedText(item.name);
boundedText(item.version);
}
const capacity = exactRecord(record.declaredCapacity, [
'cpuCores',
'diskBytes',
'gpuCount',
'memoryBytes',
]);
nonNegativeInteger(capacity.cpuCores, true);
nonNegativeInteger(capacity.memoryBytes, true);
nonNegativeInteger(capacity.diskBytes, true);
nonNegativeInteger(capacity.gpuCount);
}
export function validateClusterWorkerCredentialManagementClientResult(
value: unknown,
command: Readonly<ClusterWorkerCredentialManagementCommand>,
@@ -143,7 +260,17 @@ export function validateClusterWorkerCredentialManagementClientResult(
? ['schemaVersion', 'operation', 'approvalStatus', 'plan', 'approval']
: operation === 'worker-credential.decide'
? ['schemaVersion', 'operation', 'status', 'approval']
: ['schemaVersion', 'operation', 'plan', 'approval', 'stale'];
: operation === 'worker-credential.inspect'
? ['schemaVersion', 'operation', 'plan', 'approval', 'stale']
: operation === 'worker-session.inspect'
? ['schemaVersion', 'operation', 'observedAtMs', 'worker']
: [
'schemaVersion',
'operation',
'observedAtMs',
'workers',
'nextCursor',
];
const record = exactRecord(value, keys);
if (record.schemaVersion !== 1 || record.operation !== operation) invalid();
if (operation === 'worker-credential.plan') {
@@ -157,10 +284,45 @@ export function validateClusterWorkerCredentialManagementClientResult(
} else if (operation === 'worker-credential.decide') {
if (!['decided', 'existing'].includes(String(record.status))) invalid();
approval(record.approval);
} else {
} else if (operation === 'worker-credential.inspect') {
if (typeof record.stale !== 'boolean') invalid();
if (record.plan !== null) plan(record.plan);
if (record.approval !== null) approval(record.approval);
} else if (operation === 'worker-session.inspect') {
nonNegativeInteger(record.observedAtMs);
if (record.worker !== null) {
workerObservation(record.worker, true);
if (
(record.worker as Record<string, unknown>).observedAtMs !==
record.observedAtMs
) {
invalid();
}
}
} else {
nonNegativeInteger(record.observedAtMs);
boundedText(record.nextCursor, true);
if (!Array.isArray(record.workers) || record.workers.length > 16) invalid();
let previous = '';
for (const worker of record.workers) {
workerObservation(worker, false);
const item = worker as Record<string, unknown>;
if (
item.observedAtMs !== record.observedAtMs ||
typeof item.workerId !== 'string' ||
item.workerId <= previous
) {
invalid();
}
previous = item.workerId;
}
if (
(record.nextCursor !== null &&
(record.workers.length === 0 || record.nextCursor !== previous)) ||
(record.nextCursor === null && record.workers.length > 16)
) {
invalid();
}
}
return Object.freeze(
record as unknown as ClusterWorkerCredentialManagementTransportResult,