mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-22 10:32:40 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,260 @@
|
||||
// Remote Execution owns mounted Secret resolution for authenticated delivery.
|
||||
import { createHash } from 'node:crypto';
|
||||
import { constants } from 'node:fs';
|
||||
import {
|
||||
lstat,
|
||||
open,
|
||||
realpath,
|
||||
} from 'node:fs/promises';
|
||||
import {
|
||||
isAbsolute,
|
||||
join,
|
||||
normalize,
|
||||
parse,
|
||||
relative,
|
||||
} from 'node:path';
|
||||
import {
|
||||
MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES,
|
||||
MAX_REMOTE_SECRET_VALUE_BYTES,
|
||||
normalizeRemoteWorkerSecretDeliveryAuthority,
|
||||
type RemoteWorkerSecretDeliveryAuthority,
|
||||
type RemoteWorkerSecretResolution,
|
||||
type RemoteWorkerSecretValueProvider,
|
||||
} from '@qinglong/runtime-core/remote-secret-delivery';
|
||||
import { parseSecretRef } from '@qinglong/runtime-core/secret-reference';
|
||||
|
||||
const MAX_SECRET_ROOT_BYTES = 4096;
|
||||
const SECRET_FILE_NAME = /^[0-9a-f]{64}$/;
|
||||
|
||||
export interface ClusterMountedSecretProviderOptions {
|
||||
/**
|
||||
* Read-only directory whose file names are SHA-256(canonical SecretRef).
|
||||
* Kubernetes projected-volume symlinks are accepted only when their resolved
|
||||
* regular file remains below this directory.
|
||||
*/
|
||||
readonly rootDirectory: string;
|
||||
}
|
||||
|
||||
export class ClusterMountedSecretProviderError extends Error {
|
||||
readonly code = 'QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE';
|
||||
|
||||
constructor(
|
||||
readonly reason:
|
||||
| 'invalid_configuration'
|
||||
| 'root_unavailable'
|
||||
| 'material_unavailable',
|
||||
options?: ErrorOptions,
|
||||
) {
|
||||
super(`Cluster mounted Secret provider failed: ${reason}`, options);
|
||||
this.name = 'ClusterMountedSecretProviderError';
|
||||
}
|
||||
}
|
||||
|
||||
function rootDirectory(value: string): string {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
!isAbsolute(value) ||
|
||||
parse(value).root === value ||
|
||||
normalize(value) !== value ||
|
||||
value.includes('\0') ||
|
||||
Buffer.byteLength(value, 'utf8') > MAX_SECRET_ROOT_BYTES
|
||||
) {
|
||||
throw new ClusterMountedSecretProviderError('invalid_configuration');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Kubernetes Secret keys cannot contain a SecretRef directly. This stable,
|
||||
* non-reversible name also prevents Project/name input from becoming a path.
|
||||
*/
|
||||
export function clusterMountedSecretFileName(secretRef: string): string {
|
||||
let canonical: string;
|
||||
try {
|
||||
const parsed = parseSecretRef(secretRef);
|
||||
canonical = secretRef;
|
||||
if (
|
||||
parsed.projectId.length < 1 ||
|
||||
parsed.name.length < 1
|
||||
) throw new Error('invalid SecretRef');
|
||||
} catch (error) {
|
||||
throw new ClusterMountedSecretProviderError(
|
||||
'invalid_configuration',
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
const name = createHash('sha256').update(canonical, 'utf8').digest('hex');
|
||||
if (!SECRET_FILE_NAME.test(name)) {
|
||||
throw new ClusterMountedSecretProviderError('invalid_configuration');
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
function remainsBelow(root: string, candidate: string): boolean {
|
||||
const suffix = relative(root, candidate);
|
||||
return (
|
||||
suffix.length > 0 &&
|
||||
!isAbsolute(suffix) &&
|
||||
suffix !== '..' &&
|
||||
!suffix.startsWith(`..${process.platform === 'win32' ? '\\' : '/'}`)
|
||||
);
|
||||
}
|
||||
|
||||
async function resolvedRoot(path: string): Promise<string> {
|
||||
try {
|
||||
const configured = await lstat(path);
|
||||
if (!configured.isDirectory() || configured.isSymbolicLink()) {
|
||||
throw new Error('root is not a direct directory');
|
||||
}
|
||||
return await realpath(path);
|
||||
} catch (error) {
|
||||
throw new ClusterMountedSecretProviderError(
|
||||
'root_unavailable',
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function readMaterial(
|
||||
root: string,
|
||||
secretRef: string,
|
||||
): Promise<Buffer> {
|
||||
const candidate = join(root, clusterMountedSecretFileName(secretRef));
|
||||
let handle;
|
||||
try {
|
||||
const target = await realpath(candidate);
|
||||
if (!remainsBelow(root, target)) {
|
||||
throw new Error('material escaped its root');
|
||||
}
|
||||
handle = await open(
|
||||
target,
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
const stat = await handle.stat();
|
||||
if (
|
||||
!stat.isFile() ||
|
||||
stat.nlink !== 1 ||
|
||||
stat.size < 0 ||
|
||||
stat.size > MAX_REMOTE_SECRET_VALUE_BYTES ||
|
||||
(stat.mode & 0o111) !== 0 ||
|
||||
(stat.mode & 0o027) !== 0
|
||||
) {
|
||||
throw new Error('material metadata is unsafe');
|
||||
}
|
||||
const bytes = await handle.readFile();
|
||||
if (
|
||||
bytes.byteLength !== stat.size ||
|
||||
bytes.byteLength > MAX_REMOTE_SECRET_VALUE_BYTES ||
|
||||
(await realpath(candidate)) !== target
|
||||
) {
|
||||
bytes.fill(0);
|
||||
throw new Error('material changed while reading');
|
||||
}
|
||||
return bytes;
|
||||
} catch (error) {
|
||||
throw new ClusterMountedSecretProviderError(
|
||||
'material_unavailable',
|
||||
{ cause: error },
|
||||
);
|
||||
} finally {
|
||||
await handle?.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
function secretValue(bytes: Buffer): string {
|
||||
try {
|
||||
const value = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
||||
if (value.includes('\0')) {
|
||||
throw new Error('Secret contains NUL');
|
||||
}
|
||||
return value;
|
||||
} catch (error) {
|
||||
throw new ClusterMountedSecretProviderError(
|
||||
'material_unavailable',
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A zero-client, zero-watcher Cluster provider for Kubernetes Secret, CSI or
|
||||
* operator-managed projected files. Every authorized delivery resolves the
|
||||
* active files again, so atomic projection replacement rotates material
|
||||
* without a timer, cache, control restart or Kubernetes API permission.
|
||||
*/
|
||||
export class ClusterMountedSecretProvider
|
||||
implements RemoteWorkerSecretValueProvider
|
||||
{
|
||||
private readonly rootDirectory: string;
|
||||
|
||||
constructor(options: ClusterMountedSecretProviderOptions) {
|
||||
if (!options || typeof options !== 'object' || Array.isArray(options)) {
|
||||
throw new ClusterMountedSecretProviderError('invalid_configuration');
|
||||
}
|
||||
this.rootDirectory = rootDirectory(options.rootDirectory);
|
||||
}
|
||||
|
||||
async verify(): Promise<void> {
|
||||
await resolvedRoot(this.rootDirectory);
|
||||
}
|
||||
|
||||
async resolve(
|
||||
authority: Readonly<RemoteWorkerSecretDeliveryAuthority>,
|
||||
): Promise<Readonly<RemoteWorkerSecretResolution>> {
|
||||
let normalized: Readonly<RemoteWorkerSecretDeliveryAuthority>;
|
||||
try {
|
||||
normalized = normalizeRemoteWorkerSecretDeliveryAuthority(authority);
|
||||
} catch (error) {
|
||||
throw new ClusterMountedSecretProviderError(
|
||||
'material_unavailable',
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
const root = await resolvedRoot(this.rootDirectory);
|
||||
const buffers: Buffer[] = [];
|
||||
try {
|
||||
const values = [];
|
||||
let totalBytes = 0;
|
||||
for (const secretRef of normalized.secretRefs) {
|
||||
const bytes = await readMaterial(root, secretRef);
|
||||
buffers.push(bytes);
|
||||
totalBytes += bytes.byteLength;
|
||||
if (totalBytes > MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES) {
|
||||
throw new ClusterMountedSecretProviderError(
|
||||
'material_unavailable',
|
||||
);
|
||||
}
|
||||
values.push(
|
||||
Object.freeze({
|
||||
secretRef,
|
||||
value: secretValue(bytes),
|
||||
}),
|
||||
);
|
||||
}
|
||||
let disposed = false;
|
||||
return Object.freeze({
|
||||
values: Object.freeze(values),
|
||||
dispose() {
|
||||
if (disposed) return;
|
||||
disposed = true;
|
||||
for (const bytes of buffers) bytes.fill(0);
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
for (const bytes of buffers) bytes.fill(0);
|
||||
if (error instanceof ClusterMountedSecretProviderError) throw error;
|
||||
throw new ClusterMountedSecretProviderError(
|
||||
'material_unavailable',
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function createClusterMountedSecretProvider(
|
||||
options: ClusterMountedSecretProviderOptions,
|
||||
): Promise<Readonly<ClusterMountedSecretProvider>> {
|
||||
const provider = new ClusterMountedSecretProvider(options);
|
||||
await provider.verify();
|
||||
return provider;
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
// Remote execution owns Worker-bound activation acknowledgements and start failure fencing.
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type {
|
||||
AcknowledgeRemoteRunRunningCommand,
|
||||
AcknowledgeRemoteRunStartingCommand,
|
||||
FailRemoteRunStartCommand,
|
||||
RemoteRunActivationRepository,
|
||||
RemoteRunActivationResult,
|
||||
} from '@qinglong/runtime-core/remote-activation';
|
||||
|
||||
export interface ClusterRemoteRunActivationPrincipal {
|
||||
readonly workerId: string;
|
||||
}
|
||||
|
||||
type ServerOwnedStartingFields = 'workerId' | 'eventId';
|
||||
type ServerOwnedRunningFields = 'workerId' | 'attemptEventId' | 'runEventId';
|
||||
|
||||
export type AcknowledgeClusterRemoteRunStartingCommand = Omit<
|
||||
AcknowledgeRemoteRunStartingCommand,
|
||||
ServerOwnedStartingFields
|
||||
>;
|
||||
|
||||
export type AcknowledgeClusterRemoteRunRunningCommand = Omit<
|
||||
AcknowledgeRemoteRunRunningCommand,
|
||||
ServerOwnedRunningFields
|
||||
>;
|
||||
|
||||
export type FailClusterRemoteRunStartCommand = Omit<
|
||||
FailRemoteRunStartCommand,
|
||||
ServerOwnedRunningFields
|
||||
>;
|
||||
|
||||
export interface ClusterRemoteRunActivationServiceOptions {
|
||||
readonly createEventId?: () => string;
|
||||
}
|
||||
|
||||
export class ClusterRemoteRunActivationService {
|
||||
private readonly createEventId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly repository: RemoteRunActivationRepository,
|
||||
options: ClusterRemoteRunActivationServiceOptions = {},
|
||||
) {
|
||||
if (
|
||||
!repository ||
|
||||
typeof repository.acknowledgeStarting !== 'function' ||
|
||||
typeof repository.acknowledgeRunning !== 'function' ||
|
||||
typeof repository.failStart !== 'function'
|
||||
) {
|
||||
throw new TypeError('Remote Run activation repository is invalid');
|
||||
}
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some((key) => key !== 'createEventId')
|
||||
) {
|
||||
throw new TypeError('Remote Run activation service options are invalid');
|
||||
}
|
||||
this.createEventId = options.createEventId ?? randomUUID;
|
||||
if (typeof this.createEventId !== 'function') {
|
||||
throw new TypeError('Remote Run activation event ID factory is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
acknowledgeStarting(
|
||||
principal: ClusterRemoteRunActivationPrincipal,
|
||||
command: AcknowledgeClusterRemoteRunStartingCommand,
|
||||
): Promise<Readonly<RemoteRunActivationResult>> {
|
||||
this.assertPrincipal(principal);
|
||||
return this.repository.acknowledgeStarting({
|
||||
...command,
|
||||
workerId: principal.workerId,
|
||||
eventId: this.createEventId(),
|
||||
});
|
||||
}
|
||||
|
||||
acknowledgeRunning(
|
||||
principal: ClusterRemoteRunActivationPrincipal,
|
||||
command: AcknowledgeClusterRemoteRunRunningCommand,
|
||||
): Promise<Readonly<RemoteRunActivationResult>> {
|
||||
this.assertPrincipal(principal);
|
||||
return this.repository.acknowledgeRunning({
|
||||
...command,
|
||||
workerId: principal.workerId,
|
||||
attemptEventId: this.createEventId(),
|
||||
runEventId: this.createEventId(),
|
||||
});
|
||||
}
|
||||
|
||||
failStart(
|
||||
principal: ClusterRemoteRunActivationPrincipal,
|
||||
command: FailClusterRemoteRunStartCommand,
|
||||
): Promise<Readonly<RemoteRunActivationResult>> {
|
||||
this.assertPrincipal(principal);
|
||||
return this.repository.failStart({
|
||||
...command,
|
||||
workerId: principal.workerId,
|
||||
attemptEventId: this.createEventId(),
|
||||
runEventId: this.createEventId(),
|
||||
});
|
||||
}
|
||||
|
||||
private assertPrincipal(principal: ClusterRemoteRunActivationPrincipal): void {
|
||||
if (
|
||||
!principal ||
|
||||
typeof principal !== 'object' ||
|
||||
Array.isArray(principal) ||
|
||||
!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(principal.workerId)
|
||||
) {
|
||||
throw new TypeError('Remote Run activation principal is invalid');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
// Remote execution owns immutable Artifact admission and fenced Worker completion.
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
InvalidRemoteWorkerCompletionError,
|
||||
MAX_REMOTE_WORKER_ARTIFACT_HEADER_BYTES,
|
||||
RemoteWorkerCompletionFenceRejectedError,
|
||||
RemoteWorkerCompletionUnavailableError,
|
||||
normalizeRemoteWorkerArtifactReceipt,
|
||||
normalizeRemoteWorkerCompletionCommand,
|
||||
normalizeRemoteWorkerCompletionResult,
|
||||
parseRemoteWorkerArtifactUploadHeader,
|
||||
type RemoteWorkerArtifactReceipt,
|
||||
type RemoteWorkerArtifactUploadAuthorityRepository,
|
||||
type RemoteWorkerArtifactUploadCommand,
|
||||
type RemoteWorkerCompletionCommand,
|
||||
type RemoteWorkerCompletionRepository,
|
||||
type RemoteWorkerCompletionResult,
|
||||
} from '@qinglong/runtime-core/remote-worker-completion';
|
||||
|
||||
export interface ClusterRemoteWorkerArtifactStorageCommand {
|
||||
readonly projectId: string;
|
||||
readonly runId: string;
|
||||
readonly attemptId: string;
|
||||
readonly logArtifactId: string;
|
||||
readonly byteLength: number;
|
||||
readonly truncated?: boolean;
|
||||
}
|
||||
|
||||
export interface ClusterRemoteWorkerArtifactLookup {
|
||||
readonly projectId: string;
|
||||
readonly runId: string;
|
||||
readonly attemptId: string;
|
||||
readonly logArtifactId: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Production implementations must be shared by every cluster-control replica
|
||||
* and provide immutable, digest-authenticated put-if-absent semantics.
|
||||
*/
|
||||
export interface ClusterRemoteWorkerArtifactStore {
|
||||
put(
|
||||
command: Readonly<ClusterRemoteWorkerArtifactStorageCommand>,
|
||||
content: AsyncIterable<Uint8Array>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Readonly<RemoteWorkerArtifactReceipt>>;
|
||||
inspect(
|
||||
lookup: Readonly<ClusterRemoteWorkerArtifactLookup>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Readonly<RemoteWorkerArtifactReceipt> | undefined>;
|
||||
}
|
||||
|
||||
export interface ClusterRemoteWorkerArtifactUploadInput {
|
||||
readonly workerId: string;
|
||||
readonly workerSessionId: string;
|
||||
readonly contentLength: number;
|
||||
readonly chunks: AsyncIterable<Uint8Array>;
|
||||
readonly signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface ClusterRemoteWorkerCompletionServiceOptions {
|
||||
readonly createEventId?: () => string;
|
||||
}
|
||||
|
||||
class BoundedArtifactStreamReader {
|
||||
private readonly iterator: AsyncIterator<Uint8Array>;
|
||||
private pending: Uint8Array | undefined;
|
||||
private pendingOffset = 0;
|
||||
|
||||
constructor(
|
||||
source: AsyncIterable<Uint8Array>,
|
||||
private readonly signal?: AbortSignal,
|
||||
) {
|
||||
if (!source || typeof source[Symbol.asyncIterator] !== 'function') {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact upload stream is invalid',
|
||||
);
|
||||
}
|
||||
this.iterator = source[Symbol.asyncIterator]();
|
||||
}
|
||||
|
||||
async readExactly(byteLength: number): Promise<Buffer> {
|
||||
const result = Buffer.allocUnsafe(byteLength);
|
||||
let written = 0;
|
||||
try {
|
||||
while (written < byteLength) {
|
||||
const chunk = await this.nextChunk();
|
||||
if (!chunk) {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact upload stream ended before its header',
|
||||
);
|
||||
}
|
||||
const available = chunk.byteLength - this.pendingOffset;
|
||||
const copied = Math.min(available, byteLength - written);
|
||||
Buffer.from(
|
||||
chunk.buffer,
|
||||
chunk.byteOffset + this.pendingOffset,
|
||||
copied,
|
||||
).copy(result, written);
|
||||
written += copied;
|
||||
this.pendingOffset += copied;
|
||||
if (this.pendingOffset === chunk.byteLength) {
|
||||
this.pending = undefined;
|
||||
this.pendingOffset = 0;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
result.fill(0);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
content(byteLength: number): Readonly<{
|
||||
chunks: AsyncIterable<Uint8Array>;
|
||||
isComplete(): boolean;
|
||||
}> {
|
||||
let complete = false;
|
||||
let started = false;
|
||||
const self = this;
|
||||
const chunks = Object.freeze({
|
||||
async *[Symbol.asyncIterator](): AsyncGenerator<Uint8Array> {
|
||||
if (started) {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact content can only be consumed once',
|
||||
);
|
||||
}
|
||||
started = true;
|
||||
let total = 0;
|
||||
while (true) {
|
||||
const chunk = await self.nextChunk();
|
||||
if (!chunk) break;
|
||||
const bytes = chunk.subarray(self.pendingOffset);
|
||||
self.pending = undefined;
|
||||
self.pendingOffset = 0;
|
||||
total += bytes.byteLength;
|
||||
if (total > byteLength) {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact content exceeds its declared length',
|
||||
);
|
||||
}
|
||||
yield bytes;
|
||||
}
|
||||
if (total !== byteLength) {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact content does not match its declared length',
|
||||
);
|
||||
}
|
||||
complete = true;
|
||||
},
|
||||
});
|
||||
return Object.freeze({ chunks, isComplete: () => complete });
|
||||
}
|
||||
|
||||
private async nextChunk(): Promise<Uint8Array | undefined> {
|
||||
if (this.signal?.aborted) throw this.signal.reason;
|
||||
if (this.pending) return this.pending;
|
||||
const next = await this.iterator.next();
|
||||
if (next.done) return undefined;
|
||||
if (!(next.value instanceof Uint8Array) || next.value.byteLength === 0) {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact upload chunk is invalid',
|
||||
);
|
||||
}
|
||||
this.pending = next.value;
|
||||
this.pendingOffset = 0;
|
||||
return this.pending;
|
||||
}
|
||||
}
|
||||
|
||||
function storageCommand(
|
||||
command: RemoteWorkerArtifactUploadCommand,
|
||||
): Readonly<ClusterRemoteWorkerArtifactStorageCommand> {
|
||||
return Object.freeze({
|
||||
projectId: command.projectId,
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
logArtifactId: command.logArtifactId,
|
||||
byteLength: command.byteLength,
|
||||
...(command.truncated === undefined
|
||||
? {}
|
||||
: { truncated: command.truncated }),
|
||||
});
|
||||
}
|
||||
|
||||
function assertReceiptMatches(
|
||||
command: ClusterRemoteWorkerArtifactStorageCommand,
|
||||
value: RemoteWorkerArtifactReceipt,
|
||||
): Readonly<RemoteWorkerArtifactReceipt> {
|
||||
const receipt = normalizeRemoteWorkerArtifactReceipt(value);
|
||||
if (
|
||||
receipt.projectId !== command.projectId ||
|
||||
receipt.runId !== command.runId ||
|
||||
receipt.attemptId !== command.attemptId ||
|
||||
receipt.logArtifactId !== command.logArtifactId ||
|
||||
receipt.byteLength !== command.byteLength ||
|
||||
receipt.truncated !== command.truncated
|
||||
) {
|
||||
throw new RemoteWorkerCompletionUnavailableError();
|
||||
}
|
||||
return receipt;
|
||||
}
|
||||
|
||||
function eventId(factory: () => string): string {
|
||||
const value = factory();
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
value.length > 36 ||
|
||||
/[\u0000-\u001f\u007f]/.test(value)
|
||||
) {
|
||||
throw new RemoteWorkerCompletionUnavailableError();
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export class ClusterRemoteWorkerArtifactService {
|
||||
constructor(
|
||||
private readonly authority: RemoteWorkerArtifactUploadAuthorityRepository,
|
||||
private readonly store: ClusterRemoteWorkerArtifactStore,
|
||||
) {
|
||||
if (
|
||||
typeof authority?.authorizeArtifactUpload !== 'function' ||
|
||||
typeof store?.put !== 'function' ||
|
||||
typeof store?.inspect !== 'function'
|
||||
) {
|
||||
throw new TypeError('Remote Worker Artifact service is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
async upload(
|
||||
input: ClusterRemoteWorkerArtifactUploadInput,
|
||||
): Promise<Readonly<RemoteWorkerArtifactReceipt>> {
|
||||
if (
|
||||
!input ||
|
||||
!Number.isSafeInteger(input.contentLength) ||
|
||||
input.contentLength < 6
|
||||
) {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact upload envelope is invalid',
|
||||
);
|
||||
}
|
||||
const reader = new BoundedArtifactStreamReader(input.chunks, input.signal);
|
||||
const prefix = await reader.readExactly(4);
|
||||
const headerLength = prefix.readUInt32BE(0);
|
||||
prefix.fill(0);
|
||||
if (
|
||||
headerLength < 2 ||
|
||||
headerLength > MAX_REMOTE_WORKER_ARTIFACT_HEADER_BYTES
|
||||
) {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact upload header length is invalid',
|
||||
);
|
||||
}
|
||||
const header = await reader.readExactly(headerLength);
|
||||
let command: Readonly<RemoteWorkerArtifactUploadCommand>;
|
||||
try {
|
||||
command = parseRemoteWorkerArtifactUploadHeader(header, {
|
||||
workerId: input.workerId,
|
||||
workerSessionId: input.workerSessionId,
|
||||
});
|
||||
} finally {
|
||||
header.fill(0);
|
||||
}
|
||||
if (input.contentLength !== 4 + headerLength + command.byteLength) {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact upload envelope length does not match its header',
|
||||
);
|
||||
}
|
||||
try {
|
||||
await this.authority.authorizeArtifactUpload(command);
|
||||
const target = storageCommand(command);
|
||||
const content = reader.content(command.byteLength);
|
||||
const receipt = await this.store.put(
|
||||
target,
|
||||
content.chunks,
|
||||
input.signal,
|
||||
);
|
||||
if (!content.isComplete()) {
|
||||
throw new Error('Artifact store did not consume the complete body');
|
||||
}
|
||||
return assertReceiptMatches(target, receipt);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof InvalidRemoteWorkerCompletionError ||
|
||||
error instanceof RemoteWorkerCompletionFenceRejectedError ||
|
||||
error instanceof RemoteWorkerCompletionUnavailableError
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
throw new RemoteWorkerCompletionUnavailableError({ cause: error });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRemoteWorkerCompletionService {
|
||||
private readonly createEventId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly repository: RemoteWorkerCompletionRepository,
|
||||
private readonly store: Pick<ClusterRemoteWorkerArtifactStore, 'inspect'>,
|
||||
options: ClusterRemoteWorkerCompletionServiceOptions = {},
|
||||
) {
|
||||
if (
|
||||
typeof repository?.complete !== 'function' ||
|
||||
typeof store?.inspect !== 'function' ||
|
||||
(options.createEventId !== undefined &&
|
||||
typeof options.createEventId !== 'function')
|
||||
) {
|
||||
throw new TypeError('Remote Worker completion service is invalid');
|
||||
}
|
||||
this.createEventId = options.createEventId ?? randomUUID;
|
||||
}
|
||||
|
||||
async complete(
|
||||
value: RemoteWorkerCompletionCommand,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Readonly<RemoteWorkerCompletionResult>> {
|
||||
const command = normalizeRemoteWorkerCompletionCommand(value);
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
const lookup = Object.freeze({
|
||||
projectId: command.projectId,
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
logArtifactId: command.artifact.logArtifactId,
|
||||
});
|
||||
let stored: Readonly<RemoteWorkerArtifactReceipt> | undefined;
|
||||
try {
|
||||
stored = await this.store.inspect(lookup, signal);
|
||||
} catch (error) {
|
||||
throw new RemoteWorkerCompletionUnavailableError({ cause: error });
|
||||
}
|
||||
if (!stored) {
|
||||
throw new RemoteWorkerCompletionFenceRejectedError(
|
||||
command.attemptId,
|
||||
'state_mismatch',
|
||||
);
|
||||
}
|
||||
const receipt = assertReceiptMatches(
|
||||
{
|
||||
...lookup,
|
||||
byteLength: command.artifact.byteLength,
|
||||
...(command.artifact.truncated === undefined
|
||||
? {}
|
||||
: { truncated: command.artifact.truncated }),
|
||||
},
|
||||
stored,
|
||||
);
|
||||
if (receipt.sha256 !== command.artifact.sha256) {
|
||||
throw new RemoteWorkerCompletionFenceRejectedError(
|
||||
command.attemptId,
|
||||
'replay_mismatch',
|
||||
);
|
||||
}
|
||||
try {
|
||||
const result = normalizeRemoteWorkerCompletionResult(
|
||||
await this.repository.complete(Object.freeze({
|
||||
...command,
|
||||
attemptEventId: eventId(this.createEventId),
|
||||
runEventId: eventId(this.createEventId),
|
||||
})),
|
||||
);
|
||||
if (
|
||||
result.runId !== command.runId ||
|
||||
result.attemptId !== command.attemptId ||
|
||||
result.callbackSequence !== command.callbackSequence
|
||||
) {
|
||||
throw new Error('Remote Worker completion authority drifted');
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof RemoteWorkerCompletionFenceRejectedError ||
|
||||
error instanceof RemoteWorkerCompletionUnavailableError
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
throw new RemoteWorkerCompletionUnavailableError({ cause: error });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
// Remote execution owns bounded offer selection, placement, and lease claiming.
|
||||
import { randomBytes, randomUUID } from 'node:crypto';
|
||||
import type {
|
||||
ClusterDispatchCandidate,
|
||||
ClusterDispatchCandidateCursor,
|
||||
ClusterDispatchSource,
|
||||
ClusterRemoteExecutionOffer,
|
||||
} from '@qinglong/runtime-core/remote-dispatch';
|
||||
import {
|
||||
assertRemoteDispatchPageSize,
|
||||
createClusterRemoteExecutionOffer,
|
||||
evaluateRemoteWorkerPlacement,
|
||||
leaseTokenMatchesDigest,
|
||||
normalizeClusterDispatchCandidate,
|
||||
} from '@qinglong/runtime-core/remote-dispatch';
|
||||
import type {
|
||||
ClusterTaskExecutionRevision,
|
||||
ClusterTaskExecutionRevisionSource,
|
||||
} from '@qinglong/runtime-core/cluster-execution-revision';
|
||||
import type {
|
||||
ClaimRunDispatchLeaseResult,
|
||||
RunDispatchLeaseRepository,
|
||||
WorkerSessionRecord,
|
||||
WorkerSessionRepository,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
assertRunDispatchId,
|
||||
assertRunDispatchLeaseDuration,
|
||||
assertRunDispatchLeaseToken,
|
||||
assertWorkerId,
|
||||
assertWorkerSessionId,
|
||||
} from '@qinglong/runtime-core';
|
||||
import { parseTaskDefinitionRevisionRef } from '@qinglong/runtime-core/task-definition-execution-compiler';
|
||||
|
||||
const DEFAULT_PAGE_SIZE = 8;
|
||||
const DEFAULT_MAX_PAGES = 2;
|
||||
const DEFAULT_MAX_CLAIMS = 8;
|
||||
const DEFAULT_LEASE_MS = 30_000;
|
||||
const MAX_PAGES = 16;
|
||||
const MAX_CLAIMS = 64;
|
||||
|
||||
export interface ClusterRemoteWorkerOfferPrincipal {
|
||||
readonly workerId: string;
|
||||
}
|
||||
|
||||
export interface ClaimClusterRemoteWorkerOfferCommand {
|
||||
readonly workerSessionId: string;
|
||||
readonly workerGeneration: number;
|
||||
/** Worker-generated stable idempotency key for this poll attempt. */
|
||||
readonly offerId: string;
|
||||
/** Worker-generated high-entropy capability; PostgreSQL stores only its digest. */
|
||||
readonly leaseToken: string;
|
||||
}
|
||||
|
||||
export interface ClusterRemoteWorkerOfferStats {
|
||||
readonly pages: number;
|
||||
readonly candidates: number;
|
||||
readonly plansUnavailable: number;
|
||||
readonly placementMismatches: number;
|
||||
readonly claimAttempts: number;
|
||||
readonly claimRaces: number;
|
||||
}
|
||||
|
||||
type MutableClusterRemoteWorkerOfferStats = {
|
||||
-readonly [Key in keyof ClusterRemoteWorkerOfferStats]: ClusterRemoteWorkerOfferStats[Key];
|
||||
};
|
||||
|
||||
export type ClaimClusterRemoteWorkerOfferResult =
|
||||
| Readonly<{
|
||||
status: 'offered';
|
||||
offer: ClusterRemoteExecutionOffer;
|
||||
stats: ClusterRemoteWorkerOfferStats;
|
||||
truncated: boolean;
|
||||
}>
|
||||
| Readonly<{
|
||||
status: 'idle';
|
||||
reason:
|
||||
| 'worker_unavailable'
|
||||
| 'no_candidates'
|
||||
| 'no_match'
|
||||
| 'plans_unavailable'
|
||||
| 'claim_raced'
|
||||
| 'claim_budget_exhausted'
|
||||
| 'scan_budget_exhausted';
|
||||
stats: ClusterRemoteWorkerOfferStats;
|
||||
truncated: boolean;
|
||||
}>;
|
||||
|
||||
export class ClusterRemoteWorkerOfferFenceRejectedError extends Error {
|
||||
readonly code = 'REMOTE_WORKER_OFFER_FENCED';
|
||||
|
||||
constructor() {
|
||||
super('Remote Worker offer authority was fenced');
|
||||
this.name = 'ClusterRemoteWorkerOfferFenceRejectedError';
|
||||
}
|
||||
}
|
||||
|
||||
export interface ClusterRemoteWorkerOfferClaimServiceOptions {
|
||||
readonly pageSize?: number;
|
||||
readonly maxPages?: number;
|
||||
readonly maxClaimAttempts?: number;
|
||||
readonly leaseDurationMs?: number;
|
||||
readonly createEventId?: () => string;
|
||||
}
|
||||
|
||||
function bounded(name: string, value: number, minimum: number, maximum: number): number {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function emptyStats(): MutableClusterRemoteWorkerOfferStats {
|
||||
return {
|
||||
pages: 0,
|
||||
candidates: 0,
|
||||
plansUnavailable: 0,
|
||||
placementMismatches: 0,
|
||||
claimAttempts: 0,
|
||||
claimRaces: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function cursor(candidate: ClusterDispatchCandidate): ClusterDispatchCandidateCursor {
|
||||
return Object.freeze({
|
||||
priority: candidate.priority,
|
||||
queuedAtMs: candidate.queuedAtMs,
|
||||
attemptCreatedAtMs: candidate.attemptCreatedAtMs,
|
||||
attemptId: candidate.attemptId,
|
||||
});
|
||||
}
|
||||
|
||||
export class ClusterRemoteWorkerOfferClaimService {
|
||||
private readonly pageSize: number;
|
||||
private readonly maxPages: number;
|
||||
private readonly maxClaimAttempts: number;
|
||||
private readonly leaseDurationMs: number;
|
||||
private readonly createEventId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly source: ClusterDispatchSource,
|
||||
private readonly workers: Pick<WorkerSessionRepository, 'findById'>,
|
||||
private readonly revisions: ClusterTaskExecutionRevisionSource,
|
||||
private readonly leases: Pick<RunDispatchLeaseRepository, 'claim'>,
|
||||
options: ClusterRemoteWorkerOfferClaimServiceOptions = {},
|
||||
) {
|
||||
if (
|
||||
!source || typeof source.listClusterDispatchCandidates !== 'function' ||
|
||||
typeof source.findClusterDispatchRecovery !== 'function' ||
|
||||
!workers || typeof workers.findById !== 'function' ||
|
||||
!revisions || typeof revisions.resolveClusterTaskExecutionRevision !== 'function' ||
|
||||
!leases || typeof leases.claim !== 'function'
|
||||
) throw new TypeError('Remote Worker offer service dependencies are invalid');
|
||||
const allowed = new Set([
|
||||
'createEventId', 'leaseDurationMs', 'maxClaimAttempts', 'maxPages', 'pageSize',
|
||||
]);
|
||||
if (!options || typeof options !== 'object' || Array.isArray(options) || Object.keys(options).some((key) => !allowed.has(key))) {
|
||||
throw new TypeError('Remote Worker offer service options are invalid');
|
||||
}
|
||||
this.pageSize = options.pageSize ?? DEFAULT_PAGE_SIZE;
|
||||
assertRemoteDispatchPageSize(this.pageSize);
|
||||
this.maxPages = bounded('Remote Worker offer maxPages', options.maxPages ?? DEFAULT_MAX_PAGES, 1, MAX_PAGES);
|
||||
this.maxClaimAttempts = bounded('Remote Worker offer maxClaimAttempts', options.maxClaimAttempts ?? DEFAULT_MAX_CLAIMS, 1, MAX_CLAIMS);
|
||||
this.leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_MS;
|
||||
assertRunDispatchLeaseDuration(this.leaseDurationMs);
|
||||
this.createEventId = options.createEventId ?? randomUUID;
|
||||
if (typeof this.createEventId !== 'function') {
|
||||
throw new TypeError('Remote Worker offer event ID factory is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
async claimNext(
|
||||
principal: ClusterRemoteWorkerOfferPrincipal,
|
||||
command: ClaimClusterRemoteWorkerOfferCommand,
|
||||
): Promise<ClaimClusterRemoteWorkerOfferResult> {
|
||||
this.assertCommand(principal, command);
|
||||
const stats = emptyStats();
|
||||
const recovered = await this.source.findClusterDispatchRecovery(command.offerId);
|
||||
if (recovered) {
|
||||
if (
|
||||
recovered.lease.status !== 'leased' ||
|
||||
recovered.lease.expiresAtMs <= recovered.observedAtMs ||
|
||||
!recovered.workerCurrent ||
|
||||
recovered.lease.workerId !== principal.workerId ||
|
||||
recovered.lease.workerSessionId !== command.workerSessionId ||
|
||||
recovered.lease.workerGeneration !== command.workerGeneration ||
|
||||
!leaseTokenMatchesDigest(command.leaseToken, recovered.lease.leaseTokenDigest)
|
||||
) throw new ClusterRemoteWorkerOfferFenceRejectedError();
|
||||
const revision = await this.resolveRevision(recovered.candidate);
|
||||
if (!revision) throw new ClusterRemoteWorkerOfferFenceRejectedError();
|
||||
return Object.freeze({
|
||||
status: 'offered' as const,
|
||||
offer: this.offer(
|
||||
'lease_recovery', command, recovered.candidate,
|
||||
recovered.lease, revision, 0,
|
||||
),
|
||||
stats: Object.freeze(stats),
|
||||
truncated: false,
|
||||
});
|
||||
}
|
||||
|
||||
let after: ClusterDispatchCandidateCursor | undefined;
|
||||
let worker: WorkerSessionRecord | null | undefined;
|
||||
let sawCandidate = false;
|
||||
let sawMatch = false;
|
||||
let sawRace = false;
|
||||
let lastTruncated = false;
|
||||
for (let pageIndex = 0; pageIndex < this.maxPages; pageIndex += 1) {
|
||||
const page = await this.source.listClusterDispatchCandidates({
|
||||
limit: this.pageSize,
|
||||
...(after === undefined ? {} : { after }),
|
||||
});
|
||||
stats.pages += 1;
|
||||
lastTruncated = page.truncated;
|
||||
if (page.candidates.length > this.pageSize) {
|
||||
throw new RangeError('Remote Worker candidate source exceeded page size');
|
||||
}
|
||||
worker ??= await this.workers.findById(principal.workerId);
|
||||
if (
|
||||
!worker || worker.sessionId !== command.workerSessionId ||
|
||||
worker.generation !== command.workerGeneration ||
|
||||
worker.status !== 'online' || worker.availableSlots < 1 ||
|
||||
worker.leaseExpiresAtMs <= page.observedAtMs
|
||||
) return this.idle('worker_unavailable', stats, false);
|
||||
|
||||
for (const rawCandidate of page.candidates) {
|
||||
const candidate = normalizeClusterDispatchCandidate(rawCandidate);
|
||||
sawCandidate = true;
|
||||
stats.candidates += 1;
|
||||
const revision = await this.resolveRevision(candidate);
|
||||
if (!revision) {
|
||||
stats.plansUnavailable += 1;
|
||||
continue;
|
||||
}
|
||||
const placement = evaluateRemoteWorkerPlacement(
|
||||
worker,
|
||||
revision.placement ?? {},
|
||||
page.observedAtMs,
|
||||
);
|
||||
if (!placement.matches) {
|
||||
stats.placementMismatches += 1;
|
||||
continue;
|
||||
}
|
||||
sawMatch = true;
|
||||
if (stats.claimAttempts >= this.maxClaimAttempts) {
|
||||
return this.idle('claim_budget_exhausted', stats, true);
|
||||
}
|
||||
const eventId = this.createEventId();
|
||||
assertRunDispatchId('eventId', eventId);
|
||||
stats.claimAttempts += 1;
|
||||
const claim = await this.leases.claim({
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
workerId: principal.workerId,
|
||||
workerSessionId: command.workerSessionId,
|
||||
workerGeneration: command.workerGeneration,
|
||||
leaseToken: command.leaseToken,
|
||||
leaseDurationMs: this.leaseDurationMs,
|
||||
eventId,
|
||||
offerId: command.offerId,
|
||||
});
|
||||
if (claim.status === 'claimed' || claim.status === 'idempotent') {
|
||||
return Object.freeze({
|
||||
status: 'offered' as const,
|
||||
offer: this.offer(
|
||||
'new_claim', command, candidate, claim.lease, revision,
|
||||
placement.score,
|
||||
),
|
||||
stats: Object.freeze(stats),
|
||||
truncated: page.truncated,
|
||||
});
|
||||
}
|
||||
if (claim.status === 'worker_unavailable' || claim.status === 'capacity_exhausted') {
|
||||
return this.idle('worker_unavailable', stats, false);
|
||||
}
|
||||
stats.claimRaces += 1;
|
||||
sawRace = true;
|
||||
}
|
||||
if (!page.truncated || page.candidates.length === 0) break;
|
||||
const last = page.candidates.at(-1);
|
||||
if (!last) break;
|
||||
const next = page.next ?? cursor(last);
|
||||
if (after && next.attemptId === after.attemptId) {
|
||||
throw new Error('Remote Worker candidate cursor did not advance');
|
||||
}
|
||||
after = next;
|
||||
}
|
||||
if (lastTruncated) return this.idle('scan_budget_exhausted', stats, true);
|
||||
if (!sawCandidate) return this.idle('no_candidates', stats, false);
|
||||
if (stats.plansUnavailable === stats.candidates) return this.idle('plans_unavailable', stats, false);
|
||||
return this.idle(sawRace ? 'claim_raced' : sawMatch ? 'claim_raced' : 'no_match', stats, false);
|
||||
}
|
||||
|
||||
private async resolveRevision(
|
||||
candidate: ClusterDispatchCandidate,
|
||||
): Promise<ClusterTaskExecutionRevision | null> {
|
||||
let sourceRevision: number;
|
||||
try {
|
||||
sourceRevision = parseTaskDefinitionRevisionRef(candidate.taskRevision).revision;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const revision = await this.revisions.resolveClusterTaskExecutionRevision({
|
||||
projectId: candidate.projectId,
|
||||
taskId: candidate.taskId,
|
||||
sourceRevision,
|
||||
});
|
||||
if (
|
||||
!revision || revision.projectId !== candidate.projectId ||
|
||||
revision.taskId !== candidate.taskId ||
|
||||
revision.taskRevision !== candidate.taskRevision
|
||||
) return null;
|
||||
return revision;
|
||||
}
|
||||
|
||||
private offer(
|
||||
deliveryKind: ClusterRemoteExecutionOffer['deliveryKind'],
|
||||
command: ClaimClusterRemoteWorkerOfferCommand,
|
||||
candidate: ClusterDispatchCandidate,
|
||||
lease: Extract<ClaimRunDispatchLeaseResult, { lease: unknown }>['lease'],
|
||||
revision: ClusterTaskExecutionRevision,
|
||||
placementScore: number,
|
||||
): ClusterRemoteExecutionOffer {
|
||||
return createClusterRemoteExecutionOffer({
|
||||
offerId: command.offerId,
|
||||
deliveryKind,
|
||||
executionDigest: revision.contentDigest,
|
||||
candidate,
|
||||
worker: {
|
||||
workerId: lease.workerId,
|
||||
sessionId: lease.workerSessionId,
|
||||
generation: lease.workerGeneration,
|
||||
},
|
||||
lease,
|
||||
leaseToken: command.leaseToken,
|
||||
executionRevision: revision,
|
||||
placementScore,
|
||||
});
|
||||
}
|
||||
|
||||
private idle(
|
||||
reason: Extract<ClaimClusterRemoteWorkerOfferResult, { status: 'idle' }>['reason'],
|
||||
stats: ClusterRemoteWorkerOfferStats,
|
||||
truncated: boolean,
|
||||
): ClaimClusterRemoteWorkerOfferResult {
|
||||
return Object.freeze({
|
||||
status: 'idle' as const,
|
||||
reason,
|
||||
stats: Object.freeze({ ...stats }),
|
||||
truncated,
|
||||
});
|
||||
}
|
||||
|
||||
private assertCommand(
|
||||
principal: ClusterRemoteWorkerOfferPrincipal,
|
||||
command: ClaimClusterRemoteWorkerOfferCommand,
|
||||
): void {
|
||||
if (!principal || typeof principal !== 'object' || Array.isArray(principal)) {
|
||||
throw new TypeError('Remote Worker offer principal is invalid');
|
||||
}
|
||||
assertWorkerId(principal.workerId);
|
||||
if (!command || typeof command !== 'object' || Array.isArray(command)) {
|
||||
throw new TypeError('Remote Worker offer command is invalid');
|
||||
}
|
||||
const keys = Object.keys(command).sort().join(',');
|
||||
if (keys !== 'leaseToken,offerId,workerGeneration,workerSessionId') {
|
||||
throw new TypeError('Remote Worker offer command shape is invalid');
|
||||
}
|
||||
assertWorkerSessionId(command.workerSessionId);
|
||||
if (!Number.isSafeInteger(command.workerGeneration) || command.workerGeneration < 1) {
|
||||
throw new RangeError('Remote Worker offer generation is invalid');
|
||||
}
|
||||
assertRunDispatchId('offerId', command.offerId);
|
||||
assertRunDispatchLeaseToken(command.leaseToken);
|
||||
}
|
||||
}
|
||||
|
||||
export function createRemoteWorkerLeaseToken(): string {
|
||||
return randomBytes(32).toString('base64url');
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
// Remote execution owns fenced Worker lease renewal, release, and timeout authority.
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
RemoteWorkerLeaseControlUnavailableError,
|
||||
assertRemoteWorkerLeaseControlDuration,
|
||||
normalizeRemoteWorkerLeaseControlCommand,
|
||||
normalizeRemoteWorkerLeaseControlResult,
|
||||
type RemoteWorkerLeaseControlCommand,
|
||||
type RemoteWorkerLeaseControlRepository,
|
||||
type RemoteWorkerLeaseControlResult,
|
||||
} from '@qinglong/runtime-core/remote-worker-lease-control';
|
||||
|
||||
export interface ClusterRemoteWorkerLeaseControlServiceOptions {
|
||||
readonly leaseDurationMs?: number;
|
||||
readonly createEventId?: () => string;
|
||||
}
|
||||
|
||||
function eventId(factory: () => string): string {
|
||||
const value = factory();
|
||||
if (
|
||||
typeof value !== 'string' || value.length < 1 || value.length > 36 ||
|
||||
/[\u0000-\u001f\u007f]/.test(value)
|
||||
) throw new RemoteWorkerLeaseControlUnavailableError();
|
||||
return value;
|
||||
}
|
||||
|
||||
export class ClusterRemoteWorkerLeaseControlService {
|
||||
private readonly leaseDurationMs: number;
|
||||
private readonly createEventId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly repository: RemoteWorkerLeaseControlRepository,
|
||||
options: ClusterRemoteWorkerLeaseControlServiceOptions = {},
|
||||
) {
|
||||
if (
|
||||
typeof repository?.control !== 'function' ||
|
||||
(options.createEventId !== undefined &&
|
||||
typeof options.createEventId !== 'function')
|
||||
) throw new TypeError('Remote Worker lease control service is invalid');
|
||||
const leaseDurationMs = options.leaseDurationMs ?? 30_000;
|
||||
assertRemoteWorkerLeaseControlDuration(leaseDurationMs);
|
||||
this.leaseDurationMs = leaseDurationMs;
|
||||
this.createEventId = options.createEventId ?? randomUUID;
|
||||
}
|
||||
|
||||
async control(
|
||||
value: RemoteWorkerLeaseControlCommand,
|
||||
): Promise<Readonly<RemoteWorkerLeaseControlResult>> {
|
||||
const command = normalizeRemoteWorkerLeaseControlCommand(value);
|
||||
return normalizeRemoteWorkerLeaseControlResult(
|
||||
await this.repository.control(Object.freeze({
|
||||
...command,
|
||||
leaseDurationMs: this.leaseDurationMs,
|
||||
timeoutEventId: eventId(this.createEventId),
|
||||
})),
|
||||
);
|
||||
}
|
||||
}
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
// Remote execution owns offer-bound Secret delivery without retaining plaintext authority.
|
||||
import {
|
||||
InvalidRemoteWorkerSecretDeliveryError,
|
||||
RemoteWorkerSecretDeliveryFenceRejectedError,
|
||||
RemoteWorkerSecretDeliveryUnavailableError,
|
||||
createRemoteWorkerSecretDeliveryResponseBody,
|
||||
normalizeRemoteWorkerSecretDeliveryAuthority,
|
||||
normalizeRemoteWorkerSecretDeliveryCommand,
|
||||
type RemoteWorkerSecretDeliveryAuthorityRepository,
|
||||
type RemoteWorkerSecretDeliveryCommand,
|
||||
type RemoteWorkerSecretDeliveryResult,
|
||||
type RemoteWorkerSecretValueProvider,
|
||||
} from '@qinglong/runtime-core/remote-secret-delivery';
|
||||
|
||||
export interface ClusterRemoteWorkerSecretDeliveryPrincipal {
|
||||
readonly workerId: string;
|
||||
}
|
||||
|
||||
export type ClusterRemoteWorkerSecretDeliveryCommand = Omit<
|
||||
RemoteWorkerSecretDeliveryCommand,
|
||||
'workerId'
|
||||
>;
|
||||
|
||||
export class ClusterRemoteWorkerSecretDeliveryService {
|
||||
constructor(
|
||||
private readonly authority: RemoteWorkerSecretDeliveryAuthorityRepository,
|
||||
private readonly secrets: RemoteWorkerSecretValueProvider,
|
||||
) {
|
||||
if (
|
||||
!authority ||
|
||||
typeof authority.authorize !== 'function' ||
|
||||
!secrets ||
|
||||
typeof secrets.resolve !== 'function'
|
||||
) throw new TypeError('Remote Worker Secret delivery service is invalid');
|
||||
}
|
||||
|
||||
async deliver(
|
||||
principal: ClusterRemoteWorkerSecretDeliveryPrincipal,
|
||||
input: ClusterRemoteWorkerSecretDeliveryCommand,
|
||||
): Promise<Readonly<RemoteWorkerSecretDeliveryResult>> {
|
||||
if (
|
||||
!principal ||
|
||||
typeof principal !== 'object' ||
|
||||
Array.isArray(principal) ||
|
||||
!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(principal.workerId)
|
||||
) throw new TypeError('Remote Worker Secret delivery principal is invalid');
|
||||
const command = normalizeRemoteWorkerSecretDeliveryCommand({
|
||||
...input,
|
||||
workerId: principal.workerId,
|
||||
});
|
||||
let authorized;
|
||||
try {
|
||||
authorized = normalizeRemoteWorkerSecretDeliveryAuthority(
|
||||
await this.authority.authorize(command),
|
||||
);
|
||||
if (
|
||||
authorized.workerId !== command.workerId ||
|
||||
authorized.workerSessionId !== command.workerSessionId ||
|
||||
authorized.workerGeneration !== command.workerGeneration ||
|
||||
authorized.runId !== command.runId ||
|
||||
authorized.attemptId !== command.attemptId ||
|
||||
authorized.projectId !== command.projectId ||
|
||||
authorized.taskId !== command.taskId ||
|
||||
authorized.taskRevision !== command.taskRevision ||
|
||||
authorized.executionDigest !== command.executionDigest ||
|
||||
authorized.offerId !== command.offerId ||
|
||||
authorized.leaseGeneration !== command.leaseGeneration ||
|
||||
authorized.leaseVersion !== command.expectedLeaseVersion ||
|
||||
JSON.stringify(authorized.secretRefs) !== JSON.stringify(command.secretRefs)
|
||||
) throw new InvalidRemoteWorkerSecretDeliveryError(
|
||||
'repository authority does not match command',
|
||||
);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof RemoteWorkerSecretDeliveryFenceRejectedError ||
|
||||
error instanceof RemoteWorkerSecretDeliveryUnavailableError
|
||||
) throw error;
|
||||
throw new RemoteWorkerSecretDeliveryUnavailableError();
|
||||
}
|
||||
let resolution;
|
||||
try {
|
||||
resolution = await this.secrets.resolve(authorized);
|
||||
} catch {
|
||||
throw new RemoteWorkerSecretDeliveryUnavailableError();
|
||||
}
|
||||
if (!resolution) throw new RemoteWorkerSecretDeliveryUnavailableError();
|
||||
try {
|
||||
if (
|
||||
typeof resolution !== 'object' ||
|
||||
Array.isArray(resolution) ||
|
||||
Object.keys(resolution).some((key) => key !== 'values' && key !== 'dispose') ||
|
||||
(resolution.dispose !== undefined &&
|
||||
typeof resolution.dispose !== 'function')
|
||||
) throw new InvalidRemoteWorkerSecretDeliveryError(
|
||||
'provider response shape is invalid',
|
||||
);
|
||||
const body = createRemoteWorkerSecretDeliveryResponseBody({
|
||||
runId: authorized.runId,
|
||||
attemptId: authorized.attemptId,
|
||||
offerId: authorized.offerId,
|
||||
executionDigest: authorized.executionDigest,
|
||||
values: resolution.values,
|
||||
}, authorized.secretRefs);
|
||||
return Object.freeze({
|
||||
runId: body.runId,
|
||||
attemptId: body.attemptId,
|
||||
offerId: body.offerId,
|
||||
executionDigest: body.executionDigest,
|
||||
values: body.values,
|
||||
...(resolution.dispose === undefined
|
||||
? {}
|
||||
: { dispose: resolution.dispose }),
|
||||
});
|
||||
} catch (error) {
|
||||
try { await resolution.dispose?.(); } catch { /* preserve root */ }
|
||||
if (error instanceof InvalidRemoteWorkerSecretDeliveryError) {
|
||||
throw new RemoteWorkerSecretDeliveryUnavailableError();
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
// Remote execution owns the least-privilege assembly of Worker-facing runtime capabilities.
|
||||
import type { PostgresPool } from '@qinglong/runtime-core';
|
||||
import type { RemoteWorkerSecretValueProvider } from '@qinglong/runtime-core/remote-secret-delivery';
|
||||
import {
|
||||
PostgresClusterDispatchSource,
|
||||
PostgresRemoteRunActivationRepository,
|
||||
PostgresRemoteWorkerCompletionRepository,
|
||||
PostgresRemoteWorkerLeaseControlRepository,
|
||||
PostgresRemoteWorkerSecretDeliveryAuthorityRepository,
|
||||
PostgresRunDispatchLeaseRepository,
|
||||
PostgresTaskExecutionRevisionSource,
|
||||
PostgresWorkerSessionRepository,
|
||||
} from '@qinglong/cluster-postgres/runtime';
|
||||
import {
|
||||
ClusterRemoteWorkerOfferClaimService,
|
||||
} from './remoteWorkerDispatcher';
|
||||
import {
|
||||
ClusterRemoteRunActivationService,
|
||||
} from './remoteRunActivationService';
|
||||
import {
|
||||
ClusterRemoteWorkerSecretDeliveryService,
|
||||
} from './remoteWorkerSecretDeliveryService';
|
||||
import {
|
||||
ClusterRemoteWorkerArtifactService,
|
||||
ClusterRemoteWorkerCompletionService,
|
||||
type ClusterRemoteWorkerArtifactStore,
|
||||
} from './remoteWorkerCompletionService';
|
||||
import {
|
||||
ClusterRemoteWorkerLeaseControlService,
|
||||
} from './remoteWorkerLeaseControlService';
|
||||
import type { WorkerIngressPipelineOptions } from '../worker-ingress/workerIngressPipeline';
|
||||
|
||||
export interface ClusterWorkerRuntimeDependencies {
|
||||
readonly artifactStore: ClusterRemoteWorkerArtifactStore;
|
||||
readonly secretProvider?: RemoteWorkerSecretValueProvider;
|
||||
}
|
||||
|
||||
/**
|
||||
* The in-process capability boundary from the runtime authority to the
|
||||
* Worker-facing transport. It exposes reviewed operations, never the runtime
|
||||
* Pool or mutation repositories.
|
||||
*/
|
||||
export interface ClusterWorkerRuntimePort {
|
||||
readonly offers: NonNullable<WorkerIngressPipelineOptions['offers']>;
|
||||
readonly activation: NonNullable<WorkerIngressPipelineOptions['activation']>;
|
||||
readonly secrets?: NonNullable<WorkerIngressPipelineOptions['secrets']>;
|
||||
readonly artifacts: NonNullable<WorkerIngressPipelineOptions['artifacts']>;
|
||||
readonly completion: NonNullable<WorkerIngressPipelineOptions['completion']>;
|
||||
readonly leaseControl: NonNullable<
|
||||
WorkerIngressPipelineOptions['leaseControl']
|
||||
>;
|
||||
}
|
||||
|
||||
export function createClusterWorkerRuntimePort(
|
||||
pool: PostgresPool,
|
||||
dependencies: ClusterWorkerRuntimeDependencies,
|
||||
): Readonly<ClusterWorkerRuntimePort> {
|
||||
if (!pool || typeof pool.query !== 'function') {
|
||||
throw new TypeError('Cluster Worker runtime Pool is invalid');
|
||||
}
|
||||
if (
|
||||
!dependencies ||
|
||||
typeof dependencies !== 'object' ||
|
||||
Array.isArray(dependencies)
|
||||
) {
|
||||
throw new TypeError('Cluster Worker runtime dependencies are invalid');
|
||||
}
|
||||
|
||||
const workerSessions = new PostgresWorkerSessionRepository(pool);
|
||||
const completionRepository =
|
||||
new PostgresRemoteWorkerCompletionRepository(pool);
|
||||
const secretProvider = dependencies.secretProvider;
|
||||
return Object.freeze({
|
||||
offers: new ClusterRemoteWorkerOfferClaimService(
|
||||
new PostgresClusterDispatchSource(pool),
|
||||
workerSessions,
|
||||
new PostgresTaskExecutionRevisionSource(pool),
|
||||
new PostgresRunDispatchLeaseRepository(pool),
|
||||
),
|
||||
activation: new ClusterRemoteRunActivationService(
|
||||
new PostgresRemoteRunActivationRepository(pool),
|
||||
),
|
||||
...(secretProvider === undefined
|
||||
? {}
|
||||
: {
|
||||
secrets: new ClusterRemoteWorkerSecretDeliveryService(
|
||||
new PostgresRemoteWorkerSecretDeliveryAuthorityRepository(pool),
|
||||
secretProvider,
|
||||
),
|
||||
}),
|
||||
artifacts: new ClusterRemoteWorkerArtifactService(
|
||||
completionRepository,
|
||||
dependencies.artifactStore,
|
||||
),
|
||||
completion: new ClusterRemoteWorkerCompletionService(
|
||||
completionRepository,
|
||||
dependencies.artifactStore,
|
||||
),
|
||||
leaseControl: new ClusterRemoteWorkerLeaseControlService(
|
||||
new PostgresRemoteWorkerLeaseControlRepository(pool),
|
||||
),
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user