mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 18:08:20 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,247 @@
|
||||
import type {
|
||||
LocalCompletionReceiptJournal,
|
||||
LocalCompletionReceiptJournalCursor,
|
||||
} from '@qinglong/runtime-core/local-completion-receipt-journal';
|
||||
import {
|
||||
MAX_LOCAL_COMPLETION_RECEIPT_JOURNAL_PAGE,
|
||||
assertLocalCompletionReceiptJournalLimit,
|
||||
} from '@qinglong/runtime-core/local-completion-receipt-journal';
|
||||
import type { CompletionReceiptStore } from './completionReceiptFileStore';
|
||||
|
||||
const TERMINAL_ATTEMPT_STATUSES = new Set([
|
||||
'succeeded',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'timed_out',
|
||||
'lost',
|
||||
]);
|
||||
const MAX_RETENTION_MS = 24 * 60 * 60_000;
|
||||
|
||||
interface MaintainedCompletionReceiptStore extends CompletionReceiptStore {
|
||||
quarantine?(attemptId: string): Promise<string | undefined>;
|
||||
purgeQuarantine?(attemptId: string): Promise<boolean>;
|
||||
}
|
||||
|
||||
export interface LocalCompletionReceiptCleanupSummary {
|
||||
readonly scanned: number;
|
||||
readonly removed: number;
|
||||
readonly expiredMissing: number;
|
||||
readonly purgedQuarantines: number;
|
||||
readonly remaining: number;
|
||||
readonly failed: number;
|
||||
readonly truncated: boolean;
|
||||
readonly nextCursor?: LocalCompletionReceiptJournalCursor;
|
||||
}
|
||||
|
||||
export interface LocalCompletionReceiptCleanupOptions {
|
||||
readonly terminalMissingRetentionMs?: number;
|
||||
readonly clock?: { now(): number };
|
||||
}
|
||||
|
||||
export class LocalCompletionReceiptCleanupScanner {
|
||||
private readonly terminalMissingRetentionMs: number;
|
||||
private readonly clock: { now(): number };
|
||||
|
||||
constructor(
|
||||
private readonly journal: LocalCompletionReceiptJournal,
|
||||
private readonly receipts: MaintainedCompletionReceiptStore,
|
||||
options: LocalCompletionReceiptCleanupOptions = {},
|
||||
) {
|
||||
this.terminalMissingRetentionMs =
|
||||
options.terminalMissingRetentionMs ?? 60_000;
|
||||
if (
|
||||
!Number.isSafeInteger(this.terminalMissingRetentionMs) ||
|
||||
this.terminalMissingRetentionMs < 0 ||
|
||||
this.terminalMissingRetentionMs > MAX_RETENTION_MS
|
||||
) {
|
||||
throw new RangeError(
|
||||
'terminalMissingRetentionMs must be between 0 and 24 hours',
|
||||
);
|
||||
}
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
}
|
||||
|
||||
async scan(
|
||||
options: {
|
||||
readonly cursor?: LocalCompletionReceiptJournalCursor;
|
||||
readonly limit?: number;
|
||||
} = {},
|
||||
): Promise<LocalCompletionReceiptCleanupSummary> {
|
||||
const observedAtMs = this.clock.now();
|
||||
if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0) {
|
||||
throw new RangeError('Completion receipt cleanup clock is invalid');
|
||||
}
|
||||
const limit = options.limit ?? 32;
|
||||
assertLocalCompletionReceiptJournalLimit(limit);
|
||||
const page = await this.journal.listCandidates({
|
||||
observedAtMs,
|
||||
limit,
|
||||
...(options.cursor === undefined ? {} : { cursor: options.cursor }),
|
||||
});
|
||||
if (
|
||||
!page ||
|
||||
!Array.isArray(page.candidates) ||
|
||||
page.candidates.length > MAX_LOCAL_COMPLETION_RECEIPT_JOURNAL_PAGE ||
|
||||
typeof page.truncated !== 'boolean'
|
||||
) {
|
||||
throw new TypeError('Completion receipt cleanup page is invalid');
|
||||
}
|
||||
let removed = 0;
|
||||
let expiredMissing = 0;
|
||||
let purgedQuarantines = 0;
|
||||
let remaining = 0;
|
||||
let failed = 0;
|
||||
for (const candidate of page.candidates) {
|
||||
try {
|
||||
if (candidate.executorType !== 'local_process') {
|
||||
remaining += 1;
|
||||
continue;
|
||||
}
|
||||
if (candidate.state === 'quarantined') {
|
||||
if (!this.receipts.quarantine || !this.receipts.purgeQuarantine) {
|
||||
remaining += 1;
|
||||
continue;
|
||||
}
|
||||
await this.receipts.quarantine(candidate.attemptId);
|
||||
await this.receipts.purgeQuarantine(candidate.attemptId);
|
||||
await this.journal.resolve(candidate.attemptId);
|
||||
purgedQuarantines += 1;
|
||||
continue;
|
||||
}
|
||||
if (!TERMINAL_ATTEMPT_STATUSES.has(candidate.attemptStatus)) {
|
||||
remaining += 1;
|
||||
continue;
|
||||
}
|
||||
const receiptRemoved = await this.receipts.remove(candidate.attemptId);
|
||||
if (receiptRemoved) {
|
||||
await this.journal.resolve(candidate.attemptId);
|
||||
removed += 1;
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
candidate.finishedAtMs !== undefined &&
|
||||
candidate.finishedAtMs + this.terminalMissingRetentionMs <=
|
||||
observedAtMs
|
||||
) {
|
||||
await this.journal.resolve(candidate.attemptId);
|
||||
expiredMissing += 1;
|
||||
} else {
|
||||
remaining += 1;
|
||||
}
|
||||
} catch {
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
return Object.freeze({
|
||||
scanned: page.candidates.length,
|
||||
removed,
|
||||
expiredMissing,
|
||||
purgedQuarantines,
|
||||
remaining,
|
||||
failed,
|
||||
truncated: page.truncated,
|
||||
...(page.nextCursor === undefined ? {} : { nextCursor: page.nextCursor }),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export interface LocalCompletionReceiptCleanupLifecycleOptions {
|
||||
readonly intervalMs: number;
|
||||
readonly pageSize: number;
|
||||
readonly stopTimeoutMs?: number;
|
||||
readonly onDiagnostic?: (
|
||||
error: unknown,
|
||||
summary?: LocalCompletionReceiptCleanupSummary,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export class LocalCompletionReceiptCleanupLifecycle {
|
||||
private readonly stopTimeoutMs: number;
|
||||
private timer: NodeJS.Timeout | undefined;
|
||||
private inFlight: Promise<void> | undefined;
|
||||
private running = false;
|
||||
private cursor: LocalCompletionReceiptJournalCursor | undefined;
|
||||
|
||||
constructor(
|
||||
private readonly scanner: LocalCompletionReceiptCleanupScanner,
|
||||
private readonly options: LocalCompletionReceiptCleanupLifecycleOptions,
|
||||
) {
|
||||
if (
|
||||
!Number.isSafeInteger(options.intervalMs) ||
|
||||
options.intervalMs < 1_000 ||
|
||||
options.intervalMs > 24 * 60 * 60_000
|
||||
) {
|
||||
throw new RangeError(
|
||||
'cleanup interval must be between 1 second and 24 hours',
|
||||
);
|
||||
}
|
||||
assertLocalCompletionReceiptJournalLimit(options.pageSize);
|
||||
this.stopTimeoutMs = options.stopTimeoutMs ?? 5_000;
|
||||
if (
|
||||
!Number.isSafeInteger(this.stopTimeoutMs) ||
|
||||
this.stopTimeoutMs < 100 ||
|
||||
this.stopTimeoutMs > 30_000
|
||||
) {
|
||||
throw new RangeError(
|
||||
'cleanup stop timeout must be between 100 and 30000 ms',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
start(): void {
|
||||
if (this.running) return;
|
||||
this.running = true;
|
||||
this.schedule();
|
||||
}
|
||||
|
||||
async runOnce(): Promise<LocalCompletionReceiptCleanupSummary> {
|
||||
const summary = await this.scanner.scan({
|
||||
limit: this.options.pageSize,
|
||||
...(this.cursor === undefined ? {} : { cursor: this.cursor }),
|
||||
});
|
||||
this.cursor = summary.truncated ? summary.nextCursor : undefined;
|
||||
return summary;
|
||||
}
|
||||
|
||||
async stop(): Promise<'stopped' | 'timed_out'> {
|
||||
this.running = false;
|
||||
if (this.timer) clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
const inFlight = this.inFlight;
|
||||
if (!inFlight) return 'stopped';
|
||||
return Promise.race([
|
||||
inFlight.then(() => 'stopped' as const),
|
||||
new Promise<'timed_out'>((resolve) => {
|
||||
setTimeout(() => resolve('timed_out'), this.stopTimeoutMs).unref?.();
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
private schedule(): void {
|
||||
if (!this.running) return;
|
||||
this.timer = setTimeout(() => {
|
||||
if (!this.running || this.inFlight) {
|
||||
this.schedule();
|
||||
return;
|
||||
}
|
||||
this.inFlight = this.tick().finally(() => {
|
||||
this.inFlight = undefined;
|
||||
this.schedule();
|
||||
});
|
||||
}, this.options.intervalMs);
|
||||
this.timer.unref?.();
|
||||
}
|
||||
|
||||
private async tick(): Promise<void> {
|
||||
try {
|
||||
const summary = await this.scanner.scan({
|
||||
limit: this.options.pageSize,
|
||||
...(this.cursor === undefined ? {} : { cursor: this.cursor }),
|
||||
});
|
||||
this.cursor = summary.truncated ? summary.nextCursor : undefined;
|
||||
await this.options.onDiagnostic?.(undefined, summary);
|
||||
} catch (error) {
|
||||
await this.options.onDiagnostic?.(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
export const COMPLETION_RECEIPT_SCHEMA_VERSION = 1;
|
||||
export const MAX_COMPLETION_RECEIPT_BYTES = 4 * 1024;
|
||||
|
||||
const RECEIPT_KEYS = [
|
||||
'schemaVersion',
|
||||
'runId',
|
||||
'attemptId',
|
||||
'callbackSequence',
|
||||
'token',
|
||||
'startedAtMs',
|
||||
'finishedAtMs',
|
||||
'exitCode',
|
||||
] as const;
|
||||
const RECEIPT_KEY_SET = new Set<string>(RECEIPT_KEYS);
|
||||
const PORTABLE_LOCAL_EXECUTION_ID =
|
||||
/^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
|
||||
const TOKEN_PATTERN = /^[A-Za-z0-9_-]{32,128}$/;
|
||||
|
||||
export interface CompletionReceipt {
|
||||
schemaVersion: typeof COMPLETION_RECEIPT_SCHEMA_VERSION;
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
callbackSequence: number;
|
||||
token: string;
|
||||
startedAtMs: number;
|
||||
finishedAtMs: number;
|
||||
exitCode: number;
|
||||
}
|
||||
|
||||
export class InvalidCompletionReceiptError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'InvalidCompletionReceiptError';
|
||||
}
|
||||
}
|
||||
|
||||
function invalid(message: string): never {
|
||||
throw new InvalidCompletionReceiptError(message);
|
||||
}
|
||||
|
||||
function skipWhitespace(value: string, from: number): number {
|
||||
let index = from;
|
||||
while (/\s/.test(value[index] ?? '')) index += 1;
|
||||
return index;
|
||||
}
|
||||
|
||||
function stringEnd(value: string, from: number): number {
|
||||
if (value[from] !== '"') invalid('Completion receipt key is not a string');
|
||||
for (let index = from + 1; index < value.length; index += 1) {
|
||||
if (value[index] === '\\') {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (value[index] === '"') return index + 1;
|
||||
}
|
||||
return invalid('Completion receipt contains an unterminated string');
|
||||
}
|
||||
|
||||
/** JSON.parse silently accepts duplicate keys, so reject them before parsing. */
|
||||
function assertUniqueFlatObjectKeys(value: string): void {
|
||||
let index = skipWhitespace(value, 0);
|
||||
if (value[index] !== '{') invalid('Completion receipt must be a JSON object');
|
||||
index = skipWhitespace(value, index + 1);
|
||||
const keys = new Set<string>();
|
||||
if (value[index] === '}') return;
|
||||
|
||||
while (index < value.length) {
|
||||
const keyStart = index;
|
||||
const keyEnd = stringEnd(value, keyStart);
|
||||
let key: string;
|
||||
try {
|
||||
key = JSON.parse(value.slice(keyStart, keyEnd));
|
||||
} catch {
|
||||
return invalid('Completion receipt contains an invalid key');
|
||||
}
|
||||
if (keys.has(key)) invalid('Completion receipt contains a duplicate key');
|
||||
keys.add(key);
|
||||
|
||||
index = skipWhitespace(value, keyEnd);
|
||||
if (value[index] !== ':') invalid('Completion receipt key has no value');
|
||||
index = skipWhitespace(value, index + 1);
|
||||
if (value[index] === '"') {
|
||||
index = stringEnd(value, index);
|
||||
} else {
|
||||
if (value[index] === '{' || value[index] === '[') {
|
||||
invalid('Completion receipt values must be scalar');
|
||||
}
|
||||
const valueStart = index;
|
||||
while (
|
||||
index < value.length &&
|
||||
value[index] !== ',' &&
|
||||
value[index] !== '}'
|
||||
) {
|
||||
index += 1;
|
||||
}
|
||||
if (value.slice(valueStart, index).trim().length === 0) {
|
||||
invalid('Completion receipt contains an empty value');
|
||||
}
|
||||
}
|
||||
|
||||
index = skipWhitespace(value, index);
|
||||
if (value[index] === '}') return;
|
||||
if (value[index] !== ',')
|
||||
invalid('Completion receipt is not a flat object');
|
||||
index = skipWhitespace(value, index + 1);
|
||||
}
|
||||
invalid('Completion receipt object is incomplete');
|
||||
}
|
||||
|
||||
export function assertCompletionReceiptId(
|
||||
value: string,
|
||||
field: 'runId' | 'attemptId',
|
||||
): void {
|
||||
if (!PORTABLE_LOCAL_EXECUTION_ID.test(value)) {
|
||||
invalid(`${field} must be a bounded portable execution ID`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertSafeTimestamp(
|
||||
value: unknown,
|
||||
field: string,
|
||||
): asserts value is number {
|
||||
if (!Number.isSafeInteger(value) || (value as number) < 0) {
|
||||
invalid(`${field} must be a non-negative safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
export function validateCompletionReceipt(
|
||||
candidate: unknown,
|
||||
): CompletionReceipt {
|
||||
if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) {
|
||||
invalid('Completion receipt must be an object');
|
||||
}
|
||||
const record = candidate as Record<string, unknown>;
|
||||
const keys = Object.keys(record);
|
||||
if (
|
||||
keys.length !== RECEIPT_KEYS.length ||
|
||||
keys.some((key) => !RECEIPT_KEY_SET.has(key)) ||
|
||||
RECEIPT_KEYS.some((key) => !Object.hasOwn(record, key))
|
||||
) {
|
||||
invalid('Completion receipt fields do not match schema version 1');
|
||||
}
|
||||
if (record.schemaVersion !== COMPLETION_RECEIPT_SCHEMA_VERSION) {
|
||||
invalid('Completion receipt schema version is unsupported');
|
||||
}
|
||||
if (typeof record.runId !== 'string') invalid('runId must be a string');
|
||||
if (typeof record.attemptId !== 'string') {
|
||||
invalid('attemptId must be a string');
|
||||
}
|
||||
assertCompletionReceiptId(record.runId, 'runId');
|
||||
assertCompletionReceiptId(record.attemptId, 'attemptId');
|
||||
if (
|
||||
!Number.isSafeInteger(record.callbackSequence) ||
|
||||
(record.callbackSequence as number) < 1
|
||||
) {
|
||||
invalid('callbackSequence must be a positive safe integer');
|
||||
}
|
||||
if (typeof record.token !== 'string' || !TOKEN_PATTERN.test(record.token)) {
|
||||
invalid('token must be a bounded base64url value');
|
||||
}
|
||||
assertSafeTimestamp(record.startedAtMs, 'startedAtMs');
|
||||
assertSafeTimestamp(record.finishedAtMs, 'finishedAtMs');
|
||||
if (record.finishedAtMs < record.startedAtMs) {
|
||||
invalid('finishedAtMs must not be before startedAtMs');
|
||||
}
|
||||
if (
|
||||
!Number.isInteger(record.exitCode) ||
|
||||
(record.exitCode as number) < 0 ||
|
||||
(record.exitCode as number) > 255
|
||||
) {
|
||||
invalid('exitCode must be an integer between 0 and 255');
|
||||
}
|
||||
return {
|
||||
schemaVersion: COMPLETION_RECEIPT_SCHEMA_VERSION,
|
||||
runId: record.runId,
|
||||
attemptId: record.attemptId,
|
||||
callbackSequence: record.callbackSequence as number,
|
||||
token: record.token,
|
||||
startedAtMs: record.startedAtMs,
|
||||
finishedAtMs: record.finishedAtMs,
|
||||
exitCode: record.exitCode as number,
|
||||
};
|
||||
}
|
||||
|
||||
export function serializeCompletionReceipt(receipt: CompletionReceipt): string {
|
||||
const value = validateCompletionReceipt(receipt);
|
||||
const serialized = JSON.stringify({
|
||||
schemaVersion: value.schemaVersion,
|
||||
runId: value.runId,
|
||||
attemptId: value.attemptId,
|
||||
callbackSequence: value.callbackSequence,
|
||||
token: value.token,
|
||||
startedAtMs: value.startedAtMs,
|
||||
finishedAtMs: value.finishedAtMs,
|
||||
exitCode: value.exitCode,
|
||||
});
|
||||
if (Buffer.byteLength(serialized, 'utf8') > MAX_COMPLETION_RECEIPT_BYTES) {
|
||||
invalid('Completion receipt exceeds the byte limit');
|
||||
}
|
||||
return serialized;
|
||||
}
|
||||
|
||||
export function parseCompletionReceipt(
|
||||
input: string | Uint8Array,
|
||||
): CompletionReceipt {
|
||||
const bytes =
|
||||
typeof input === 'string' ? Buffer.from(input) : Buffer.from(input);
|
||||
if (bytes.length === 0 || bytes.length > MAX_COMPLETION_RECEIPT_BYTES) {
|
||||
invalid('Completion receipt size is outside the allowed range');
|
||||
}
|
||||
const value = bytes.toString('utf8');
|
||||
if (!Buffer.from(value, 'utf8').equals(bytes)) {
|
||||
invalid('Completion receipt must be valid UTF-8');
|
||||
}
|
||||
assertUniqueFlatObjectKeys(value);
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(value);
|
||||
} catch {
|
||||
return invalid('Completion receipt is not valid JSON');
|
||||
}
|
||||
return validateCompletionReceipt(parsed);
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
import { constants } from 'fs';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { randomBytes } from 'crypto';
|
||||
import {
|
||||
assertCompletionReceiptId,
|
||||
InvalidCompletionReceiptError,
|
||||
MAX_COMPLETION_RECEIPT_BYTES,
|
||||
parseCompletionReceipt,
|
||||
serializeCompletionReceipt,
|
||||
type CompletionReceipt,
|
||||
} from './completionReceipt';
|
||||
|
||||
export class CompletionReceiptAlreadyExistsError extends Error {
|
||||
constructor(readonly attemptId: string) {
|
||||
super(`Completion receipt already exists for Attempt ${attemptId}`);
|
||||
this.name = 'CompletionReceiptAlreadyExistsError';
|
||||
}
|
||||
}
|
||||
|
||||
function isCode(error: unknown, code: string): boolean {
|
||||
return (
|
||||
typeof error === 'object' &&
|
||||
error !== null &&
|
||||
'code' in error &&
|
||||
(error as { code?: string }).code === code
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A bounded local journal. It does not scan directories or own a lifecycle;
|
||||
* callers must discover active Attempts from the database.
|
||||
*/
|
||||
export interface CompletionReceiptStore {
|
||||
publish(receipt: CompletionReceipt): Promise<void>;
|
||||
read(attemptId: string): Promise<CompletionReceipt | undefined>;
|
||||
remove(attemptId: string): Promise<boolean>;
|
||||
}
|
||||
|
||||
export class CompletionReceiptFileStore implements CompletionReceiptStore {
|
||||
constructor(private readonly root: string) {
|
||||
if (
|
||||
!path.isAbsolute(root) ||
|
||||
path.parse(root).root === root ||
|
||||
root.includes('\0') ||
|
||||
Buffer.byteLength(root, 'utf8') > 4096
|
||||
) {
|
||||
throw new RangeError(
|
||||
'Completion receipt root must be a bounded non-root absolute path',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async publish(receipt: CompletionReceipt): Promise<void> {
|
||||
const serialized = serializeCompletionReceipt(receipt);
|
||||
const directory = this.directory(receipt.attemptId);
|
||||
const target = this.target(receipt.attemptId);
|
||||
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
const temporary = path.join(
|
||||
directory,
|
||||
`.${receipt.attemptId}.${randomBytes(16).toString('hex')}.tmp`,
|
||||
);
|
||||
let handle: fs.FileHandle | undefined;
|
||||
try {
|
||||
handle = await fs.open(temporary, 'wx', 0o600);
|
||||
await handle.writeFile(serialized, 'utf8');
|
||||
await handle.sync();
|
||||
await handle.close();
|
||||
handle = undefined;
|
||||
|
||||
// A hard link publishes the fully written inode atomically and, unlike
|
||||
// plain rename(), never replaces an existing completion fact.
|
||||
try {
|
||||
await fs.link(temporary, target);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'EEXIST')) {
|
||||
throw new CompletionReceiptAlreadyExistsError(receipt.attemptId);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
await this.bestEffortUnlink(temporary);
|
||||
await this.bestEffortSyncDirectory(directory);
|
||||
} catch (error) {
|
||||
await handle?.close().catch(() => undefined);
|
||||
await this.bestEffortUnlink(temporary);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async read(attemptId: string): Promise<CompletionReceipt | undefined> {
|
||||
const target = this.target(attemptId);
|
||||
let handle: fs.FileHandle;
|
||||
try {
|
||||
handle = await fs.open(
|
||||
target,
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return undefined;
|
||||
if (isCode(error, 'ELOOP')) {
|
||||
throw new InvalidCompletionReceiptError(
|
||||
'Completion receipt must not be a symbolic link',
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
const stat = await handle.stat();
|
||||
if (!stat.isFile()) {
|
||||
throw new InvalidCompletionReceiptError(
|
||||
'Completion receipt must be a regular file',
|
||||
);
|
||||
}
|
||||
const bytes = Buffer.allocUnsafe(MAX_COMPLETION_RECEIPT_BYTES + 1);
|
||||
const { bytesRead } = await handle.read(bytes, 0, bytes.length, 0);
|
||||
if (bytesRead > MAX_COMPLETION_RECEIPT_BYTES) {
|
||||
throw new InvalidCompletionReceiptError(
|
||||
'Completion receipt exceeds the byte limit',
|
||||
);
|
||||
}
|
||||
const receipt = parseCompletionReceipt(bytes.subarray(0, bytesRead));
|
||||
if (receipt.attemptId !== attemptId) {
|
||||
throw new InvalidCompletionReceiptError(
|
||||
'Completion receipt path and Attempt do not match',
|
||||
);
|
||||
}
|
||||
return receipt;
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
async remove(attemptId: string): Promise<boolean> {
|
||||
const target = this.target(attemptId);
|
||||
try {
|
||||
await fs.unlink(target);
|
||||
await this.bestEffortSyncDirectory(path.dirname(target));
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async quarantine(attemptId: string): Promise<string | undefined> {
|
||||
const target = this.target(attemptId);
|
||||
const reference = this.quarantineReference(attemptId);
|
||||
const relativeDirectory = path.dirname(reference);
|
||||
const directory = path.join(this.root, relativeDirectory);
|
||||
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
await fs.chmod(directory, 0o700);
|
||||
const quarantined = path.join(this.root, reference);
|
||||
try {
|
||||
await fs.link(target, quarantined);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) {
|
||||
return (await this.pathExists(quarantined)) ? reference : undefined;
|
||||
}
|
||||
if (!isCode(error, 'EEXIST')) throw error;
|
||||
}
|
||||
await this.unlinkIfPresent(target);
|
||||
await this.bestEffortSyncDirectory(path.dirname(target));
|
||||
await this.bestEffortSyncDirectory(directory);
|
||||
return reference;
|
||||
}
|
||||
|
||||
quarantineReference(attemptId: string): string {
|
||||
assertCompletionReceiptId(attemptId, 'attemptId');
|
||||
return path.posix.join(
|
||||
'.quarantine',
|
||||
attemptId.slice(0, 2),
|
||||
`${attemptId}.json`,
|
||||
);
|
||||
}
|
||||
|
||||
async purgeQuarantine(attemptId: string): Promise<boolean> {
|
||||
const target = path.join(this.root, this.quarantineReference(attemptId));
|
||||
try {
|
||||
await fs.unlink(target);
|
||||
await this.bestEffortSyncDirectory(path.dirname(target));
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private directory(attemptId: string): string {
|
||||
assertCompletionReceiptId(attemptId, 'attemptId');
|
||||
return path.join(this.root, attemptId.slice(0, 2));
|
||||
}
|
||||
|
||||
private target(attemptId: string): string {
|
||||
return path.join(this.directory(attemptId), `${attemptId}.json`);
|
||||
}
|
||||
|
||||
private async bestEffortUnlink(value: string): Promise<void> {
|
||||
try {
|
||||
await fs.unlink(value);
|
||||
} catch (error) {
|
||||
if (!isCode(error, 'ENOENT')) {
|
||||
// Temp cleanup is diagnostic-only; the immutable final fact wins.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async unlinkIfPresent(value: string): Promise<void> {
|
||||
try {
|
||||
await fs.unlink(value);
|
||||
} catch (error) {
|
||||
if (!isCode(error, 'ENOENT')) throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async pathExists(value: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.lstat(value);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async bestEffortSyncDirectory(directory: string): Promise<void> {
|
||||
try {
|
||||
const handle = await fs.open(directory, constants.O_RDONLY);
|
||||
try {
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
} catch {
|
||||
// Some supported filesystems cannot fsync directories. The receipt
|
||||
// remains restart-safe, while power-loss durability is best effort.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export * from './completion-receipt/completionReceipt';
|
||||
export * from './completion-receipt/completionReceiptFileStore';
|
||||
export * from './completion-receipt/cleanup';
|
||||
export * from './process-execution/evidence';
|
||||
export * from './process-execution/localProcessIdentity';
|
||||
export * from './process-execution/launcher';
|
||||
export * from './process-execution/controller';
|
||||
@@ -0,0 +1,194 @@
|
||||
import type { LocalPersistedExecutionInspection } from './evidence';
|
||||
import {
|
||||
LinuxProcProcessIdentityProvider,
|
||||
parseLocalProcessDurableHandle,
|
||||
type LinuxProcessIdentity,
|
||||
type LocalProcessIdentityProvider,
|
||||
} from './localProcessIdentity';
|
||||
|
||||
export const MAX_LOCAL_PROCESS_STOP_GRACE_MS = 30_000;
|
||||
|
||||
export type LocalProcessStopResult = Readonly<
|
||||
| { status: 'stopped'; signal: 'SIGTERM' | 'SIGKILL' }
|
||||
| { status: 'already_exited' }
|
||||
| {
|
||||
status: 'unknown';
|
||||
reason:
|
||||
| 'invalid_handle'
|
||||
| 'unsupported_platform'
|
||||
| 'provider_unavailable'
|
||||
| 'signal_failed';
|
||||
}
|
||||
| { status: 'timed_out' }
|
||||
>;
|
||||
|
||||
export interface LocalProcessControllerOptions {
|
||||
readonly identityProvider?: Pick<LocalProcessIdentityProvider, 'inspect'>;
|
||||
readonly terminateGraceMs?: number;
|
||||
readonly killGraceMs?: number;
|
||||
readonly pollIntervalMs?: number;
|
||||
readonly clock?: { now(): number };
|
||||
readonly wait?: (delayMs: number) => Promise<void>;
|
||||
readonly signalProcessGroup?: (
|
||||
processGroupId: number,
|
||||
signal: 'SIGTERM' | 'SIGKILL',
|
||||
) => void;
|
||||
}
|
||||
|
||||
function assertDuration(value: number, field: string): void {
|
||||
if (
|
||||
!Number.isSafeInteger(value) ||
|
||||
value < 1 ||
|
||||
value > MAX_LOCAL_PROCESS_STOP_GRACE_MS
|
||||
) {
|
||||
throw new RangeError(
|
||||
`${field} must be between 1 and ${MAX_LOCAL_PROCESS_STOP_GRACE_MS}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function unknown(
|
||||
reason: Extract<LocalProcessStopResult, { status: 'unknown' }>['reason'],
|
||||
): LocalProcessStopResult {
|
||||
return Object.freeze({ status: 'unknown' as const, reason });
|
||||
}
|
||||
|
||||
function inspectionResult(
|
||||
inspection: LocalPersistedExecutionInspection,
|
||||
): LocalProcessStopResult | undefined {
|
||||
if (inspection.status === 'not_running') {
|
||||
return Object.freeze({ status: 'already_exited' as const });
|
||||
}
|
||||
if (inspection.status === 'unknown') return unknown(inspection.reason);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export class LocalProcessController {
|
||||
private readonly identityProvider: Pick<
|
||||
LocalProcessIdentityProvider,
|
||||
'inspect'
|
||||
>;
|
||||
private readonly terminateGraceMs: number;
|
||||
private readonly killGraceMs: number;
|
||||
private readonly pollIntervalMs: number;
|
||||
private readonly clock: { now(): number };
|
||||
private readonly wait: (delayMs: number) => Promise<void>;
|
||||
private readonly signalProcessGroup: (
|
||||
processGroupId: number,
|
||||
signal: 'SIGTERM' | 'SIGKILL',
|
||||
) => void;
|
||||
|
||||
constructor(options: LocalProcessControllerOptions = {}) {
|
||||
this.identityProvider =
|
||||
options.identityProvider ?? new LinuxProcProcessIdentityProvider();
|
||||
this.terminateGraceMs = options.terminateGraceMs ?? 1_000;
|
||||
this.killGraceMs = options.killGraceMs ?? 1_000;
|
||||
this.pollIntervalMs = options.pollIntervalMs ?? 25;
|
||||
assertDuration(this.terminateGraceMs, 'terminateGraceMs');
|
||||
assertDuration(this.killGraceMs, 'killGraceMs');
|
||||
assertDuration(this.pollIntervalMs, 'pollIntervalMs');
|
||||
if (this.pollIntervalMs > this.terminateGraceMs) {
|
||||
throw new RangeError('pollIntervalMs cannot exceed terminateGraceMs');
|
||||
}
|
||||
if (this.pollIntervalMs > this.killGraceMs) {
|
||||
throw new RangeError('pollIntervalMs cannot exceed killGraceMs');
|
||||
}
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.wait =
|
||||
options.wait ??
|
||||
((delayMs) => new Promise((resolve) => setTimeout(resolve, delayMs)));
|
||||
this.signalProcessGroup =
|
||||
options.signalProcessGroup ??
|
||||
((processGroupId, signal) => process.kill(-processGroupId, signal));
|
||||
}
|
||||
|
||||
async stop(durableHandle: string): Promise<LocalProcessStopResult> {
|
||||
const parsed = parseLocalProcessDurableHandle(durableHandle);
|
||||
if (!parsed) return unknown('invalid_handle');
|
||||
|
||||
try {
|
||||
return await this.stopExact(parsed.identity);
|
||||
} catch {
|
||||
return unknown('provider_unavailable');
|
||||
}
|
||||
}
|
||||
|
||||
private async stopExact(
|
||||
identity: LinuxProcessIdentity,
|
||||
): Promise<LocalProcessStopResult> {
|
||||
const before = await this.identityProvider.inspect(identity);
|
||||
const conclusive = inspectionResult(before);
|
||||
if (conclusive) return conclusive;
|
||||
|
||||
const terminated = await this.signalAndObserve(
|
||||
identity,
|
||||
'SIGTERM',
|
||||
this.terminateGraceMs,
|
||||
);
|
||||
if (terminated !== undefined) return terminated;
|
||||
|
||||
// Revalidate the full boot/PID/start-time/process-group identity before
|
||||
// escalating. A recycled PID can never inherit stop authority.
|
||||
const beforeKill = await this.identityProvider.inspect(identity);
|
||||
const killConclusion = inspectionResult(beforeKill);
|
||||
if (killConclusion) {
|
||||
return killConclusion.status === 'already_exited'
|
||||
? Object.freeze({
|
||||
status: 'stopped' as const,
|
||||
signal: 'SIGTERM' as const,
|
||||
})
|
||||
: killConclusion;
|
||||
}
|
||||
const killed = await this.signalAndObserve(
|
||||
identity,
|
||||
'SIGKILL',
|
||||
this.killGraceMs,
|
||||
);
|
||||
return killed ?? Object.freeze({ status: 'timed_out' as const });
|
||||
}
|
||||
|
||||
private async signalAndObserve(
|
||||
identity: LinuxProcessIdentity,
|
||||
signal: 'SIGTERM' | 'SIGKILL',
|
||||
graceMs: number,
|
||||
): Promise<LocalProcessStopResult | undefined> {
|
||||
try {
|
||||
this.signalProcessGroup(identity.processGroupId, signal);
|
||||
} catch (error) {
|
||||
if (!this.isNoSuchProcess(error)) return unknown('signal_failed');
|
||||
}
|
||||
const deadline = this.safeDeadline(this.clock.now(), graceMs);
|
||||
while (true) {
|
||||
const inspection = await this.identityProvider.inspect(identity);
|
||||
if (inspection.status === 'not_running') {
|
||||
return Object.freeze({ status: 'stopped' as const, signal });
|
||||
}
|
||||
if (inspection.status === 'unknown') return unknown(inspection.reason);
|
||||
const now = this.clock.now();
|
||||
if (!Number.isSafeInteger(now) || now < 0) {
|
||||
return unknown('provider_unavailable');
|
||||
}
|
||||
if (now >= deadline) return undefined;
|
||||
await this.wait(Math.min(this.pollIntervalMs, deadline - now));
|
||||
}
|
||||
}
|
||||
|
||||
private safeDeadline(now: number, graceMs: number): number {
|
||||
if (!Number.isSafeInteger(now) || now < 0) {
|
||||
throw new RangeError('Local process controller clock is invalid');
|
||||
}
|
||||
const deadline = now + graceMs;
|
||||
if (!Number.isSafeInteger(deadline)) {
|
||||
throw new RangeError('Local process controller deadline overflowed');
|
||||
}
|
||||
return deadline;
|
||||
}
|
||||
|
||||
private isNoSuchProcess(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
'code' in error &&
|
||||
(error as NodeJS.ErrnoException).code === 'ESRCH'
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
export type LocalPersistedExecutionInspection = Readonly<
|
||||
| { status: 'running'; identityPid: number }
|
||||
| { status: 'not_running'; identityPid: number }
|
||||
| {
|
||||
status: 'unknown';
|
||||
reason:
|
||||
| 'invalid_handle'
|
||||
| 'unsupported_platform'
|
||||
| 'provider_unavailable';
|
||||
}
|
||||
>;
|
||||
|
||||
export interface LocalPersistedExecutionInspector {
|
||||
readonly executorType: 'local_process';
|
||||
inspect(durableHandle: string): Promise<LocalPersistedExecutionInspection>;
|
||||
}
|
||||
@@ -0,0 +1,500 @@
|
||||
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
||||
import { constants } from 'node:fs';
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { spawn, type ChildProcess } from 'node:child_process';
|
||||
import type { LocalCompletionReceiptJournal } from '@qinglong/runtime-core/local-completion-receipt-journal';
|
||||
import { assertCompletionReceiptId } from '../completion-receipt/completionReceipt';
|
||||
import {
|
||||
createLocalProcessDurableHandle,
|
||||
LinuxProcProcessIdentityProvider,
|
||||
type LocalProcessIdentityProvider,
|
||||
} from './localProcessIdentity';
|
||||
|
||||
export const BUNDLED_LOCAL_PROCESS_LAUNCHER_SHA256 =
|
||||
'db4342ea57f8f7f19e385e204889ac03e97a2f82f42b01f2e59291be4b569153';
|
||||
export const MAX_LOCAL_PROCESS_ENVIRONMENT_ENTRIES = 256;
|
||||
export const MAX_LOCAL_PROCESS_ENVIRONMENT_BYTES = 64 * 1024;
|
||||
export const MAX_LOCAL_PROCESS_ARGUMENTS = 256;
|
||||
export const MAX_LOCAL_PROCESS_COMMAND_BYTES = 64 * 1024;
|
||||
|
||||
const CALLBACK_TOKEN_PATTERN = /^[A-Za-z0-9_-]{32,128}$/;
|
||||
const ENVIRONMENT_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/;
|
||||
|
||||
export type LocalProcessCommand =
|
||||
| Readonly<{
|
||||
kind: 'argv';
|
||||
file: string;
|
||||
args: readonly string[];
|
||||
}>
|
||||
| Readonly<{
|
||||
kind: 'shell';
|
||||
command: string;
|
||||
shell?: '/bin/sh' | '/bin/bash';
|
||||
}>;
|
||||
|
||||
export interface LocalProcessLaunchRequest {
|
||||
readonly runId: string;
|
||||
readonly attemptId: string;
|
||||
/** Optional durable pre-spawn timestamp supplied by a higher-level journal. */
|
||||
readonly startedAtMs?: number;
|
||||
readonly callbackSequence: number;
|
||||
readonly callbackToken: string;
|
||||
readonly command: LocalProcessCommand;
|
||||
readonly environment?: Readonly<Record<string, string>>;
|
||||
readonly workingDirectory?: string;
|
||||
readonly output?: LocalProcessOutputPlan;
|
||||
}
|
||||
|
||||
export interface LocalProcessOutputPlan {
|
||||
readonly filePath: string;
|
||||
readonly maximumBytes: number;
|
||||
readonly logArtifactId: string;
|
||||
}
|
||||
|
||||
export interface LocalProcessLaunchCompletion {
|
||||
readonly exitCode: number | null;
|
||||
readonly signal: NodeJS.Signals | null;
|
||||
}
|
||||
|
||||
export interface LocalProcessLaunchHandle {
|
||||
readonly handleId: string;
|
||||
readonly pid: number;
|
||||
readonly durableHandle: string;
|
||||
readonly startedAtMs: number;
|
||||
readonly completion: Promise<LocalProcessLaunchCompletion>;
|
||||
}
|
||||
|
||||
export interface LocalProcessLauncherOptions {
|
||||
readonly receiptRoot: string;
|
||||
readonly launcherPath?: string;
|
||||
readonly expectedLauncherSha256?: string;
|
||||
readonly identityProvider?: LocalProcessIdentityProvider;
|
||||
readonly clock?: { now(): number };
|
||||
readonly createHandleId?: () => string;
|
||||
}
|
||||
|
||||
export class LocalProcessLaunchError extends Error {
|
||||
readonly code = 'LOCAL_PROCESS_LAUNCH_FAILED';
|
||||
|
||||
constructor(
|
||||
message: string,
|
||||
readonly cause?: unknown,
|
||||
readonly spawnOutcome: 'no_spawn' | 'unknown' = 'no_spawn',
|
||||
) {
|
||||
super(`Local process launch failed: ${message}`);
|
||||
this.name = 'LocalProcessLaunchError';
|
||||
}
|
||||
}
|
||||
|
||||
function assertBoundedString(
|
||||
value: string,
|
||||
field: string,
|
||||
limit: number,
|
||||
): void {
|
||||
if (
|
||||
value.length === 0 ||
|
||||
value.includes('\0') ||
|
||||
Buffer.byteLength(value, 'utf8') > limit
|
||||
) {
|
||||
throw new LocalProcessLaunchError(`${field} is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertAbsoluteBoundedPath(value: string, field: string): void {
|
||||
if (
|
||||
!path.isAbsolute(value) ||
|
||||
path.parse(value).root === value ||
|
||||
value.includes('\0') ||
|
||||
Buffer.byteLength(value, 'utf8') > 4096
|
||||
) {
|
||||
throw new LocalProcessLaunchError(
|
||||
`${field} must be a bounded absolute path`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function assertRequest(request: LocalProcessLaunchRequest): void {
|
||||
assertCompletionReceiptId(request.runId, 'runId');
|
||||
assertCompletionReceiptId(request.attemptId, 'attemptId');
|
||||
if (
|
||||
request.startedAtMs !== undefined &&
|
||||
(!Number.isSafeInteger(request.startedAtMs) || request.startedAtMs < 0)
|
||||
) {
|
||||
throw new LocalProcessLaunchError('started timestamp is invalid');
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(request.callbackSequence) ||
|
||||
request.callbackSequence < 1
|
||||
) {
|
||||
throw new LocalProcessLaunchError('callback sequence is invalid');
|
||||
}
|
||||
if (!CALLBACK_TOKEN_PATTERN.test(request.callbackToken)) {
|
||||
throw new LocalProcessLaunchError('callback token is invalid');
|
||||
}
|
||||
if (request.workingDirectory !== undefined) {
|
||||
assertAbsoluteBoundedPath(request.workingDirectory, 'working directory');
|
||||
}
|
||||
if (request.output !== undefined) {
|
||||
if (
|
||||
!request.output ||
|
||||
typeof request.output !== 'object' ||
|
||||
Array.isArray(request.output)
|
||||
) {
|
||||
throw new LocalProcessLaunchError('output plan is invalid');
|
||||
}
|
||||
assertAbsoluteBoundedPath(request.output.filePath, 'output file');
|
||||
if (
|
||||
!Number.isSafeInteger(request.output.maximumBytes) ||
|
||||
request.output.maximumBytes < 64 * 1024 ||
|
||||
request.output.maximumBytes > 1024 * 1024 * 1024
|
||||
) {
|
||||
throw new LocalProcessLaunchError('output quota is invalid');
|
||||
}
|
||||
if (!/^(?:local|wlog)-[0-9a-f]{30}$/.test(request.output.logArtifactId)) {
|
||||
throw new LocalProcessLaunchError('output Artifact identity is invalid');
|
||||
}
|
||||
if (
|
||||
path.basename(request.output.filePath) !==
|
||||
`${request.output.logArtifactId}.log`
|
||||
) {
|
||||
throw new LocalProcessLaunchError(
|
||||
'output path does not match its Artifact identity',
|
||||
);
|
||||
}
|
||||
}
|
||||
if (request.command.kind === 'argv') {
|
||||
assertAbsoluteBoundedPath(request.command.file, 'command file');
|
||||
if (request.command.args.length > MAX_LOCAL_PROCESS_ARGUMENTS) {
|
||||
throw new LocalProcessLaunchError('command has too many arguments');
|
||||
}
|
||||
let bytes = Buffer.byteLength(request.command.file, 'utf8');
|
||||
for (const argument of request.command.args) {
|
||||
assertBoundedString(argument, 'command argument', 16 * 1024);
|
||||
bytes += Buffer.byteLength(argument, 'utf8');
|
||||
}
|
||||
if (bytes > MAX_LOCAL_PROCESS_COMMAND_BYTES) {
|
||||
throw new LocalProcessLaunchError('command exceeds its byte budget');
|
||||
}
|
||||
} else {
|
||||
assertBoundedString(
|
||||
request.command.command,
|
||||
'shell command',
|
||||
MAX_LOCAL_PROCESS_COMMAND_BYTES,
|
||||
);
|
||||
if (
|
||||
request.command.shell !== undefined &&
|
||||
request.command.shell !== '/bin/sh' &&
|
||||
request.command.shell !== '/bin/bash'
|
||||
) {
|
||||
throw new LocalProcessLaunchError('shell is not allowlisted');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function normalizedEnvironment(
|
||||
source: Readonly<Record<string, string>> | undefined,
|
||||
): NodeJS.ProcessEnv {
|
||||
const entries = Object.entries(source ?? {});
|
||||
if (entries.length > MAX_LOCAL_PROCESS_ENVIRONMENT_ENTRIES) {
|
||||
throw new LocalProcessLaunchError('environment entry budget exceeded');
|
||||
}
|
||||
let bytes = 0;
|
||||
const environment: NodeJS.ProcessEnv = {};
|
||||
for (const [name, value] of entries) {
|
||||
if (!ENVIRONMENT_NAME_PATTERN.test(name)) {
|
||||
throw new LocalProcessLaunchError('environment name is invalid');
|
||||
}
|
||||
assertBoundedString(value, 'environment value', 16 * 1024);
|
||||
bytes += Buffer.byteLength(name, 'utf8') + Buffer.byteLength(value, 'utf8');
|
||||
if (bytes > MAX_LOCAL_PROCESS_ENVIRONMENT_BYTES) {
|
||||
throw new LocalProcessLaunchError('environment byte budget exceeded');
|
||||
}
|
||||
if (!name.startsWith('QL3_RECEIPT_') && !name.startsWith('QL3_LAUNCH_')) {
|
||||
environment[name] = value;
|
||||
}
|
||||
}
|
||||
return environment;
|
||||
}
|
||||
|
||||
function bundledLauncherPath(): string {
|
||||
return path.resolve(__dirname, '../../assets/ql3-launcher.sh');
|
||||
}
|
||||
|
||||
function completionOf(
|
||||
child: ChildProcess,
|
||||
): Promise<LocalProcessLaunchCompletion> {
|
||||
return new Promise((resolve) => {
|
||||
child.once('close', (exitCode, signal) => {
|
||||
resolve(Object.freeze({ exitCode, signal }));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export class LocalProcessLauncher {
|
||||
private readonly receiptRoot: string;
|
||||
private readonly launcherPath: string;
|
||||
private readonly expectedLauncherSha256: string;
|
||||
private readonly identityProvider: LocalProcessIdentityProvider;
|
||||
private readonly clock: { now(): number };
|
||||
private readonly createHandleId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly journal: Pick<LocalCompletionReceiptJournal, 'register'>,
|
||||
options: LocalProcessLauncherOptions,
|
||||
) {
|
||||
assertAbsoluteBoundedPath(options.receiptRoot, 'receipt root');
|
||||
this.receiptRoot = path.resolve(options.receiptRoot);
|
||||
this.launcherPath = options.launcherPath ?? bundledLauncherPath();
|
||||
assertAbsoluteBoundedPath(this.launcherPath, 'launcher path');
|
||||
this.expectedLauncherSha256 =
|
||||
options.expectedLauncherSha256 ?? BUNDLED_LOCAL_PROCESS_LAUNCHER_SHA256;
|
||||
if (!/^[a-f0-9]{64}$/.test(this.expectedLauncherSha256)) {
|
||||
throw new LocalProcessLaunchError('launcher digest is invalid');
|
||||
}
|
||||
this.identityProvider =
|
||||
options.identityProvider ?? new LinuxProcProcessIdentityProvider();
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.createHandleId = options.createHandleId ?? randomUUID;
|
||||
}
|
||||
|
||||
async start(
|
||||
request: LocalProcessLaunchRequest,
|
||||
): Promise<LocalProcessLaunchHandle> {
|
||||
assertRequest(request);
|
||||
const environment = normalizedEnvironment(request.environment);
|
||||
const observedAtMs = this.clock.now();
|
||||
if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0) {
|
||||
throw new LocalProcessLaunchError('clock returned an invalid timestamp');
|
||||
}
|
||||
const startedAtMs = request.startedAtMs ?? observedAtMs;
|
||||
if (startedAtMs > observedAtMs) {
|
||||
throw new LocalProcessLaunchError(
|
||||
'started timestamp cannot be after launch observation',
|
||||
);
|
||||
}
|
||||
const handleId = this.createHandleId();
|
||||
assertBoundedString(handleId, 'handle id', 255);
|
||||
const verifiedLauncher = await this.verifyLauncher();
|
||||
|
||||
const receiptDirectory = path.join(
|
||||
this.receiptRoot,
|
||||
request.attemptId.slice(0, 2),
|
||||
);
|
||||
const receiptTarget = path.join(
|
||||
receiptDirectory,
|
||||
`${request.attemptId}.json`,
|
||||
);
|
||||
const receiptTemporary = path.join(
|
||||
receiptDirectory,
|
||||
`.${request.attemptId}.${randomBytes(16).toString('hex')}.tmp`,
|
||||
);
|
||||
let output: fs.FileHandle | undefined;
|
||||
let spawnConfirmed = false;
|
||||
try {
|
||||
await fs.mkdir(receiptDirectory, { recursive: true, mode: 0o700 });
|
||||
await fs.chmod(receiptDirectory, 0o700);
|
||||
let outputQuotaFifo: string | undefined;
|
||||
let outputQuotaRemainingBytes: number | undefined;
|
||||
let outputTruncationTarget: string | undefined;
|
||||
let outputTruncationTemporary: string | undefined;
|
||||
if (request.output !== undefined) {
|
||||
await fs.mkdir(path.dirname(request.output.filePath), {
|
||||
recursive: true,
|
||||
mode: 0o700,
|
||||
});
|
||||
output = await fs.open(
|
||||
request.output.filePath,
|
||||
constants.O_WRONLY |
|
||||
constants.O_CREAT |
|
||||
constants.O_APPEND |
|
||||
(constants.O_NOFOLLOW ?? 0),
|
||||
0o600,
|
||||
);
|
||||
const stat = await output.stat();
|
||||
if (!stat.isFile()) {
|
||||
throw new LocalProcessLaunchError(
|
||||
'output target is not a regular file',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(stat.size) ||
|
||||
stat.size < 0 ||
|
||||
stat.size > request.output.maximumBytes
|
||||
) {
|
||||
throw new LocalProcessLaunchError('output target exceeds its quota');
|
||||
}
|
||||
await output.chmod(0o600);
|
||||
const directory = path.dirname(request.output.filePath);
|
||||
const base = path.basename(request.output.filePath);
|
||||
outputQuotaFifo = path.join(directory, `.${base}.fifo`);
|
||||
outputQuotaRemainingBytes = request.output.maximumBytes - stat.size;
|
||||
outputTruncationTarget = path.join(
|
||||
directory,
|
||||
`.${base}.truncated.json`,
|
||||
);
|
||||
outputTruncationTemporary = path.join(
|
||||
directory,
|
||||
`.${base}.truncated.tmp`,
|
||||
);
|
||||
}
|
||||
|
||||
// This is the durable pre-spawn barrier. A crash after this point leaves
|
||||
// a database-indexed cleanup candidate and never requires directory scan.
|
||||
await this.journal.register({
|
||||
runId: request.runId,
|
||||
attemptId: request.attemptId,
|
||||
registeredAtMs: startedAtMs,
|
||||
});
|
||||
|
||||
const launcherEnvironment: NodeJS.ProcessEnv = {
|
||||
...environment,
|
||||
QL3_RECEIPT_RUN_ID: request.runId,
|
||||
QL3_RECEIPT_ATTEMPT_ID: request.attemptId,
|
||||
QL3_RECEIPT_CALLBACK_SEQUENCE: String(request.callbackSequence),
|
||||
QL3_RECEIPT_CALLBACK_TOKEN: request.callbackToken,
|
||||
QL3_RECEIPT_STARTED_AT_MS: String(startedAtMs),
|
||||
QL3_RECEIPT_TARGET: receiptTarget,
|
||||
QL3_RECEIPT_TEMPORARY: receiptTemporary,
|
||||
...(request.output === undefined
|
||||
? {}
|
||||
: {
|
||||
QL3_OUTPUT_QUOTA_FIFO: outputQuotaFifo!,
|
||||
QL3_OUTPUT_QUOTA_REMAINING_BYTES: String(
|
||||
outputQuotaRemainingBytes,
|
||||
),
|
||||
QL3_OUTPUT_ARTIFACT_ID: request.output.logArtifactId,
|
||||
QL3_OUTPUT_MAXIMUM_BYTES: String(request.output.maximumBytes),
|
||||
QL3_OUTPUT_TRUNCATION_TARGET: outputTruncationTarget!,
|
||||
QL3_OUTPUT_TRUNCATION_TEMPORARY: outputTruncationTemporary!,
|
||||
}),
|
||||
...(request.command.kind === 'shell'
|
||||
? {
|
||||
QL3_LAUNCH_SHELL: request.command.shell ?? '/bin/sh',
|
||||
QL3_LAUNCH_SHELL_COMMAND: request.command.command,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
const args =
|
||||
request.command.kind === 'argv'
|
||||
? ['/dev/fd/3', 'argv', request.command.file, ...request.command.args]
|
||||
: ['/dev/fd/3', 'shell'];
|
||||
const child = spawn('/bin/sh', args, {
|
||||
cwd: request.workingDirectory,
|
||||
env: launcherEnvironment,
|
||||
detached: true,
|
||||
stdio: output
|
||||
? ['ignore', output.fd, output.fd, verifiedLauncher.fd]
|
||||
: ['ignore', 'ignore', 'ignore', verifiedLauncher.fd],
|
||||
});
|
||||
const completion = completionOf(child);
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
child.once('spawn', resolve);
|
||||
child.once('error', reject);
|
||||
});
|
||||
spawnConfirmed = true;
|
||||
} finally {
|
||||
await output?.close().catch(() => undefined);
|
||||
output = undefined;
|
||||
await verifiedLauncher.close().catch(() => undefined);
|
||||
}
|
||||
if (!child.pid) {
|
||||
throw new LocalProcessLaunchError(
|
||||
'spawn did not return a PID',
|
||||
undefined,
|
||||
'unknown',
|
||||
);
|
||||
}
|
||||
let identity;
|
||||
try {
|
||||
identity = await this.identityProvider.capture(child.pid);
|
||||
} catch (error) {
|
||||
await this.stopUnownedProcess(child.pid, completion);
|
||||
throw new LocalProcessLaunchError(
|
||||
'durable identity capture failed',
|
||||
error,
|
||||
'unknown',
|
||||
);
|
||||
}
|
||||
if (!identity) {
|
||||
await this.stopUnownedProcess(child.pid, completion);
|
||||
throw new LocalProcessLaunchError(
|
||||
'platform cannot prove durable local process identity',
|
||||
undefined,
|
||||
'unknown',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
handleId,
|
||||
pid: child.pid,
|
||||
durableHandle: createLocalProcessDurableHandle(handleId, identity),
|
||||
startedAtMs,
|
||||
completion,
|
||||
});
|
||||
} catch (error) {
|
||||
await output?.close().catch(() => undefined);
|
||||
await verifiedLauncher.close().catch(() => undefined);
|
||||
if (error instanceof LocalProcessLaunchError) throw error;
|
||||
throw new LocalProcessLaunchError(
|
||||
'launcher preparation or spawn failed',
|
||||
error,
|
||||
spawnConfirmed ? 'unknown' : 'no_spawn',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async verifyLauncher(): Promise<fs.FileHandle> {
|
||||
let launcher: fs.FileHandle | undefined;
|
||||
try {
|
||||
launcher = await fs.open(
|
||||
this.launcherPath,
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
const stat = await launcher.stat();
|
||||
if (!stat.isFile() || stat.size < 1 || stat.size > 64 * 1024) {
|
||||
throw new LocalProcessLaunchError(
|
||||
'launcher is not a bounded regular file',
|
||||
);
|
||||
}
|
||||
const bytes = Buffer.allocUnsafe(stat.size);
|
||||
const { bytesRead } = await launcher.read(bytes, 0, bytes.length, 0);
|
||||
if (bytesRead !== bytes.length) {
|
||||
throw new LocalProcessLaunchError('launcher changed while being read');
|
||||
}
|
||||
const digest = createHash('sha256').update(bytes).digest('hex');
|
||||
if (digest !== this.expectedLauncherSha256) {
|
||||
throw new LocalProcessLaunchError(
|
||||
'launcher digest does not match review',
|
||||
);
|
||||
}
|
||||
return launcher;
|
||||
} catch (error) {
|
||||
await launcher?.close().catch(() => undefined);
|
||||
if (error instanceof LocalProcessLaunchError) throw error;
|
||||
throw new LocalProcessLaunchError('launcher verification failed', error);
|
||||
}
|
||||
}
|
||||
|
||||
private async stopUnownedProcess(
|
||||
pid: number,
|
||||
completion: Promise<LocalProcessLaunchCompletion>,
|
||||
): Promise<void> {
|
||||
try {
|
||||
process.kill(-pid, 'SIGTERM');
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const closed = await Promise.race([
|
||||
completion.then(() => true),
|
||||
new Promise<false>((resolve) => setTimeout(() => resolve(false), 1_000)),
|
||||
]);
|
||||
if (!closed) {
|
||||
try {
|
||||
process.kill(-pid, 'SIGKILL');
|
||||
} catch {
|
||||
// The process may have exited between the timeout and the signal.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
import { readFile } from 'fs/promises';
|
||||
import type {
|
||||
LocalPersistedExecutionInspection,
|
||||
LocalPersistedExecutionInspector,
|
||||
} from './evidence';
|
||||
|
||||
export const LOCAL_PROCESS_DURABLE_HANDLE_PREFIX = 'ql3lp1.';
|
||||
export const MAX_LOCAL_PROCESS_DURABLE_HANDLE_BYTES = 512;
|
||||
|
||||
const LINUX_BOOT_ID_PATH = '/proc/sys/kernel/random/boot_id';
|
||||
|
||||
export interface LinuxProcessIdentity {
|
||||
platform: 'linux';
|
||||
bootId: string;
|
||||
pid: number;
|
||||
processGroupId: number;
|
||||
startTimeTicks: string;
|
||||
}
|
||||
|
||||
interface LinuxProcessSnapshot extends LinuxProcessIdentity {
|
||||
state: string;
|
||||
}
|
||||
|
||||
export interface LocalProcessIdentityProvider {
|
||||
capture(pid: number): Promise<LinuxProcessIdentity | null>;
|
||||
inspect(
|
||||
identity: LinuxProcessIdentity,
|
||||
): Promise<LocalPersistedExecutionInspection>;
|
||||
}
|
||||
|
||||
export interface LinuxProcProcessIdentityProviderOptions {
|
||||
platform?: NodeJS.Platform;
|
||||
readTextFile?: (path: string) => Promise<string>;
|
||||
}
|
||||
|
||||
interface DurableHandlePayload {
|
||||
v: 1;
|
||||
h: string;
|
||||
b: string;
|
||||
p: number;
|
||||
g: number;
|
||||
s: string;
|
||||
}
|
||||
|
||||
function isMissingFileError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
'code' in error &&
|
||||
['ENOENT', 'ESRCH'].includes((error as NodeJS.ErrnoException).code ?? '')
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeBootId(value: string): string {
|
||||
const bootId = value.trim();
|
||||
if (!/^[A-Za-z0-9-]{1,64}$/.test(bootId)) {
|
||||
throw new Error('Linux boot id has an invalid format');
|
||||
}
|
||||
return bootId;
|
||||
}
|
||||
|
||||
function assertPositiveSafeInteger(value: number, name: string): void {
|
||||
if (!Number.isSafeInteger(value) || value < 1) {
|
||||
throw new Error(`${name} must be a positive safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertPositiveStartTimeTicks(value: string): void {
|
||||
if (!/^\d{1,32}$/.test(value) || BigInt(value) < BigInt(1)) {
|
||||
throw new Error('Linux process start time has an invalid format');
|
||||
}
|
||||
}
|
||||
|
||||
function parseLinuxProcStat(pid: number, value: string): LinuxProcessSnapshot {
|
||||
assertPositiveSafeInteger(pid, 'pid');
|
||||
const open = value.indexOf('(');
|
||||
const close = value.lastIndexOf(')');
|
||||
if (open < 1 || close <= open) {
|
||||
throw new Error('Linux process stat has an invalid command field');
|
||||
}
|
||||
const observedPid = Number(value.slice(0, open).trim());
|
||||
if (observedPid !== pid) {
|
||||
throw new Error('Linux process stat PID does not match the requested PID');
|
||||
}
|
||||
|
||||
// Values after comm begin at field 3 (state); starttime is field 22.
|
||||
const fields = value
|
||||
.slice(close + 1)
|
||||
.trim()
|
||||
.split(/\s+/);
|
||||
if (fields.length < 20) {
|
||||
throw new Error('Linux process stat is missing identity fields');
|
||||
}
|
||||
const state = fields[0];
|
||||
const processGroupIdValue = fields[2];
|
||||
const startTimeTicks = fields[19];
|
||||
if (
|
||||
state === undefined ||
|
||||
processGroupIdValue === undefined ||
|
||||
startTimeTicks === undefined
|
||||
) {
|
||||
throw new Error('Linux process stat is missing identity values');
|
||||
}
|
||||
const processGroupId = Number(processGroupIdValue);
|
||||
assertPositiveSafeInteger(processGroupId, 'processGroupId');
|
||||
assertPositiveStartTimeTicks(startTimeTicks);
|
||||
return {
|
||||
platform: 'linux',
|
||||
bootId: '',
|
||||
pid,
|
||||
processGroupId,
|
||||
startTimeTicks,
|
||||
state,
|
||||
};
|
||||
}
|
||||
|
||||
function validateIdentity(identity: LinuxProcessIdentity): void {
|
||||
if (identity.platform !== 'linux') {
|
||||
throw new Error('Local process identity has an unsupported platform');
|
||||
}
|
||||
normalizeBootId(identity.bootId);
|
||||
assertPositiveSafeInteger(identity.pid, 'pid');
|
||||
assertPositiveSafeInteger(identity.processGroupId, 'processGroupId');
|
||||
assertPositiveStartTimeTicks(identity.startTimeTicks);
|
||||
}
|
||||
|
||||
export function createLocalProcessDurableHandle(
|
||||
handleId: string,
|
||||
identity: LinuxProcessIdentity,
|
||||
): string {
|
||||
if (!handleId || handleId.length > 255 || handleId.includes('\0')) {
|
||||
throw new Error('Local process handle id has an invalid format');
|
||||
}
|
||||
validateIdentity(identity);
|
||||
const payload: DurableHandlePayload = {
|
||||
v: 1,
|
||||
h: handleId,
|
||||
b: identity.bootId,
|
||||
p: identity.pid,
|
||||
g: identity.processGroupId,
|
||||
s: identity.startTimeTicks,
|
||||
};
|
||||
const durableHandle = `${LOCAL_PROCESS_DURABLE_HANDLE_PREFIX}${Buffer.from(
|
||||
JSON.stringify(payload),
|
||||
'utf8',
|
||||
).toString('base64url')}`;
|
||||
if (
|
||||
Buffer.byteLength(durableHandle, 'utf8') >
|
||||
MAX_LOCAL_PROCESS_DURABLE_HANDLE_BYTES
|
||||
) {
|
||||
throw new Error('Local process durable handle exceeds its size limit');
|
||||
}
|
||||
return durableHandle;
|
||||
}
|
||||
|
||||
export function parseLocalProcessDurableHandle(
|
||||
durableHandle: string,
|
||||
): { handleId: string; identity: LinuxProcessIdentity } | null {
|
||||
if (
|
||||
!durableHandle.startsWith(LOCAL_PROCESS_DURABLE_HANDLE_PREFIX) ||
|
||||
Buffer.byteLength(durableHandle, 'utf8') >
|
||||
MAX_LOCAL_PROCESS_DURABLE_HANDLE_BYTES
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const encoded = durableHandle.slice(
|
||||
LOCAL_PROCESS_DURABLE_HANDLE_PREFIX.length,
|
||||
);
|
||||
if (!encoded || !/^[A-Za-z0-9_-]+$/.test(encoded)) return null;
|
||||
|
||||
try {
|
||||
const payload = JSON.parse(
|
||||
Buffer.from(encoded, 'base64url').toString('utf8'),
|
||||
) as Partial<DurableHandlePayload>;
|
||||
if (
|
||||
payload.v !== 1 ||
|
||||
typeof payload.h !== 'string' ||
|
||||
typeof payload.b !== 'string' ||
|
||||
typeof payload.p !== 'number' ||
|
||||
typeof payload.g !== 'number' ||
|
||||
typeof payload.s !== 'string'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const identity: LinuxProcessIdentity = {
|
||||
platform: 'linux',
|
||||
bootId: payload.b,
|
||||
pid: payload.p,
|
||||
processGroupId: payload.g,
|
||||
startTimeTicks: payload.s,
|
||||
};
|
||||
if (!payload.h || payload.h.length > 255 || payload.h.includes('\0')) {
|
||||
return null;
|
||||
}
|
||||
validateIdentity(identity);
|
||||
return { handleId: payload.h, identity };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export class LinuxProcProcessIdentityProvider
|
||||
implements LocalProcessIdentityProvider
|
||||
{
|
||||
private readonly platform: NodeJS.Platform;
|
||||
private readonly readTextFile: (path: string) => Promise<string>;
|
||||
|
||||
constructor(options: LinuxProcProcessIdentityProviderOptions = {}) {
|
||||
this.platform = options.platform ?? process.platform;
|
||||
this.readTextFile =
|
||||
options.readTextFile ?? ((path) => readFile(path, { encoding: 'utf8' }));
|
||||
}
|
||||
|
||||
async capture(pid: number): Promise<LinuxProcessIdentity | null> {
|
||||
if (this.platform !== 'linux') return null;
|
||||
try {
|
||||
const [bootIdValue, statValue] = await Promise.all([
|
||||
this.readTextFile(LINUX_BOOT_ID_PATH),
|
||||
this.readTextFile(`/proc/${pid}/stat`),
|
||||
]);
|
||||
const snapshot = parseLinuxProcStat(pid, statValue);
|
||||
return {
|
||||
platform: 'linux',
|
||||
bootId: normalizeBootId(bootIdValue),
|
||||
pid,
|
||||
processGroupId: snapshot.processGroupId,
|
||||
startTimeTicks: snapshot.startTimeTicks,
|
||||
};
|
||||
} catch (error) {
|
||||
if (isMissingFileError(error)) return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async inspect(
|
||||
identity: LinuxProcessIdentity,
|
||||
): Promise<LocalPersistedExecutionInspection> {
|
||||
if (this.platform !== 'linux')
|
||||
return { status: 'unknown', reason: 'unsupported_platform' };
|
||||
validateIdentity(identity);
|
||||
|
||||
let bootId: string;
|
||||
try {
|
||||
bootId = normalizeBootId(await this.readTextFile(LINUX_BOOT_ID_PATH));
|
||||
} catch (error) {
|
||||
if (isMissingFileError(error))
|
||||
return { status: 'unknown', reason: 'provider_unavailable' };
|
||||
throw error;
|
||||
}
|
||||
if (bootId !== identity.bootId)
|
||||
return { status: 'not_running', identityPid: identity.pid };
|
||||
|
||||
let snapshot: LinuxProcessSnapshot;
|
||||
try {
|
||||
snapshot = parseLinuxProcStat(
|
||||
identity.pid,
|
||||
await this.readTextFile(`/proc/${identity.pid}/stat`),
|
||||
);
|
||||
} catch (error) {
|
||||
if (isMissingFileError(error))
|
||||
return { status: 'not_running', identityPid: identity.pid };
|
||||
throw error;
|
||||
}
|
||||
if (
|
||||
snapshot.processGroupId !== identity.processGroupId ||
|
||||
snapshot.startTimeTicks !== identity.startTimeTicks
|
||||
) {
|
||||
return { status: 'not_running', identityPid: identity.pid };
|
||||
}
|
||||
if (['Z', 'X', 'x'].includes(snapshot.state))
|
||||
return { status: 'not_running', identityPid: identity.pid };
|
||||
return { status: 'running', identityPid: identity.pid };
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalProcessPersistedExecutionInspector
|
||||
implements LocalPersistedExecutionInspector
|
||||
{
|
||||
readonly executorType = 'local_process' as const;
|
||||
|
||||
constructor(
|
||||
private readonly identityProvider: LocalProcessIdentityProvider = new LinuxProcProcessIdentityProvider(),
|
||||
) {}
|
||||
|
||||
async inspect(
|
||||
durableHandle: string,
|
||||
): Promise<LocalPersistedExecutionInspection> {
|
||||
const parsed = parseLocalProcessDurableHandle(durableHandle);
|
||||
if (!parsed) return { status: 'unknown', reason: 'invalid_handle' };
|
||||
return this.identityProvider.inspect(parsed.identity);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user