mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,226 @@
|
||||
import { randomBytes } from 'crypto';
|
||||
import { constants } from 'fs';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import type { ExecutionContext, ExecutionSpec } from '../../domain/execution';
|
||||
import { assertCompletionReceiptId } from '../../domain/completionReceipt';
|
||||
import {
|
||||
ExecutorCapabilityUnavailableError,
|
||||
InvalidExecutionSpecError,
|
||||
} from '../../domain/executorErrors';
|
||||
import { assertLocalExecutionArtifactId } from '../../domain/localExecutionArtifact';
|
||||
import type { DurableLocalProcessOutput } from './durableLocalProcessOutput';
|
||||
|
||||
const CALLBACK_TOKEN_PATTERN = /^[A-Za-z0-9_-]{32,128}$/;
|
||||
|
||||
export interface DurableLocalProcessLaunch {
|
||||
file: string;
|
||||
args: readonly string[];
|
||||
environment: NodeJS.ProcessEnv;
|
||||
outputDescriptor: number;
|
||||
closeParentOutput(): Promise<void>;
|
||||
}
|
||||
|
||||
function assertCallback(
|
||||
callback: ExecutionContext['completionCallback'],
|
||||
): asserts callback is NonNullable<ExecutionContext['completionCallback']> {
|
||||
if (!callback || !CALLBACK_TOKEN_PATTERN.test(callback.token)) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable completion requires a bounded base64url callback token',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(callback.callbackSequence) ||
|
||||
callback.callbackSequence < 1
|
||||
) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable completion requires a positive callback sequence',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function launcherEnvironment(
|
||||
environment: NodeJS.ProcessEnv,
|
||||
spec: ExecutionSpec,
|
||||
context: ExecutionContext,
|
||||
capability: DurableLocalProcessOutput,
|
||||
receiptTarget: string,
|
||||
receiptTemporary: string,
|
||||
startedAtMs: number,
|
||||
quotaFifo: string | undefined,
|
||||
quotaRemainingBytes: number | undefined,
|
||||
truncationTarget: string | undefined,
|
||||
truncationTemporary: string | undefined,
|
||||
): NodeJS.ProcessEnv {
|
||||
const callback = context.completionCallback!;
|
||||
return {
|
||||
...environment,
|
||||
QL3_RECEIPT_RUN_ID: spec.runId,
|
||||
QL3_RECEIPT_ATTEMPT_ID: spec.attemptId,
|
||||
QL3_RECEIPT_CALLBACK_SEQUENCE: String(callback.callbackSequence),
|
||||
QL3_RECEIPT_CALLBACK_TOKEN: callback.token,
|
||||
QL3_RECEIPT_STARTED_AT_MS: String(startedAtMs),
|
||||
QL3_RECEIPT_TARGET: receiptTarget,
|
||||
QL3_RECEIPT_TEMPORARY: receiptTemporary,
|
||||
...(quotaFifo === undefined
|
||||
? {}
|
||||
: {
|
||||
QL3_OUTPUT_QUOTA_FIFO: quotaFifo,
|
||||
QL3_OUTPUT_QUOTA_REMAINING_BYTES: String(quotaRemainingBytes),
|
||||
QL3_OUTPUT_ARTIFACT_ID: capability.logArtifactId!,
|
||||
QL3_OUTPUT_MAXIMUM_BYTES: String(capability.maximumBytes),
|
||||
QL3_OUTPUT_TRUNCATION_TARGET: truncationTarget!,
|
||||
QL3_OUTPUT_TRUNCATION_TEMPORARY: truncationTemporary!,
|
||||
}),
|
||||
...(spec.command.kind === 'shell'
|
||||
? {
|
||||
QL3_LAUNCH_SHELL: spec.command.shell ?? '/bin/bash',
|
||||
QL3_LAUNCH_SHELL_COMMAND: spec.command.command,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function prepareDurableLocalProcessLaunch(
|
||||
spec: ExecutionSpec,
|
||||
context: ExecutionContext,
|
||||
environment: NodeJS.ProcessEnv,
|
||||
capability: DurableLocalProcessOutput,
|
||||
launcherPath: string | undefined,
|
||||
startedAtMs: number,
|
||||
): Promise<DurableLocalProcessLaunch> {
|
||||
if (!launcherPath) {
|
||||
throw new ExecutorCapabilityUnavailableError('durableLocalCompletion');
|
||||
}
|
||||
if (!path.isAbsolute(launcherPath) || launcherPath.includes('\0')) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable launcher path must be absolute and contain no NUL',
|
||||
);
|
||||
}
|
||||
assertCallback(context.completionCallback);
|
||||
assertCompletionReceiptId(spec.runId, 'runId');
|
||||
assertCompletionReceiptId(spec.attemptId, 'attemptId');
|
||||
if (!Number.isSafeInteger(startedAtMs) || startedAtMs < 0) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable completion start time must be a non-negative safe integer',
|
||||
);
|
||||
}
|
||||
|
||||
const launcher = await fs.lstat(launcherPath);
|
||||
if (!launcher.isFile()) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable launcher must be a regular file',
|
||||
);
|
||||
}
|
||||
|
||||
const receiptDirectory = path.join(
|
||||
capability.completionReceiptRoot,
|
||||
spec.attemptId.slice(0, 2),
|
||||
);
|
||||
const receiptTarget = path.join(receiptDirectory, `${spec.attemptId}.json`);
|
||||
const receiptTemporary = path.join(
|
||||
receiptDirectory,
|
||||
`.${spec.attemptId}.${randomBytes(16).toString('hex')}.tmp`,
|
||||
);
|
||||
await fs.mkdir(path.dirname(capability.outputFilePath), {
|
||||
recursive: true,
|
||||
mode: 0o700,
|
||||
});
|
||||
await fs.mkdir(receiptDirectory, { recursive: true, mode: 0o700 });
|
||||
|
||||
const output = await fs.open(
|
||||
capability.outputFilePath,
|
||||
constants.O_WRONLY |
|
||||
constants.O_CREAT |
|
||||
constants.O_APPEND |
|
||||
(constants.O_NOFOLLOW ?? 0),
|
||||
0o600,
|
||||
);
|
||||
let quotaFifo: string | undefined;
|
||||
let quotaRemainingBytes: number | undefined;
|
||||
let truncationTarget: string | undefined;
|
||||
let truncationTemporary: string | undefined;
|
||||
try {
|
||||
const stat = await output.stat();
|
||||
if (!stat.isFile()) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable output target must be a regular file',
|
||||
);
|
||||
}
|
||||
await output.chmod(0o600);
|
||||
if (capability.maximumBytes !== undefined) {
|
||||
if (
|
||||
!Number.isSafeInteger(capability.maximumBytes) ||
|
||||
capability.maximumBytes < 1 ||
|
||||
!Number.isSafeInteger(stat.size) ||
|
||||
stat.size < 0 ||
|
||||
stat.size > capability.maximumBytes
|
||||
) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable output quota or existing size is invalid',
|
||||
);
|
||||
}
|
||||
quotaRemainingBytes = capability.maximumBytes - stat.size;
|
||||
if (!capability.logArtifactId) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable output quota requires a Local Artifact identity',
|
||||
);
|
||||
}
|
||||
try {
|
||||
assertLocalExecutionArtifactId(capability.logArtifactId);
|
||||
} catch {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable output Local Artifact identity is invalid',
|
||||
);
|
||||
}
|
||||
if (
|
||||
path.basename(capability.outputFilePath) !==
|
||||
`${capability.logArtifactId}.log`
|
||||
) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable output path does not match its Local Artifact identity',
|
||||
);
|
||||
}
|
||||
quotaFifo = path.join(
|
||||
path.dirname(capability.outputFilePath),
|
||||
`.${path.basename(capability.outputFilePath)}.fifo`,
|
||||
);
|
||||
truncationTarget = path.join(
|
||||
path.dirname(capability.outputFilePath),
|
||||
`.${path.basename(capability.outputFilePath)}.truncated.json`,
|
||||
);
|
||||
truncationTemporary = path.join(
|
||||
path.dirname(capability.outputFilePath),
|
||||
`.${path.basename(capability.outputFilePath)}.truncated.tmp`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
await output.close().catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
|
||||
const launchMode = spec.command.kind;
|
||||
const args =
|
||||
spec.command.kind === 'argv'
|
||||
? [launcherPath, launchMode, spec.command.file, ...spec.command.args]
|
||||
: [launcherPath, launchMode];
|
||||
return {
|
||||
file: '/bin/sh',
|
||||
args,
|
||||
environment: launcherEnvironment(
|
||||
environment,
|
||||
spec,
|
||||
context,
|
||||
capability,
|
||||
receiptTarget,
|
||||
receiptTemporary,
|
||||
startedAtMs,
|
||||
quotaFifo,
|
||||
quotaRemainingBytes,
|
||||
truncationTarget,
|
||||
truncationTemporary,
|
||||
),
|
||||
outputDescriptor: output.fd,
|
||||
closeParentOutput: () => output.close(),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import path from 'path';
|
||||
import type { ExecutionOutputSink } from '../../domain/execution';
|
||||
import { assertLocalExecutionArtifactId } from '../../domain/localExecutionArtifact';
|
||||
|
||||
const DURABLE_LOCAL_PROCESS_OUTPUT = Symbol('durable-local-process-output');
|
||||
|
||||
export interface DurableLocalProcessOutput {
|
||||
outputFilePath: string;
|
||||
completionReceiptRoot: string;
|
||||
maximumBytes?: number;
|
||||
logArtifactId?: string;
|
||||
}
|
||||
|
||||
type CapableExecutionOutputSink = ExecutionOutputSink & {
|
||||
[DURABLE_LOCAL_PROCESS_OUTPUT]?: DurableLocalProcessOutput;
|
||||
};
|
||||
|
||||
function assertAbsolutePath(value: string, name: string): void {
|
||||
if (!path.isAbsolute(value) || value.includes('\0')) {
|
||||
throw new RangeError(`${name} must be an absolute path containing no NUL`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds an adapter-local launch capability without widening ExecutionContext.
|
||||
* The symbol is deliberately non-enumerable so paths cannot leak through
|
||||
* routine context serialization or diagnostic logging.
|
||||
*/
|
||||
export function enableDurableLocalProcessOutput<T extends ExecutionOutputSink>(
|
||||
output: T,
|
||||
capability: DurableLocalProcessOutput,
|
||||
): T {
|
||||
assertAbsolutePath(capability.outputFilePath, 'outputFilePath');
|
||||
assertAbsolutePath(capability.completionReceiptRoot, 'completionReceiptRoot');
|
||||
if (
|
||||
capability.maximumBytes !== undefined &&
|
||||
(!Number.isSafeInteger(capability.maximumBytes) ||
|
||||
capability.maximumBytes < 1)
|
||||
) {
|
||||
throw new RangeError('maximumBytes must be a positive safe integer');
|
||||
}
|
||||
if (
|
||||
(capability.maximumBytes === undefined) !==
|
||||
(capability.logArtifactId === undefined)
|
||||
) {
|
||||
throw new RangeError(
|
||||
'maximumBytes and logArtifactId must be provided together',
|
||||
);
|
||||
}
|
||||
if (capability.logArtifactId !== undefined) {
|
||||
assertLocalExecutionArtifactId(capability.logArtifactId);
|
||||
}
|
||||
Object.defineProperty(output, DURABLE_LOCAL_PROCESS_OUTPUT, {
|
||||
configurable: false,
|
||||
enumerable: false,
|
||||
writable: false,
|
||||
value: Object.freeze({ ...capability }),
|
||||
});
|
||||
return output;
|
||||
}
|
||||
|
||||
export function durableLocalProcessOutput(
|
||||
output: ExecutionOutputSink,
|
||||
): DurableLocalProcessOutput | undefined {
|
||||
return (output as CapableExecutionOutputSink)[DURABLE_LOCAL_PROCESS_OUTPUT];
|
||||
}
|
||||
@@ -0,0 +1,580 @@
|
||||
import { ChildProcess, SpawnOptions } from 'child_process';
|
||||
import { Readable } from 'stream';
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import { spawn } from 'cross-spawn';
|
||||
import type {
|
||||
ExecutionContext,
|
||||
ExecutionDiagnostic,
|
||||
ExecutionHandle,
|
||||
ExecutionInspection,
|
||||
ExecutionOutputStream,
|
||||
ExecutionResult,
|
||||
ExecutionSpec,
|
||||
ExecutionStopReason,
|
||||
ExecutionStopResult,
|
||||
ExecutorCapabilities,
|
||||
} from '../../domain/execution';
|
||||
import {
|
||||
ExecutorCapabilityUnavailableError,
|
||||
ExecutorHandleNotFoundError,
|
||||
ExecutorStartError,
|
||||
InvalidExecutionSpecError,
|
||||
} from '../../domain/executorErrors';
|
||||
import { assertExecutionSpec as assertDomainExecutionSpec } from '../../domain/executionSpec';
|
||||
export {
|
||||
MAX_EXECUTION_ARGUMENTS,
|
||||
MAX_EXECUTION_COMMAND_BYTES,
|
||||
MAX_EXECUTION_TIMEOUT_MS,
|
||||
MAX_TERMINATION_GRACE_MS,
|
||||
} from '../../domain/executionSpec';
|
||||
import type { Executor } from '../../ports/executor';
|
||||
import {
|
||||
createLocalProcessDurableHandle,
|
||||
LinuxProcProcessIdentityProvider,
|
||||
type LocalProcessIdentityProvider,
|
||||
} from './localProcessIdentity';
|
||||
import {
|
||||
PosixProcessTerminator,
|
||||
type ProcessTerminator,
|
||||
} from './processTerminator';
|
||||
import { durableLocalProcessOutput } from './durableLocalProcessOutput';
|
||||
import {
|
||||
prepareDurableLocalProcessLaunch,
|
||||
type DurableLocalProcessLaunch,
|
||||
} from './durableLocalProcessLaunch';
|
||||
|
||||
export const MAX_EXECUTION_ENVIRONMENT_ENTRIES = 1024;
|
||||
export const MAX_EXECUTION_ENVIRONMENT_BYTES = 512 * 1024;
|
||||
|
||||
const DEFAULT_POSIX_SHELL = '/bin/bash';
|
||||
const ISOLATED_ENVIRONMENT_KEYS = [
|
||||
'PATH',
|
||||
'LANG',
|
||||
'LC_ALL',
|
||||
'LC_CTYPE',
|
||||
'TZ',
|
||||
'TMPDIR',
|
||||
] as const;
|
||||
|
||||
const LOCAL_PROCESS_CAPABILITIES: ExecutorCapabilities = Object.freeze({
|
||||
timeout: true,
|
||||
processGroupTermination: process.platform !== 'win32',
|
||||
workingDirectory: true,
|
||||
isolatedEnvironment: true,
|
||||
memoryLimit: 'none',
|
||||
cpuLimit: 'none',
|
||||
filesystemIsolation: 'none',
|
||||
networkIsolation: 'none',
|
||||
});
|
||||
|
||||
export interface ExecutorClock {
|
||||
now(): number;
|
||||
}
|
||||
|
||||
export interface LocalProcessExecutorOptions {
|
||||
clock?: ExecutorClock;
|
||||
createHandleId?: () => string;
|
||||
terminator?: ProcessTerminator;
|
||||
identityProvider?: LocalProcessIdentityProvider;
|
||||
durableLauncherPath?: string;
|
||||
}
|
||||
|
||||
interface LocalExecutionLifecycle {
|
||||
startedAtMs: number;
|
||||
closedObserved: boolean;
|
||||
finished: boolean;
|
||||
result?: ExecutionResult;
|
||||
terminationReason?: ExecutionStopReason;
|
||||
runtimeError: boolean;
|
||||
diagnostics: ExecutionDiagnostic[];
|
||||
timeout?: NodeJS.Timeout;
|
||||
removeAbortListener?: () => void;
|
||||
}
|
||||
|
||||
interface LocalExecutionState {
|
||||
child: ChildProcess;
|
||||
processGroup: boolean;
|
||||
graceMs: number;
|
||||
closed: Promise<void>;
|
||||
lifecycle: LocalExecutionLifecycle;
|
||||
stopPromise?: Promise<ExecutionStopResult>;
|
||||
}
|
||||
|
||||
function assertHandleIdentifier(value: string): void {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
value.length > 255 ||
|
||||
/[\u0000-\u001f\u007f]/.test(value)
|
||||
) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'handleId must be between 1 and 255 characters and contain no control characters',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function assertResourcePolicy(spec: ExecutionSpec): void {
|
||||
const policy = spec.resourcePolicy;
|
||||
if (!policy) return;
|
||||
|
||||
if (policy.memoryBytes?.enforcement === 'required') {
|
||||
throw new ExecutorCapabilityUnavailableError('memoryLimit');
|
||||
}
|
||||
if (policy.cpuMillisPerSecond?.enforcement === 'required') {
|
||||
throw new ExecutorCapabilityUnavailableError('cpuLimit');
|
||||
}
|
||||
if (policy.filesystemIsolation === 'required') {
|
||||
throw new ExecutorCapabilityUnavailableError('filesystemIsolation');
|
||||
}
|
||||
if (policy.networkIsolation === 'required') {
|
||||
throw new ExecutorCapabilityUnavailableError('networkIsolation');
|
||||
}
|
||||
}
|
||||
|
||||
function resourcePolicyDiagnostics(spec: ExecutionSpec): ExecutionDiagnostic[] {
|
||||
const policy = spec.resourcePolicy;
|
||||
if (!policy) return [];
|
||||
|
||||
const unavailable = [
|
||||
policy.memoryBytes?.enforcement === 'best_effort' ? 'memoryLimit' : null,
|
||||
policy.cpuMillisPerSecond?.enforcement === 'best_effort'
|
||||
? 'cpuLimit'
|
||||
: null,
|
||||
policy.filesystemIsolation === 'best_effort' ? 'filesystemIsolation' : null,
|
||||
policy.networkIsolation === 'best_effort' ? 'networkIsolation' : null,
|
||||
].filter((value): value is string => value !== null);
|
||||
return unavailable.length === 0
|
||||
? []
|
||||
: [
|
||||
{
|
||||
code: 'RESOURCE_POLICY_BEST_EFFORT_UNAVAILABLE',
|
||||
summary: `Best-effort capabilities were unavailable: ${unavailable.join(
|
||||
', ',
|
||||
)}`,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function assertExecutionSpec(spec: ExecutionSpec): void {
|
||||
assertDomainExecutionSpec(spec);
|
||||
assertResourcePolicy(spec);
|
||||
}
|
||||
|
||||
function environmentBytes(environment: NodeJS.ProcessEnv): number {
|
||||
return Object.entries(environment).reduce(
|
||||
(total, [key, value]) =>
|
||||
total +
|
||||
Buffer.byteLength(key, 'utf8') +
|
||||
Buffer.byteLength(value ?? '', 'utf8'),
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
function buildEnvironment(
|
||||
policy: ExecutionSpec['environmentPolicy'],
|
||||
supplied: Readonly<Record<string, string>>,
|
||||
): NodeJS.ProcessEnv {
|
||||
if (Object.keys(supplied).length > MAX_EXECUTION_ENVIRONMENT_ENTRIES) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'execution environment has too many entries',
|
||||
);
|
||||
}
|
||||
|
||||
const environment: NodeJS.ProcessEnv = {};
|
||||
if (policy === 'inherit') {
|
||||
Object.assign(environment, process.env);
|
||||
} else {
|
||||
for (const key of ISOLATED_ENVIRONMENT_KEYS) {
|
||||
if (process.env[key] !== undefined) environment[key] = process.env[key];
|
||||
}
|
||||
}
|
||||
|
||||
for (const [key, value] of Object.entries(supplied)) {
|
||||
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key) || value.includes('\0')) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'execution environment contains an invalid key or NUL value',
|
||||
);
|
||||
}
|
||||
environment[key] = value;
|
||||
}
|
||||
|
||||
if (environmentBytes(environment) > MAX_EXECUTION_ENVIRONMENT_BYTES) {
|
||||
throw new InvalidExecutionSpecError('execution environment is too large');
|
||||
}
|
||||
return environment;
|
||||
}
|
||||
|
||||
function diagnosticOnce(
|
||||
lifecycle: LocalExecutionLifecycle,
|
||||
diagnostic: ExecutionDiagnostic,
|
||||
): void {
|
||||
if (!lifecycle.diagnostics.some((item) => item.code === diagnostic.code)) {
|
||||
lifecycle.diagnostics.push(diagnostic);
|
||||
}
|
||||
}
|
||||
|
||||
function createResult(
|
||||
lifecycle: LocalExecutionLifecycle,
|
||||
code: number | null,
|
||||
signal: NodeJS.Signals | null,
|
||||
finishedAtMs: number,
|
||||
): ExecutionResult {
|
||||
const base = {
|
||||
startedAtMs: lifecycle.startedAtMs,
|
||||
finishedAtMs: Math.max(finishedAtMs, lifecycle.startedAtMs),
|
||||
...(code === null ? {} : { exitCode: code }),
|
||||
...(signal === null ? {} : { signal }),
|
||||
...(lifecycle.diagnostics.length === 0
|
||||
? {}
|
||||
: { diagnostics: [...lifecycle.diagnostics] }),
|
||||
};
|
||||
|
||||
if (lifecycle.terminationReason?.kind === 'timeout') {
|
||||
return {
|
||||
...base,
|
||||
outcome: 'timed_out',
|
||||
errorCode: 'EXECUTION_TIMED_OUT',
|
||||
errorSummary: 'Execution exceeded its configured timeout',
|
||||
};
|
||||
}
|
||||
if (lifecycle.terminationReason) {
|
||||
return {
|
||||
...base,
|
||||
outcome: 'cancelled',
|
||||
errorCode: 'EXECUTION_CANCELLED',
|
||||
errorSummary: 'Execution was cancelled',
|
||||
};
|
||||
}
|
||||
if (lifecycle.runtimeError) {
|
||||
return {
|
||||
...base,
|
||||
outcome: 'failed',
|
||||
errorCode: 'PROCESS_RUNTIME_ERROR',
|
||||
errorSummary: 'The child process reported a runtime error',
|
||||
};
|
||||
}
|
||||
if (code === 0) return { ...base, outcome: 'succeeded' };
|
||||
if (code !== null) {
|
||||
return {
|
||||
...base,
|
||||
outcome: 'failed',
|
||||
errorCode: 'PROCESS_EXIT_NON_ZERO',
|
||||
errorSummary: `Process exited with code ${code}`,
|
||||
};
|
||||
}
|
||||
if (signal !== null) {
|
||||
return {
|
||||
...base,
|
||||
outcome: 'failed',
|
||||
errorCode: 'PROCESS_SIGNALLED',
|
||||
errorSummary: `Process exited after signal ${signal}`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
outcome: 'failed',
|
||||
errorCode: 'PROCESS_EXIT_UNKNOWN',
|
||||
errorSummary: 'Process exited without an exit code or signal',
|
||||
};
|
||||
}
|
||||
|
||||
export class LocalProcessExecutor implements Executor {
|
||||
readonly type = 'local_process' as const;
|
||||
|
||||
private readonly clock: ExecutorClock;
|
||||
private readonly createHandleId: () => string;
|
||||
private readonly terminator: ProcessTerminator;
|
||||
private readonly identityProvider: LocalProcessIdentityProvider;
|
||||
private readonly durableLauncherPath?: string;
|
||||
private readonly states = new WeakMap<ExecutionHandle, LocalExecutionState>();
|
||||
|
||||
constructor(options: LocalProcessExecutorOptions = {}) {
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.createHandleId = options.createHandleId ?? uuidV7;
|
||||
this.terminator = options.terminator ?? new PosixProcessTerminator();
|
||||
this.identityProvider =
|
||||
options.identityProvider ?? new LinuxProcProcessIdentityProvider();
|
||||
this.durableLauncherPath = options.durableLauncherPath;
|
||||
}
|
||||
|
||||
capabilities(): ExecutorCapabilities {
|
||||
return LOCAL_PROCESS_CAPABILITIES;
|
||||
}
|
||||
|
||||
async start(
|
||||
spec: ExecutionSpec,
|
||||
context: ExecutionContext,
|
||||
): Promise<ExecutionHandle> {
|
||||
assertExecutionSpec(spec);
|
||||
const environment = buildEnvironment(
|
||||
spec.environmentPolicy,
|
||||
context.environment,
|
||||
);
|
||||
const handleId = this.createHandleId();
|
||||
assertHandleIdentifier(handleId);
|
||||
if (context.signal?.aborted) {
|
||||
throw new ExecutorStartError(
|
||||
new Error('Execution was aborted before spawn'),
|
||||
);
|
||||
}
|
||||
if (
|
||||
context.signal &&
|
||||
(!context.signal.addEventListener || !context.signal.removeEventListener)
|
||||
) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'Execution abort signal must support event listeners',
|
||||
);
|
||||
}
|
||||
|
||||
const processGroup = process.platform !== 'win32';
|
||||
const durableOutput = durableLocalProcessOutput(context.output);
|
||||
let durableLaunch: DurableLocalProcessLaunch | undefined;
|
||||
if (durableOutput) {
|
||||
durableLaunch = await prepareDurableLocalProcessLaunch(
|
||||
spec,
|
||||
context,
|
||||
environment,
|
||||
durableOutput,
|
||||
this.durableLauncherPath,
|
||||
this.clock.now(),
|
||||
);
|
||||
}
|
||||
const options: SpawnOptions = {
|
||||
cwd: spec.workingDirectory,
|
||||
env: durableLaunch?.environment ?? environment,
|
||||
detached: processGroup,
|
||||
stdio: durableLaunch
|
||||
? [
|
||||
'ignore',
|
||||
durableLaunch.outputDescriptor,
|
||||
durableLaunch.outputDescriptor,
|
||||
]
|
||||
: ['ignore', 'pipe', 'pipe'],
|
||||
};
|
||||
let child: ChildProcess;
|
||||
try {
|
||||
child = durableLaunch
|
||||
? spawn(durableLaunch.file, [...durableLaunch.args], options)
|
||||
: spec.command.kind === 'argv'
|
||||
? spawn(spec.command.file, [...spec.command.args], options)
|
||||
: spawn(spec.command.command, {
|
||||
...options,
|
||||
shell: spec.command.shell ?? DEFAULT_POSIX_SHELL,
|
||||
});
|
||||
} catch (error) {
|
||||
await durableLaunch?.closeParentOutput().catch(() => undefined);
|
||||
throw new ExecutorStartError(error);
|
||||
}
|
||||
|
||||
const lifecycle: LocalExecutionLifecycle = {
|
||||
startedAtMs: 0,
|
||||
closedObserved: false,
|
||||
finished: false,
|
||||
runtimeError: false,
|
||||
diagnostics: resourcePolicyDiagnostics(spec),
|
||||
};
|
||||
let resolveClosed: () => void = () => undefined;
|
||||
const closed = new Promise<void>((resolve) => {
|
||||
resolveClosed = resolve;
|
||||
});
|
||||
const outputPumps = durableLaunch
|
||||
? []
|
||||
: [
|
||||
this.pumpOutput(child.stdout, 'stdout', context, lifecycle),
|
||||
this.pumpOutput(child.stderr, 'stderr', context, lifecycle),
|
||||
];
|
||||
|
||||
let spawnConfirmed = false;
|
||||
const spawned = new Promise<void>((resolve, reject) => {
|
||||
child.once('spawn', () => {
|
||||
spawnConfirmed = true;
|
||||
lifecycle.startedAtMs = this.clock.now();
|
||||
resolve();
|
||||
});
|
||||
child.on('error', (error) => {
|
||||
if (!spawnConfirmed) reject(error);
|
||||
else lifecycle.runtimeError = true;
|
||||
});
|
||||
});
|
||||
|
||||
const completion = new Promise<ExecutionResult>((resolve) => {
|
||||
child.once('close', (code, signal) => {
|
||||
lifecycle.closedObserved = true;
|
||||
resolveClosed();
|
||||
void Promise.all(outputPumps).then(() => {
|
||||
if (lifecycle.timeout) clearTimeout(lifecycle.timeout);
|
||||
lifecycle.removeAbortListener?.();
|
||||
const result = createResult(
|
||||
lifecycle,
|
||||
code,
|
||||
signal,
|
||||
this.clock.now(),
|
||||
);
|
||||
lifecycle.result = result;
|
||||
lifecycle.finished = true;
|
||||
resolve(result);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
try {
|
||||
await spawned;
|
||||
} catch (error) {
|
||||
await durableLaunch?.closeParentOutput().catch(() => undefined);
|
||||
throw new ExecutorStartError(error);
|
||||
}
|
||||
await durableLaunch?.closeParentOutput().catch(() => undefined);
|
||||
if (!child.pid) {
|
||||
throw new ExecutorStartError(new Error('Spawn did not return a PID'));
|
||||
}
|
||||
|
||||
let durableHandle: string | undefined;
|
||||
try {
|
||||
const identity = await this.identityProvider.capture(child.pid);
|
||||
if (identity) {
|
||||
durableHandle = createLocalProcessDurableHandle(handleId, identity);
|
||||
}
|
||||
} catch {
|
||||
// Recovery identity is optional; the Reconciler will conservatively mark
|
||||
// an unprovable execution lost and will never signal by PID alone.
|
||||
}
|
||||
|
||||
const handle: ExecutionHandle = {
|
||||
id: handleId,
|
||||
...(durableHandle === undefined ? {} : { durableHandle }),
|
||||
executorType: this.type,
|
||||
runId: spec.runId,
|
||||
attemptId: spec.attemptId,
|
||||
startedAtMs: lifecycle.startedAtMs,
|
||||
pid: child.pid,
|
||||
completion,
|
||||
};
|
||||
const state: LocalExecutionState = {
|
||||
child,
|
||||
processGroup,
|
||||
graceMs: spec.terminationGraceMs,
|
||||
closed,
|
||||
lifecycle,
|
||||
};
|
||||
this.states.set(handle, state);
|
||||
|
||||
if (!lifecycle.closedObserved && spec.timeoutMs !== undefined) {
|
||||
lifecycle.timeout = setTimeout(() => {
|
||||
void this.stop(handle, {
|
||||
kind: 'timeout',
|
||||
requestedAtMs: this.clock.now(),
|
||||
}).catch(() => {
|
||||
diagnosticOnce(lifecycle, {
|
||||
code: 'TIMEOUT_STOP_FAILED',
|
||||
summary: 'Executor could not stop the process after timeout',
|
||||
});
|
||||
});
|
||||
}, spec.timeoutMs);
|
||||
lifecycle.timeout.unref?.();
|
||||
}
|
||||
|
||||
if (!lifecycle.closedObserved && context.signal) {
|
||||
const onAbort = () => {
|
||||
void this.stop(handle, {
|
||||
kind: 'user',
|
||||
requestedAtMs: this.clock.now(),
|
||||
}).catch(() => {
|
||||
diagnosticOnce(lifecycle, {
|
||||
code: 'ABORT_STOP_FAILED',
|
||||
summary: 'Executor could not stop the process after abort',
|
||||
});
|
||||
});
|
||||
};
|
||||
context.signal.addEventListener!('abort', onAbort, { once: true });
|
||||
lifecycle.removeAbortListener = () =>
|
||||
context.signal?.removeEventListener?.('abort', onAbort);
|
||||
if (context.signal.aborted) onAbort();
|
||||
}
|
||||
|
||||
return handle;
|
||||
}
|
||||
|
||||
async stop(
|
||||
handle: ExecutionHandle,
|
||||
reason: ExecutionStopReason,
|
||||
): Promise<ExecutionStopResult> {
|
||||
const state = this.states.get(handle);
|
||||
if (!state) throw new ExecutorHandleNotFoundError(handle.id);
|
||||
if (state.lifecycle.closedObserved) {
|
||||
return {
|
||||
status: 'already_exited',
|
||||
termSignalSent: false,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
if (state.stopPromise) return state.stopPromise;
|
||||
|
||||
state.lifecycle.terminationReason = reason;
|
||||
state.stopPromise = this.terminator
|
||||
.terminate({
|
||||
pid: state.child.pid!,
|
||||
processGroup: state.processGroup,
|
||||
graceMs: state.graceMs,
|
||||
closed: state.closed,
|
||||
})
|
||||
.then((result) => ({
|
||||
status: result.alreadyExited
|
||||
? ('already_exited' as const)
|
||||
: ('termination_requested' as const),
|
||||
termSignalSent: result.termSignalSent,
|
||||
killSignalSent: result.killSignalSent,
|
||||
}));
|
||||
return state.stopPromise;
|
||||
}
|
||||
|
||||
async inspect(handle: ExecutionHandle): Promise<ExecutionInspection> {
|
||||
const state = this.states.get(handle);
|
||||
if (!state) throw new ExecutorHandleNotFoundError(handle.id);
|
||||
if (state.lifecycle.closedObserved) {
|
||||
return {
|
||||
status: 'exited',
|
||||
...(state.lifecycle.result === undefined
|
||||
? {}
|
||||
: { result: state.lifecycle.result }),
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: state.stopPromise ? 'stopping' : 'running',
|
||||
};
|
||||
}
|
||||
|
||||
private async pumpOutput(
|
||||
stream: Readable | null,
|
||||
outputStream: ExecutionOutputStream,
|
||||
context: ExecutionContext,
|
||||
lifecycle: LocalExecutionLifecycle,
|
||||
): Promise<void> {
|
||||
if (!stream) return;
|
||||
let sinkAvailable = true;
|
||||
try {
|
||||
for await (const value of stream) {
|
||||
if (!sinkAvailable) continue;
|
||||
try {
|
||||
const chunk =
|
||||
value instanceof Uint8Array ? value : Buffer.from(String(value));
|
||||
await context.output.write({
|
||||
stream: outputStream,
|
||||
chunk,
|
||||
observedAtMs: this.clock.now(),
|
||||
});
|
||||
} catch {
|
||||
sinkAvailable = false;
|
||||
diagnosticOnce(lifecycle, {
|
||||
code: 'OUTPUT_SINK_FAILED',
|
||||
summary: 'Execution output sink failed; output may be incomplete',
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
diagnosticOnce(lifecycle, {
|
||||
code: 'OUTPUT_STREAM_FAILED',
|
||||
summary: 'Execution output stream failed; output may be incomplete',
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
import { readFile } from 'fs/promises';
|
||||
import type {
|
||||
PersistedExecutionInspection,
|
||||
PersistedExecutionInspector,
|
||||
} from '../../ports/persistedExecutionInspector';
|
||||
|
||||
export const LOCAL_PROCESS_DURABLE_HANDLE_PREFIX = 'ql3lp1.';
|
||||
export const MAX_LOCAL_PROCESS_DURABLE_HANDLE_BYTES = 512;
|
||||
|
||||
const LINUX_BOOT_ID_PATH = '/proc/sys/kernel/random/boot_id';
|
||||
|
||||
export interface LinuxProcessIdentity {
|
||||
platform: 'linux';
|
||||
bootId: string;
|
||||
pid: number;
|
||||
processGroupId: number;
|
||||
startTimeTicks: string;
|
||||
}
|
||||
|
||||
interface LinuxProcessSnapshot extends LinuxProcessIdentity {
|
||||
state: string;
|
||||
}
|
||||
|
||||
export interface LocalProcessIdentityProvider {
|
||||
capture(pid: number): Promise<LinuxProcessIdentity | null>;
|
||||
inspect(
|
||||
identity: LinuxProcessIdentity,
|
||||
): Promise<PersistedExecutionInspection>;
|
||||
}
|
||||
|
||||
export interface LinuxProcProcessIdentityProviderOptions {
|
||||
platform?: NodeJS.Platform;
|
||||
readTextFile?: (path: string) => Promise<string>;
|
||||
}
|
||||
|
||||
interface DurableHandlePayload {
|
||||
v: 1;
|
||||
h: string;
|
||||
b: string;
|
||||
p: number;
|
||||
g: number;
|
||||
s: string;
|
||||
}
|
||||
|
||||
function isMissingFileError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
'code' in error &&
|
||||
['ENOENT', 'ESRCH'].includes((error as NodeJS.ErrnoException).code ?? '')
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeBootId(value: string): string {
|
||||
const bootId = value.trim();
|
||||
if (!/^[A-Za-z0-9-]{1,64}$/.test(bootId)) {
|
||||
throw new Error('Linux boot id has an invalid format');
|
||||
}
|
||||
return bootId;
|
||||
}
|
||||
|
||||
function assertPositiveSafeInteger(value: number, name: string): void {
|
||||
if (!Number.isSafeInteger(value) || value < 1) {
|
||||
throw new Error(`${name} must be a positive safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertPositiveStartTimeTicks(value: string): void {
|
||||
if (!/^\d{1,32}$/.test(value) || BigInt(value) < BigInt(1)) {
|
||||
throw new Error('Linux process start time has an invalid format');
|
||||
}
|
||||
}
|
||||
|
||||
function parseLinuxProcStat(pid: number, value: string): LinuxProcessSnapshot {
|
||||
assertPositiveSafeInteger(pid, 'pid');
|
||||
const open = value.indexOf('(');
|
||||
const close = value.lastIndexOf(')');
|
||||
if (open < 1 || close <= open) {
|
||||
throw new Error('Linux process stat has an invalid command field');
|
||||
}
|
||||
const observedPid = Number(value.slice(0, open).trim());
|
||||
if (observedPid !== pid) {
|
||||
throw new Error('Linux process stat PID does not match the requested PID');
|
||||
}
|
||||
|
||||
// Values after comm begin at field 3 (state); starttime is field 22.
|
||||
const fields = value
|
||||
.slice(close + 1)
|
||||
.trim()
|
||||
.split(/\s+/);
|
||||
if (fields.length < 20) {
|
||||
throw new Error('Linux process stat is missing identity fields');
|
||||
}
|
||||
const state = fields[0];
|
||||
const processGroupId = Number(fields[2]);
|
||||
const startTimeTicks = fields[19];
|
||||
assertPositiveSafeInteger(processGroupId, 'processGroupId');
|
||||
assertPositiveStartTimeTicks(startTimeTicks);
|
||||
return {
|
||||
platform: 'linux',
|
||||
bootId: '',
|
||||
pid,
|
||||
processGroupId,
|
||||
startTimeTicks,
|
||||
state,
|
||||
};
|
||||
}
|
||||
|
||||
function validateIdentity(identity: LinuxProcessIdentity): void {
|
||||
if (identity.platform !== 'linux') {
|
||||
throw new Error('Local process identity has an unsupported platform');
|
||||
}
|
||||
normalizeBootId(identity.bootId);
|
||||
assertPositiveSafeInteger(identity.pid, 'pid');
|
||||
assertPositiveSafeInteger(identity.processGroupId, 'processGroupId');
|
||||
assertPositiveStartTimeTicks(identity.startTimeTicks);
|
||||
}
|
||||
|
||||
export function createLocalProcessDurableHandle(
|
||||
handleId: string,
|
||||
identity: LinuxProcessIdentity,
|
||||
): string {
|
||||
if (!handleId || handleId.length > 255 || handleId.includes('\0')) {
|
||||
throw new Error('Local process handle id has an invalid format');
|
||||
}
|
||||
validateIdentity(identity);
|
||||
const payload: DurableHandlePayload = {
|
||||
v: 1,
|
||||
h: handleId,
|
||||
b: identity.bootId,
|
||||
p: identity.pid,
|
||||
g: identity.processGroupId,
|
||||
s: identity.startTimeTicks,
|
||||
};
|
||||
const durableHandle = `${LOCAL_PROCESS_DURABLE_HANDLE_PREFIX}${Buffer.from(
|
||||
JSON.stringify(payload),
|
||||
'utf8',
|
||||
).toString('base64url')}`;
|
||||
if (
|
||||
Buffer.byteLength(durableHandle, 'utf8') >
|
||||
MAX_LOCAL_PROCESS_DURABLE_HANDLE_BYTES
|
||||
) {
|
||||
throw new Error('Local process durable handle exceeds its size limit');
|
||||
}
|
||||
return durableHandle;
|
||||
}
|
||||
|
||||
export function parseLocalProcessDurableHandle(
|
||||
durableHandle: string,
|
||||
): { handleId: string; identity: LinuxProcessIdentity } | null {
|
||||
if (
|
||||
!durableHandle.startsWith(LOCAL_PROCESS_DURABLE_HANDLE_PREFIX) ||
|
||||
Buffer.byteLength(durableHandle, 'utf8') >
|
||||
MAX_LOCAL_PROCESS_DURABLE_HANDLE_BYTES
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const encoded = durableHandle.slice(
|
||||
LOCAL_PROCESS_DURABLE_HANDLE_PREFIX.length,
|
||||
);
|
||||
if (!encoded || !/^[A-Za-z0-9_-]+$/.test(encoded)) return null;
|
||||
|
||||
try {
|
||||
const payload = JSON.parse(
|
||||
Buffer.from(encoded, 'base64url').toString('utf8'),
|
||||
) as Partial<DurableHandlePayload>;
|
||||
if (
|
||||
payload.v !== 1 ||
|
||||
typeof payload.h !== 'string' ||
|
||||
typeof payload.b !== 'string' ||
|
||||
typeof payload.p !== 'number' ||
|
||||
typeof payload.g !== 'number' ||
|
||||
typeof payload.s !== 'string'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const identity: LinuxProcessIdentity = {
|
||||
platform: 'linux',
|
||||
bootId: payload.b,
|
||||
pid: payload.p,
|
||||
processGroupId: payload.g,
|
||||
startTimeTicks: payload.s,
|
||||
};
|
||||
if (!payload.h || payload.h.length > 255 || payload.h.includes('\0')) {
|
||||
return null;
|
||||
}
|
||||
validateIdentity(identity);
|
||||
return { handleId: payload.h, identity };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export class LinuxProcProcessIdentityProvider
|
||||
implements LocalProcessIdentityProvider
|
||||
{
|
||||
private readonly platform: NodeJS.Platform;
|
||||
private readonly readTextFile: (path: string) => Promise<string>;
|
||||
|
||||
constructor(options: LinuxProcProcessIdentityProviderOptions = {}) {
|
||||
this.platform = options.platform ?? process.platform;
|
||||
this.readTextFile =
|
||||
options.readTextFile ?? ((path) => readFile(path, { encoding: 'utf8' }));
|
||||
}
|
||||
|
||||
async capture(pid: number): Promise<LinuxProcessIdentity | null> {
|
||||
if (this.platform !== 'linux') return null;
|
||||
try {
|
||||
const [bootIdValue, statValue] = await Promise.all([
|
||||
this.readTextFile(LINUX_BOOT_ID_PATH),
|
||||
this.readTextFile(`/proc/${pid}/stat`),
|
||||
]);
|
||||
const snapshot = parseLinuxProcStat(pid, statValue);
|
||||
return {
|
||||
platform: 'linux',
|
||||
bootId: normalizeBootId(bootIdValue),
|
||||
pid,
|
||||
processGroupId: snapshot.processGroupId,
|
||||
startTimeTicks: snapshot.startTimeTicks,
|
||||
};
|
||||
} catch (error) {
|
||||
if (isMissingFileError(error)) return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async inspect(
|
||||
identity: LinuxProcessIdentity,
|
||||
): Promise<PersistedExecutionInspection> {
|
||||
if (this.platform !== 'linux') return { status: 'unsupported' };
|
||||
validateIdentity(identity);
|
||||
|
||||
let bootId: string;
|
||||
try {
|
||||
bootId = normalizeBootId(await this.readTextFile(LINUX_BOOT_ID_PATH));
|
||||
} catch (error) {
|
||||
if (isMissingFileError(error)) return { status: 'unsupported' };
|
||||
throw error;
|
||||
}
|
||||
if (bootId !== identity.bootId) return { status: 'identity_mismatch' };
|
||||
|
||||
let snapshot: LinuxProcessSnapshot;
|
||||
try {
|
||||
snapshot = parseLinuxProcStat(
|
||||
identity.pid,
|
||||
await this.readTextFile(`/proc/${identity.pid}/stat`),
|
||||
);
|
||||
} catch (error) {
|
||||
if (isMissingFileError(error)) return { status: 'exited' };
|
||||
throw error;
|
||||
}
|
||||
if (
|
||||
snapshot.processGroupId !== identity.processGroupId ||
|
||||
snapshot.startTimeTicks !== identity.startTimeTicks
|
||||
) {
|
||||
return { status: 'identity_mismatch' };
|
||||
}
|
||||
if (['Z', 'X', 'x'].includes(snapshot.state)) return { status: 'exited' };
|
||||
return { status: 'running' };
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalProcessPersistedExecutionInspector
|
||||
implements PersistedExecutionInspector
|
||||
{
|
||||
readonly executorType = 'local_process' as const;
|
||||
|
||||
constructor(
|
||||
private readonly identityProvider: LocalProcessIdentityProvider = new LinuxProcProcessIdentityProvider(),
|
||||
) {}
|
||||
|
||||
async inspect(durableHandle: string): Promise<PersistedExecutionInspection> {
|
||||
const parsed = parseLocalProcessDurableHandle(durableHandle);
|
||||
if (!parsed) return { status: 'invalid' };
|
||||
return {
|
||||
...(await this.identityProvider.inspect(parsed.identity)),
|
||||
identityPid: parsed.identity.pid,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
import { ExecutorStopError } from '../../domain/executorErrors';
|
||||
import type {
|
||||
PersistedExecutionController,
|
||||
PersistedExecutionStopResult,
|
||||
PersistedExecutionStopStatus,
|
||||
} from '../../ports/persistedExecutionController';
|
||||
import {
|
||||
LinuxProcProcessIdentityProvider,
|
||||
parseLocalProcessDurableHandle,
|
||||
type LocalProcessIdentityProvider,
|
||||
type LinuxProcessIdentity,
|
||||
} from './localProcessIdentity';
|
||||
|
||||
export const MAX_PERSISTED_LOCAL_STOP_GRACE_MS = 60_000;
|
||||
|
||||
export type PersistedLocalProcessSignalSender = (
|
||||
pid: number,
|
||||
signal: NodeJS.Signals,
|
||||
) => void;
|
||||
|
||||
export interface PersistedLocalProcessControllerOptions {
|
||||
identityProvider?: LocalProcessIdentityProvider;
|
||||
sendSignal?: PersistedLocalProcessSignalSender;
|
||||
graceMs?: number;
|
||||
pollIntervalMs?: number;
|
||||
sleep?: (delayMs: number) => Promise<void>;
|
||||
}
|
||||
|
||||
function isNoSuchProcessError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
'code' in error &&
|
||||
(error as NodeJS.ErrnoException).code === 'ESRCH'
|
||||
);
|
||||
}
|
||||
|
||||
function withoutSignal(status: PersistedExecutionStopStatus) {
|
||||
return { status, termSignalSent: false, killSignalSent: false } as const;
|
||||
}
|
||||
|
||||
function mappedInspectionStatus(
|
||||
status: 'identity_mismatch' | 'unsupported' | 'invalid',
|
||||
termSignalSent: boolean,
|
||||
): PersistedExecutionStopResult {
|
||||
return {
|
||||
status,
|
||||
termSignalSent,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
|
||||
export class LocalProcessPersistedExecutionController
|
||||
implements PersistedExecutionController
|
||||
{
|
||||
readonly executorType = 'local_process' as const;
|
||||
private readonly identityProvider: LocalProcessIdentityProvider;
|
||||
private readonly sendSignal: PersistedLocalProcessSignalSender;
|
||||
private readonly graceMs: number;
|
||||
private readonly pollIntervalMs: number;
|
||||
private readonly sleep: (delayMs: number) => Promise<void>;
|
||||
|
||||
constructor(options: PersistedLocalProcessControllerOptions = {}) {
|
||||
this.identityProvider =
|
||||
options.identityProvider ?? new LinuxProcProcessIdentityProvider();
|
||||
this.sendSignal = options.sendSignal ?? process.kill;
|
||||
this.graceMs = options.graceMs ?? 5_000;
|
||||
this.pollIntervalMs = options.pollIntervalMs ?? 50;
|
||||
this.sleep =
|
||||
options.sleep ??
|
||||
((delayMs) =>
|
||||
new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, delayMs);
|
||||
}));
|
||||
if (
|
||||
!Number.isSafeInteger(this.graceMs) ||
|
||||
this.graceMs < 0 ||
|
||||
this.graceMs > MAX_PERSISTED_LOCAL_STOP_GRACE_MS
|
||||
) {
|
||||
throw new RangeError('Persisted local stop graceMs is invalid');
|
||||
}
|
||||
if (!Number.isSafeInteger(this.pollIntervalMs) || this.pollIntervalMs < 1) {
|
||||
throw new RangeError(
|
||||
'Persisted local stop pollIntervalMs must be a positive integer',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async stop({
|
||||
durableHandle,
|
||||
expectedPid,
|
||||
}: Parameters<
|
||||
PersistedExecutionController['stop']
|
||||
>[0]): Promise<PersistedExecutionStopResult> {
|
||||
const parsed = parseLocalProcessDurableHandle(durableHandle);
|
||||
if (!parsed) return withoutSignal('invalid');
|
||||
const identity = parsed.identity;
|
||||
if (expectedPid !== undefined && expectedPid !== identity.pid) {
|
||||
return withoutSignal('pid_mismatch');
|
||||
}
|
||||
// LocalProcessExecutor uses a detached child as process-group leader.
|
||||
if (identity.processGroupId !== identity.pid) {
|
||||
return withoutSignal('identity_mismatch');
|
||||
}
|
||||
|
||||
const initial = await this.identityProvider.inspect(identity);
|
||||
if (initial.status === 'exited') return withoutSignal('already_exited');
|
||||
if (initial.status !== 'running') {
|
||||
return mappedInspectionStatus(initial.status, false);
|
||||
}
|
||||
|
||||
if (!this.trySignal(identity, 'SIGTERM')) {
|
||||
return withoutSignal('already_exited');
|
||||
}
|
||||
|
||||
let waitedMs = 0;
|
||||
while (waitedMs < this.graceMs) {
|
||||
const delayMs = Math.min(this.pollIntervalMs, this.graceMs - waitedMs);
|
||||
await this.sleep(delayMs);
|
||||
waitedMs += delayMs;
|
||||
const inspection = await this.identityProvider.inspect(identity);
|
||||
if (inspection.status === 'exited') {
|
||||
return {
|
||||
status: 'termination_requested',
|
||||
termSignalSent: true,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
if (inspection.status !== 'running') {
|
||||
return mappedInspectionStatus(inspection.status, true);
|
||||
}
|
||||
}
|
||||
|
||||
const finalInspection = await this.identityProvider.inspect(identity);
|
||||
if (finalInspection.status === 'exited') {
|
||||
return {
|
||||
status: 'termination_requested',
|
||||
termSignalSent: true,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
if (finalInspection.status !== 'running') {
|
||||
return mappedInspectionStatus(finalInspection.status, true);
|
||||
}
|
||||
const killSignalSent = this.trySignal(identity, 'SIGKILL');
|
||||
return {
|
||||
status: 'termination_requested',
|
||||
termSignalSent: true,
|
||||
killSignalSent,
|
||||
};
|
||||
}
|
||||
|
||||
private trySignal(
|
||||
identity: LinuxProcessIdentity,
|
||||
signal: NodeJS.Signals,
|
||||
): boolean {
|
||||
try {
|
||||
this.sendSignal(-identity.processGroupId, signal);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isNoSuchProcessError(error)) return false;
|
||||
throw new ExecutorStopError(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import { ExecutorStopError } from '../../domain/executorErrors';
|
||||
|
||||
export interface ProcessTerminationRequest {
|
||||
pid: number;
|
||||
processGroup: boolean;
|
||||
graceMs: number;
|
||||
closed: Promise<void>;
|
||||
}
|
||||
|
||||
export interface ProcessTerminationResult {
|
||||
alreadyExited: boolean;
|
||||
termSignalSent: boolean;
|
||||
killSignalSent: boolean;
|
||||
}
|
||||
|
||||
export interface ProcessTerminator {
|
||||
terminate(
|
||||
request: ProcessTerminationRequest,
|
||||
): Promise<ProcessTerminationResult>;
|
||||
}
|
||||
|
||||
export type ProcessSignalSender = (pid: number, signal: NodeJS.Signals) => void;
|
||||
|
||||
function isNoSuchProcessError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
'code' in error &&
|
||||
(error as NodeJS.ErrnoException).code === 'ESRCH'
|
||||
);
|
||||
}
|
||||
|
||||
async function exitsWithin(closed: Promise<void>, timeoutMs: number) {
|
||||
if (timeoutMs === 0) return false;
|
||||
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
try {
|
||||
return await Promise.race([
|
||||
closed.then(() => true),
|
||||
new Promise<boolean>((resolve) => {
|
||||
timer = setTimeout(() => resolve(false), timeoutMs);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
export class PosixProcessTerminator implements ProcessTerminator {
|
||||
constructor(
|
||||
private readonly sendSignal: ProcessSignalSender = process.kill,
|
||||
) {}
|
||||
|
||||
async terminate(
|
||||
request: ProcessTerminationRequest,
|
||||
): Promise<ProcessTerminationResult> {
|
||||
const targetPid = request.processGroup ? -request.pid : request.pid;
|
||||
try {
|
||||
this.sendSignal(targetPid, 'SIGTERM');
|
||||
} catch (error) {
|
||||
if (isNoSuchProcessError(error)) {
|
||||
return {
|
||||
alreadyExited: true,
|
||||
termSignalSent: false,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
throw new ExecutorStopError(error);
|
||||
}
|
||||
|
||||
if (await exitsWithin(request.closed, request.graceMs)) {
|
||||
return {
|
||||
alreadyExited: false,
|
||||
termSignalSent: true,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
this.sendSignal(targetPid, 'SIGKILL');
|
||||
return {
|
||||
alreadyExited: false,
|
||||
termSignalSent: true,
|
||||
killSignalSent: true,
|
||||
};
|
||||
} catch (error) {
|
||||
if (isNoSuchProcessError(error)) {
|
||||
return {
|
||||
alreadyExited: false,
|
||||
termSignalSent: true,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
throw new ExecutorStopError(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user