feat(ql3): add profile-aware run log reads

This commit is contained in:
whyour
2026-08-12 01:43:14 +08:00
parent c699c32461
commit 308aa75d89
33 changed files with 2880 additions and 306 deletions
+8
View File
@@ -232,6 +232,9 @@
],
"bounded-run-step-list-projection": [
"dist/run/projection/boundedRunStepListProjection.d.ts"
],
"run-attempt-log-read": [
"dist/run/log-read/runAttemptLogRead.d.ts"
]
}
},
@@ -281,6 +284,11 @@
"require": "./dist/run/projection/boundedRunStepListProjection.js",
"default": "./dist/run/projection/boundedRunStepListProjection.js"
},
"./run-attempt-log-read": {
"types": "./dist/run/log-read/runAttemptLogRead.d.ts",
"require": "./dist/run/log-read/runAttemptLogRead.js",
"default": "./dist/run/log-read/runAttemptLogRead.js"
},
"./task-definition": {
"types": "./dist/task-definition/taskDefinition.d.ts",
"require": "./dist/task-definition/taskDefinition.js",
@@ -0,0 +1,343 @@
import { RUN_ATTEMPT_STATUSES, type RunAttemptStatus } from '../run';
import type { RunRepositoryReader } from '../runRepository';
export const MAX_RUN_ATTEMPT_LOG_READ_BYTES = 256 * 1024;
const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
const TERMINAL_ATTEMPT_STATUSES = new Set<RunAttemptStatus>([
'succeeded',
'failed',
'cancelled',
'timed_out',
'lost',
]);
export interface RunAttemptLogReadRange {
readonly offset: number;
readonly length: number;
}
export interface RunAttemptLogReadIdentity {
readonly projectId: string;
readonly runId: string;
readonly attemptId: string;
readonly logArtifactId: string;
}
export interface RunAttemptLogTruncationView {
readonly truncated: boolean | 'unknown';
readonly maximumBytes?: number;
readonly observedAtMs?: number;
}
export type RunAttemptLogRangeReadResult =
| Readonly<{ readonly status: 'missing' }>
| Readonly<{
readonly status: 'available';
readonly content: Uint8Array;
readonly start: number;
readonly endExclusive: number;
readonly totalBytes: number;
readonly nextOffset?: number;
readonly truncation: Readonly<RunAttemptLogTruncationView>;
}>;
export interface RunAttemptLogRangeReader {
read(
identity: Readonly<RunAttemptLogReadIdentity>,
range: Readonly<RunAttemptLogReadRange>,
signal?: AbortSignal,
): Promise<RunAttemptLogRangeReadResult>;
}
export interface RunAttemptLogReadRequest {
readonly projectId: string;
readonly runId: string;
readonly attemptId: string;
readonly range: Readonly<RunAttemptLogReadRange>;
readonly signal?: AbortSignal;
}
export type RunAttemptLogReadResult =
| Readonly<{ readonly status: 'not_found' }>
| Readonly<{
readonly status: 'pending';
readonly projectId: string;
readonly runId: string;
readonly attemptId: string;
readonly logArtifactId?: string;
}>
| (Readonly<RunAttemptLogReadIdentity> &
Readonly<{ readonly status: 'missing' }>)
| (Readonly<RunAttemptLogReadIdentity> &
Extract<RunAttemptLogRangeReadResult, { readonly status: 'available' }>);
export interface RunAttemptLogReadServiceOptions {
readonly executorType: 'local_process' | 'remote_worker';
readonly artifactIdPattern: RegExp;
readonly maximumReadBytes: number;
readonly activeMissingIsPending?: boolean;
}
export class InvalidRunAttemptLogReadError extends TypeError {
constructor(message: string) {
super(`Run Attempt log read is invalid: ${message}`);
this.name = 'InvalidRunAttemptLogReadError';
}
}
export class RunAttemptLogReadUnavailableError extends Error {
constructor(options?: ErrorOptions) {
super('Run Attempt log read is unavailable', options);
this.name = 'RunAttemptLogReadUnavailableError';
}
}
function canonicalId(name: string, value: unknown): string {
if (typeof value !== 'string' || !ID_PATTERN.test(value)) {
throw new InvalidRunAttemptLogReadError(`${name} is invalid`);
}
return value;
}
function exactKeys(
value: object,
required: readonly string[],
optional: readonly string[],
name: string,
): void {
const keys = Object.keys(value);
const allowed = new Set([...required, ...optional]);
if (
required.some((key) => !Object.hasOwn(value, key)) ||
keys.some((key) => !allowed.has(key))
) {
throw new InvalidRunAttemptLogReadError(`${name} shape is invalid`);
}
}
export function normalizeRunAttemptLogReadRange(
value: Readonly<RunAttemptLogReadRange>,
maximumReadBytes = MAX_RUN_ATTEMPT_LOG_READ_BYTES,
): Readonly<RunAttemptLogReadRange> {
if (
!Number.isSafeInteger(maximumReadBytes) ||
maximumReadBytes < 1 ||
maximumReadBytes > MAX_RUN_ATTEMPT_LOG_READ_BYTES
) {
throw new InvalidRunAttemptLogReadError('maximum read bytes is invalid');
}
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidRunAttemptLogReadError('range is invalid');
}
exactKeys(value, ['length', 'offset'], [], 'range');
if (!Number.isSafeInteger(value.offset) || value.offset < 0) {
throw new InvalidRunAttemptLogReadError('offset is invalid');
}
if (
!Number.isSafeInteger(value.length) ||
value.length < 1 ||
value.length > maximumReadBytes
) {
throw new InvalidRunAttemptLogReadError('length is invalid');
}
return Object.freeze({ offset: value.offset, length: value.length });
}
function prepareOptions(
options: RunAttemptLogReadServiceOptions,
): Readonly<RunAttemptLogReadServiceOptions> {
if (
!options ||
typeof options !== 'object' ||
Array.isArray(options) ||
(options.executorType !== 'local_process' &&
options.executorType !== 'remote_worker') ||
!(options.artifactIdPattern instanceof RegExp) ||
options.artifactIdPattern.global ||
options.artifactIdPattern.sticky ||
(options.activeMissingIsPending !== undefined &&
typeof options.activeMissingIsPending !== 'boolean')
) {
throw new InvalidRunAttemptLogReadError('service options are invalid');
}
exactKeys(
options,
['artifactIdPattern', 'executorType', 'maximumReadBytes'],
['activeMissingIsPending'],
'service options',
);
normalizeRunAttemptLogReadRange(
{ offset: 0, length: options.maximumReadBytes },
options.maximumReadBytes,
);
return Object.freeze({ ...options });
}
function truncation(
value: Readonly<RunAttemptLogTruncationView>,
): Readonly<RunAttemptLogTruncationView> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
(value.truncated !== true &&
value.truncated !== false &&
value.truncated !== 'unknown') ||
(value.maximumBytes !== undefined &&
(!Number.isSafeInteger(value.maximumBytes) || value.maximumBytes < 1)) ||
(value.observedAtMs !== undefined &&
(!Number.isSafeInteger(value.observedAtMs) || value.observedAtMs < 0)) ||
(value.truncated === 'unknown' &&
(value.maximumBytes !== undefined || value.observedAtMs !== undefined))
) {
throw new RunAttemptLogReadUnavailableError();
}
return Object.freeze({ ...value });
}
function available(
identity: Readonly<RunAttemptLogReadIdentity>,
range: Readonly<RunAttemptLogReadRange>,
result: Extract<
RunAttemptLogRangeReadResult,
{ readonly status: 'available' }
>,
): RunAttemptLogReadResult {
if (
!(result.content instanceof Uint8Array) ||
!Number.isSafeInteger(result.start) ||
!Number.isSafeInteger(result.endExclusive) ||
!Number.isSafeInteger(result.totalBytes) ||
result.start !== Math.min(range.offset, result.totalBytes) ||
result.endExclusive !== result.start + result.content.byteLength ||
result.endExclusive > result.totalBytes ||
result.content.byteLength > range.length ||
(result.nextOffset === undefined) !==
(result.endExclusive === result.totalBytes) ||
(result.nextOffset !== undefined &&
result.nextOffset !== result.endExclusive)
) {
throw new RunAttemptLogReadUnavailableError();
}
return Object.freeze({
status: 'available' as const,
...identity,
content: result.content,
start: result.start,
endExclusive: result.endExclusive,
totalBytes: result.totalBytes,
...(result.nextOffset === undefined
? {}
: { nextOffset: result.nextOffset }),
truncation: truncation(result.truncation),
});
}
export class RunAttemptLogReadService {
private readonly options: Readonly<RunAttemptLogReadServiceOptions>;
constructor(
private readonly runs: Pick<
RunRepositoryReader,
'findRunById' | 'findAttemptById'
>,
private readonly reader: RunAttemptLogRangeReader,
options: RunAttemptLogReadServiceOptions,
) {
if (
!runs ||
typeof runs.findRunById !== 'function' ||
typeof runs.findAttemptById !== 'function' ||
!reader ||
typeof reader.read !== 'function'
) {
throw new InvalidRunAttemptLogReadError('dependencies are invalid');
}
this.options = prepareOptions(options);
}
async read(
request: Readonly<RunAttemptLogReadRequest>,
): Promise<RunAttemptLogReadResult> {
if (!request || typeof request !== 'object' || Array.isArray(request)) {
throw new InvalidRunAttemptLogReadError('request is invalid');
}
exactKeys(
request,
['attemptId', 'projectId', 'range', 'runId'],
['signal'],
'request',
);
const projectId = canonicalId('projectId', request.projectId);
const runId = canonicalId('runId', request.runId);
const attemptId = canonicalId('attemptId', request.attemptId);
const range = normalizeRunAttemptLogReadRange(
request.range,
this.options.maximumReadBytes,
);
if (request.signal?.aborted) {
throw new RunAttemptLogReadUnavailableError({
cause: request.signal.reason,
});
}
try {
const run = await this.runs.findRunById(runId);
if (
!run ||
run.id !== runId ||
run.projectId !== projectId ||
run.executionOwner !== 'runtime'
) {
return Object.freeze({ status: 'not_found' as const });
}
const attempt = await this.runs.findAttemptById(attemptId);
if (
!attempt ||
attempt.id !== attemptId ||
attempt.runId !== runId ||
attempt.executorType !== this.options.executorType ||
!RUN_ATTEMPT_STATUSES.includes(attempt.status)
) {
return Object.freeze({ status: 'not_found' as const });
}
if (attempt.logArtifactId === undefined) {
return Object.freeze({
status: 'pending' as const,
projectId,
runId,
attemptId,
});
}
if (!this.options.artifactIdPattern.test(attempt.logArtifactId)) {
return Object.freeze({ status: 'not_found' as const });
}
const identity = Object.freeze({
projectId,
runId,
attemptId,
logArtifactId: attempt.logArtifactId,
});
const result = await this.reader.read(identity, range, request.signal);
if (result.status === 'missing') {
if (
this.options.activeMissingIsPending === true &&
!TERMINAL_ATTEMPT_STATUSES.has(attempt.status)
) {
return Object.freeze({ status: 'pending' as const, ...identity });
}
return Object.freeze({ status: 'missing' as const, ...identity });
}
return available(identity, range, result);
} catch (error) {
if (
error instanceof InvalidRunAttemptLogReadError ||
error instanceof RunAttemptLogReadUnavailableError
) {
throw error;
}
throw new RunAttemptLogReadUnavailableError({ cause: error });
}
}
}
@@ -0,0 +1,261 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
InvalidRunAttemptLogReadError,
MAX_RUN_ATTEMPT_LOG_READ_BYTES,
RunAttemptLogReadService,
RunAttemptLogReadUnavailableError,
normalizeRunAttemptLogReadRange,
} = require('../dist/run/log-read/runAttemptLogRead.js');
function run(overrides = {}) {
return {
id: 'run_123',
projectId: 'prj_default',
taskId: 'task_1',
taskRevision: 'revision_1',
triggerType: 'task_start',
executionOrigin: 'manual',
executionOwner: 'runtime',
status: 'running',
version: 2,
eventSequence: 2,
priority: 0,
createdAtMs: 1,
...overrides,
};
}
function attempt(overrides = {}) {
return {
id: 'attempt_123',
runId: 'run_123',
attempt: 1,
status: 'running',
executorType: 'local_process',
logArtifactId: `local-${'a'.repeat(30)}`,
callbackSequence: 0,
createdAtMs: 1,
...overrides,
};
}
function service(overrides = {}) {
const calls = [];
const runs = overrides.runs ?? {
async findRunById() {
return run();
},
async findAttemptById() {
return attempt();
},
};
const reader = overrides.reader ?? {
async read(identity, range, signal) {
calls.push({ identity, range, signal });
return {
status: 'available',
content: Buffer.from('log'),
start: range.offset,
endExclusive: range.offset + 3,
totalBytes: range.offset + 5,
nextOffset: range.offset + 3,
truncation: { truncated: false, maximumBytes: 1024, observedAtMs: 9 },
};
},
};
return {
calls,
value: new RunAttemptLogReadService(runs, reader, {
executorType: overrides.executorType ?? 'local_process',
artifactIdPattern: overrides.artifactIdPattern ?? /^local-[a-f0-9]{30}$/,
maximumReadBytes: overrides.maximumReadBytes ?? 32 * 1024,
...(overrides.activeMissingIsPending === undefined
? {}
: { activeMissingIsPending: overrides.activeMissingIsPending }),
}),
};
}
function request(overrides = {}) {
return {
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_123',
range: { offset: 4, length: 16 },
...overrides,
};
}
test('normalizes only bounded safe ranges', () => {
assert.deepEqual(normalizeRunAttemptLogReadRange({ offset: 0, length: 1 }), {
offset: 0,
length: 1,
});
assert.deepEqual(
normalizeRunAttemptLogReadRange({
offset: Number.MAX_SAFE_INTEGER,
length: MAX_RUN_ATTEMPT_LOG_READ_BYTES,
}),
{ offset: Number.MAX_SAFE_INTEGER, length: MAX_RUN_ATTEMPT_LOG_READ_BYTES },
);
for (const range of [
{ offset: -1, length: 1 },
{ offset: 0.5, length: 1 },
{ offset: 0, length: 0 },
{ offset: 0, length: MAX_RUN_ATTEMPT_LOG_READ_BYTES + 1 },
]) {
assert.throws(
() => normalizeRunAttemptLogReadRange(range),
InvalidRunAttemptLogReadError,
);
}
});
test('validates Project, Run, Attempt, owner and executor before storage access', async () => {
const cases = [
{ run: null },
{ run: run({ projectId: 'prj_other' }) },
{ run: run({ executionOwner: 'legacy' }) },
{ attempt: null },
{ attempt: attempt({ runId: 'run_other' }) },
{ attempt: attempt({ executorType: 'remote_worker' }) },
{ attempt: attempt({ logArtifactId: `wlog-${'a'.repeat(30)}` }) },
];
for (const values of cases) {
let reads = 0;
const { value } = service({
runs: {
async findRunById() {
return values.run === undefined ? run() : values.run;
},
async findAttemptById() {
return values.attempt === undefined ? attempt() : values.attempt;
},
},
reader: {
async read() {
reads += 1;
return { status: 'missing' };
},
},
});
assert.deepEqual(await value.read(request()), { status: 'not_found' });
assert.equal(reads, 0);
}
});
test('returns pending before Artifact binding and for active remote publication lag', async () => {
const unbound = service({
runs: {
async findRunById() {
return run();
},
async findAttemptById() {
return attempt({ logArtifactId: undefined });
},
},
});
assert.deepEqual(await unbound.value.read(request()), {
status: 'pending',
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_123',
});
assert.equal(unbound.calls.length, 0);
const remote = service({
executorType: 'remote_worker',
artifactIdPattern: /^wlog-[a-f0-9]{30}$/,
activeMissingIsPending: true,
runs: {
async findRunById() {
return run();
},
async findAttemptById() {
return attempt({
executorType: 'remote_worker',
logArtifactId: `wlog-${'b'.repeat(30)}`,
});
},
},
reader: {
async read() {
return { status: 'missing' };
},
},
});
assert.equal((await remote.value.read(request())).status, 'pending');
});
test('returns a validated bounded snapshot without copying storage bytes', async () => {
const content = Buffer.from('log');
const abort = new AbortController();
const { value, calls } = service({
reader: {
async read(identity, range, signal) {
assert.equal(signal, abort.signal);
return {
status: 'available',
content,
start: 4,
endExclusive: 7,
totalBytes: 9,
nextOffset: 7,
truncation: {
truncated: true,
maximumBytes: 64 * 1024,
observedAtMs: 10,
},
};
},
},
});
const result = await value.read(request({ signal: abort.signal }));
assert.equal(result.status, 'available');
assert.equal(result.content, content);
assert.equal(result.nextOffset, 7);
assert.deepEqual(result.truncation, {
truncated: true,
maximumBytes: 64 * 1024,
observedAtMs: 10,
});
assert.equal(calls.length, 0);
});
test('fails closed on malformed storage results and dependency failures', async () => {
const malformed = service({
reader: {
async read() {
return {
status: 'available',
content: Buffer.from('too-long'),
start: 4,
endExclusive: 12,
totalBytes: 9,
truncation: { truncated: 'unknown' },
};
},
},
});
await assert.rejects(
malformed.value.read(request()),
RunAttemptLogReadUnavailableError,
);
const failed = service({
runs: {
async findRunById() {
throw new Error('database detail');
},
async findAttemptById() {
return null;
},
},
});
await assert.rejects(
failed.value.read(request()),
RunAttemptLogReadUnavailableError,
);
});