feat(ql3): add profile-aware run log reads

This commit is contained in:
whyour
2026-08-12 01:43:14 +08:00
parent c699c32461
commit 308aa75d89
33 changed files with 2880 additions and 306 deletions
@@ -24,6 +24,7 @@ import { createClusterControlRunReadRoute } from '../run/runReadRoute';
import { createClusterControlRunListRoute } from '../run/runListRoute';
import { createClusterControlRunEventListRoute } from '../run/runEventListRoute';
import { createClusterControlRunStepListRoute } from '../run/runStepListRoute';
import { createClusterControlRunAttemptLogReadRoute } from '../run/runAttemptLogReadRoute';
import { createClusterControlTaskListRoute } from '../task/taskListRoute';
import { createClusterControlTaskReadRoute } from '../task/taskReadRoute';
import { createClusterControlTaskStartRoute } from '../task/taskStartRoute';
@@ -69,6 +70,7 @@ export const PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS = Object.freeze([
'run.list',
'run.events.list',
'run.steps.list',
'run.log.read',
'run.cancel',
'workflow.read',
'workflow.run.read',
@@ -195,6 +197,10 @@ export function createProductionClusterControlApplicationStack(
input.runs,
input.trustedToolStorage.stepRuns,
),
createClusterControlRunAttemptLogReadRoute(
input.runs,
input.workerRuntime?.runAttemptLogRead,
),
createClusterControlRunCancellationRoute(
input.runCancellation,
createEventId,
@@ -6,12 +6,19 @@ import {
ChecksumMode,
CopyObjectCommand,
DeleteObjectCommand,
GetObjectCommand,
HeadObjectCommand,
MetadataDirective,
PutObjectCommand,
S3Client,
ServerSideEncryption,
} from '@aws-sdk/client-s3';
import {
normalizeRunAttemptLogReadRange,
type RunAttemptLogRangeReadResult,
type RunAttemptLogReadIdentity,
type RunAttemptLogReadRange,
} from '@qinglong/runtime-core/run-attempt-log-read';
import {
MAX_REMOTE_WORKER_ARTIFACT_BYTES,
REMOTE_WORKER_ARTIFACT_CONTENT_TYPE,
@@ -31,7 +38,8 @@ const TEMPORARY_METADATA_SCHEMA =
const SHA256_PATTERN = /^[a-f0-9]{64}$/;
const BUCKET_PATTERN = /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/;
const PREFIX_PATTERN = /^[A-Za-z0-9][A-Za-z0-9/_=-]{0,254}$/;
const TEMPORARY_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
const TEMPORARY_ID_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
type S3SendClient = Pick<S3Client, 'send'>;
@@ -70,8 +78,7 @@ export function createS3ClusterRemoteWorkerArtifactClient(
typeof options !== 'object' ||
Array.isArray(options) ||
!/^[a-z0-9][a-z0-9-]{0,62}$/.test(options.region) ||
(options.endpoint !== undefined &&
typeof options.endpoint !== 'string') ||
(options.endpoint !== undefined && typeof options.endpoint !== 'string') ||
(options.forcePathStyle !== undefined &&
typeof options.forcePathStyle !== 'boolean')
) {
@@ -79,9 +86,7 @@ export function createS3ClusterRemoteWorkerArtifactClient(
}
return new S3Client({
region: options.region,
...(options.endpoint === undefined
? {}
: { endpoint: options.endpoint }),
...(options.endpoint === undefined ? {} : { endpoint: options.endpoint }),
forcePathStyle: options.forcePathStyle ?? false,
});
}
@@ -116,6 +121,11 @@ type ArtifactAuthority = Readonly<{
logArtifactId: string;
}>;
type StoredArtifactHead = Readonly<{
receipt: Readonly<RemoteWorkerArtifactReceipt>;
eTag?: string;
}>;
type NormalizedStorageCommand = ArtifactAuthority &
Readonly<{
byteLength: number;
@@ -127,7 +137,9 @@ const DIAGNOSTIC_CONTEXT = Object.freeze({
});
function configurationError(message: string): TypeError {
return new TypeError(`S3 Remote Worker Artifact store is invalid: ${message}`);
return new TypeError(
`S3 Remote Worker Artifact store is invalid: ${message}`,
);
}
function prepareOptions(
@@ -175,14 +187,15 @@ function prepareOptions(
throw configurationError('expected bucket owner is invalid');
}
const encryption = options.encryption;
if (!encryption || typeof encryption !== 'object' || Array.isArray(encryption)) {
if (
!encryption ||
typeof encryption !== 'object' ||
Array.isArray(encryption)
) {
throw configurationError('encryption is required');
}
let preparedEncryption: PreparedOptions['encryption'];
if (
encryption.mode === 's3' &&
Object.keys(encryption).length === 1
) {
if (encryption.mode === 's3' && Object.keys(encryption).length === 1) {
preparedEncryption = Object.freeze({
ServerSideEncryption: ServerSideEncryption.AES256,
});
@@ -383,9 +396,8 @@ function finalMetadata(
),
'ql3-byte-length': String(command.byteLength),
'ql3-content-sha256': sha256,
'ql3-truncated': command.truncated === undefined
? 'omitted'
: String(command.truncated),
'ql3-truncated':
command.truncated === undefined ? 'omitted' : String(command.truncated),
});
}
@@ -460,6 +472,99 @@ function parseStoredReceipt(
);
}
function canonicalETag(value: unknown): string {
if (
typeof value !== 'string' ||
value.length < 3 ||
value.length > 256 ||
!/^"[^"\u0000-\u001f\u007f]+"$/.test(value)
) {
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
}
return value;
}
function assertRangeMetadata(
authority: ArtifactAuthority,
receipt: Readonly<RemoteWorkerArtifactReceipt>,
output: Readonly<{
ContentLength?: number | undefined;
ContentRange?: string | undefined;
ContentType?: string | undefined;
ETag?: string | undefined;
Metadata?: Readonly<Record<string, string | undefined>> | undefined;
}>,
eTag: string,
start: number,
endExclusive: number,
): void {
const metadata = output.Metadata;
const truncated =
receipt.truncated === undefined ? 'omitted' : String(receipt.truncated);
if (
output.ContentLength !== endExclusive - start ||
output.ContentRange !==
`bytes ${start}-${endExclusive - 1}/${receipt.byteLength}` ||
output.ContentType !== REMOTE_WORKER_ARTIFACT_CONTENT_TYPE ||
canonicalETag(output.ETag) !== eTag ||
metadataValue(metadata, 'ql3-schema') !== METADATA_SCHEMA ||
metadataValue(metadata, 'ql3-project-sha256') !==
fieldDigest('project', authority.projectId) ||
metadataValue(metadata, 'ql3-run-sha256') !==
fieldDigest('run', authority.runId) ||
metadataValue(metadata, 'ql3-attempt-sha256') !==
fieldDigest('attempt', authority.attemptId) ||
metadataValue(metadata, 'ql3-log-artifact-sha256') !==
fieldDigest('log-artifact', authority.logArtifactId) ||
metadataValue(metadata, 'ql3-byte-length') !== String(receipt.byteLength) ||
metadataValue(metadata, 'ql3-content-sha256') !== receipt.sha256 ||
metadataValue(metadata, 'ql3-truncated') !== truncated
) {
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
}
}
async function readBoundedRangeBody(
body: unknown,
expectedBytes: number,
signal?: AbortSignal,
): Promise<Buffer> {
if (
!body ||
typeof body !== 'object' ||
!(Symbol.asyncIterator in body) ||
typeof (body as AsyncIterable<Uint8Array>)[Symbol.asyncIterator] !==
'function'
) {
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
}
const content = Buffer.allocUnsafe(expectedBytes);
let received = 0;
try {
for await (const chunk of body as AsyncIterable<Uint8Array>) {
if (signal?.aborted) throw signal.reason;
if (!(chunk instanceof Uint8Array) || chunk.byteLength === 0) {
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
}
if (received + chunk.byteLength > expectedBytes) {
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
}
Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength).copy(
content,
received,
);
received += chunk.byteLength;
}
if (received !== expectedBytes) {
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
}
return content;
} catch (error) {
content.fill(0);
throw error;
}
}
function isNotFound(error: unknown): boolean {
if (!error || typeof error !== 'object') return false;
const value = error as {
@@ -467,13 +572,17 @@ function isNotFound(error: unknown): boolean {
Code?: unknown;
$metadata?: { httpStatusCode?: unknown };
};
return value.name === 'NotFound' ||
return (
value.name === 'NotFound' ||
value.name === 'NoSuchKey' ||
value.Code === 'NoSuchKey' ||
value.$metadata?.httpStatusCode === 404;
value.$metadata?.httpStatusCode === 404
);
}
function requestOptions(signal?: AbortSignal): { abortSignal: AbortSignal } | undefined {
function requestOptions(
signal?: AbortSignal,
): { abortSignal: AbortSignal } | undefined {
return signal === undefined ? undefined : { abortSignal: signal };
}
@@ -549,7 +658,8 @@ class ArtifactContentDigest {
* objects are never overwritten or deleted by this adapter.
*/
export class S3ClusterRemoteWorkerArtifactStore
implements ClusterRemoteWorkerArtifactStore {
implements ClusterRemoteWorkerArtifactStore
{
private readonly options: PreparedOptions;
constructor(options: S3ClusterRemoteWorkerArtifactStoreOptions) {
@@ -561,6 +671,90 @@ export class S3ClusterRemoteWorkerArtifactStore
signal?: AbortSignal,
): Promise<Readonly<RemoteWorkerArtifactReceipt> | undefined> {
const authority = normalizeLookup(lookup);
return (await this.head(authority, signal))?.receipt;
}
async readLogRange(
rawIdentity: Readonly<RunAttemptLogReadIdentity>,
rawRange: Readonly<RunAttemptLogReadRange>,
signal?: AbortSignal,
): Promise<RunAttemptLogRangeReadResult> {
const authority = normalizeLookup(rawIdentity);
const range = normalizeRunAttemptLogReadRange(rawRange);
const stored = await this.head(authority, signal);
if (!stored) return Object.freeze({ status: 'missing' as const });
const start = Math.min(range.offset, stored.receipt.byteLength);
const endExclusive = Math.min(
start + range.length,
stored.receipt.byteLength,
);
const truncation = Object.freeze({
truncated: stored.receipt.truncated ?? ('unknown' as const),
});
if (start === endExclusive) {
return Object.freeze({
status: 'available' as const,
content: Buffer.alloc(0),
start,
endExclusive,
totalBytes: stored.receipt.byteLength,
truncation,
});
}
const eTag = canonicalETag(stored.eTag);
if (signal?.aborted) throw signal.reason;
try {
const output = await this.options.client.send(
new GetObjectCommand({
Bucket: this.options.bucket,
Key: finalObjectKey(this.options.prefix, authority),
IfMatch: eTag,
Range: `bytes=${start}-${endExclusive - 1}`,
...(this.options.expectedBucketOwner === undefined
? {}
: { ExpectedBucketOwner: this.options.expectedBucketOwner }),
}),
requestOptions(signal),
);
assertRangeMetadata(
authority,
stored.receipt,
output,
eTag,
start,
endExclusive,
);
const bytes = await readBoundedRangeBody(
output.Body,
endExclusive - start,
signal,
);
return Object.freeze({
status: 'available' as const,
content: bytes,
start,
endExclusive,
totalBytes: stored.receipt.byteLength,
...(endExclusive < stored.receipt.byteLength
? { nextOffset: endExclusive }
: {}),
truncation,
});
} catch (error) {
if (isNotFound(error)) {
return Object.freeze({ status: 'missing' as const });
}
if (error instanceof S3ClusterRemoteWorkerArtifactStoreError) throw error;
throw new S3ClusterRemoteWorkerArtifactStoreError('unavailable', {
cause: error,
});
}
}
private async head(
authority: ArtifactAuthority,
signal?: AbortSignal,
): Promise<StoredArtifactHead | undefined> {
if (signal?.aborted) throw signal.reason;
try {
const output = await this.options.client.send(
@@ -574,7 +768,10 @@ export class S3ClusterRemoteWorkerArtifactStore
}),
requestOptions(signal),
);
return parseStoredReceipt(authority, output);
return Object.freeze({
receipt: parseStoredReceipt(authority, output),
...(output.ETag === undefined ? {} : { eTag: output.ETag }),
});
} catch (error) {
if (isNotFound(error)) return undefined;
if (error instanceof S3ClusterRemoteWorkerArtifactStoreError) throw error;
@@ -604,9 +801,7 @@ export class S3ClusterRemoteWorkerArtifactStore
existing.truncated !== command.truncated ||
existing.sha256 !== incomingSha256
) {
throw new S3ClusterRemoteWorkerArtifactStoreError(
'integrity_mismatch',
);
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
}
return existing;
}
@@ -693,13 +888,11 @@ export class S3ClusterRemoteWorkerArtifactStore
stored.truncated !== command.truncated ||
stored.sha256 !== sha256
) {
throw new S3ClusterRemoteWorkerArtifactStoreError(
'integrity_mismatch',
);
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
}
result = Object.freeze({
...stored,
status: copied ? 'stored' as const : 'already_stored' as const,
status: copied ? ('stored' as const) : ('already_stored' as const),
});
} catch (error) {
primaryError = error;
@@ -770,9 +963,7 @@ export class S3ClusterRemoteWorkerArtifactStore
output.Metadata?.['ql3-owner-sha256'] !== ownerSha256 ||
canonicalChecksum(output.ChecksumSHA256) !== sha256
) {
throw new S3ClusterRemoteWorkerArtifactStoreError(
'integrity_mismatch',
);
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
}
}
}
@@ -16,6 +16,11 @@ import {
type RemoteWorkerCompletionRepository,
type RemoteWorkerCompletionResult,
} from '@qinglong/runtime-core/remote-worker-completion';
import type {
RunAttemptLogRangeReadResult,
RunAttemptLogReadIdentity,
RunAttemptLogReadRange,
} from '@qinglong/runtime-core/run-attempt-log-read';
export interface ClusterRemoteWorkerArtifactStorageCommand {
readonly projectId: string;
@@ -47,6 +52,12 @@ export interface ClusterRemoteWorkerArtifactStore {
lookup: Readonly<ClusterRemoteWorkerArtifactLookup>,
signal?: AbortSignal,
): Promise<Readonly<RemoteWorkerArtifactReceipt> | undefined>;
/** Optional during Alpha so upload-only test and alternate stores remain compatible. */
readLogRange?(
identity: Readonly<RunAttemptLogReadIdentity>,
range: Readonly<RunAttemptLogReadRange>,
signal?: AbortSignal,
): Promise<RunAttemptLogRangeReadResult>;
}
export interface ClusterRemoteWorkerArtifactUploadInput {
@@ -353,11 +364,13 @@ export class ClusterRemoteWorkerCompletionService {
}
try {
const result = normalizeRemoteWorkerCompletionResult(
await this.repository.complete(Object.freeze({
...command,
attemptEventId: eventId(this.createEventId),
runEventId: eventId(this.createEventId),
})),
await this.repository.complete(
Object.freeze({
...command,
attemptEventId: eventId(this.createEventId),
runEventId: eventId(this.createEventId),
}),
),
);
if (
result.runId !== command.runId ||
@@ -1,6 +1,7 @@
// Remote execution owns the least-privilege assembly of Worker-facing runtime capabilities.
import type { PostgresPool } from '@qinglong/runtime-core';
import type { RemoteWorkerSecretValueProvider } from '@qinglong/runtime-core/remote-secret-delivery';
import type { RunAttemptLogRangeReader } from '@qinglong/runtime-core/run-attempt-log-read';
import {
PostgresClusterDispatchSource,
PostgresRemoteRunActivationRepository,
@@ -11,23 +12,15 @@ import {
PostgresTaskExecutionRevisionSource,
PostgresWorkerSessionRepository,
} from '@qinglong/cluster-postgres/runtime';
import {
ClusterRemoteWorkerOfferClaimService,
} from './remoteWorkerDispatcher';
import {
ClusterRemoteRunActivationService,
} from './remoteRunActivationService';
import {
ClusterRemoteWorkerSecretDeliveryService,
} from './remoteWorkerSecretDeliveryService';
import { ClusterRemoteWorkerOfferClaimService } from './remoteWorkerDispatcher';
import { ClusterRemoteRunActivationService } from './remoteRunActivationService';
import { ClusterRemoteWorkerSecretDeliveryService } from './remoteWorkerSecretDeliveryService';
import {
ClusterRemoteWorkerArtifactService,
ClusterRemoteWorkerCompletionService,
type ClusterRemoteWorkerArtifactStore,
} from './remoteWorkerCompletionService';
import {
ClusterRemoteWorkerLeaseControlService,
} from './remoteWorkerLeaseControlService';
import { ClusterRemoteWorkerLeaseControlService } from './remoteWorkerLeaseControlService';
import type { WorkerIngressPipelineOptions } from '../worker-ingress/workerIngressPipeline';
export interface ClusterWorkerRuntimeDependencies {
@@ -49,6 +42,7 @@ export interface ClusterWorkerRuntimePort {
readonly leaseControl: NonNullable<
WorkerIngressPipelineOptions['leaseControl']
>;
readonly runAttemptLogRead?: RunAttemptLogRangeReader;
}
export function createClusterWorkerRuntimePort(
@@ -67,9 +61,11 @@ export function createClusterWorkerRuntimePort(
}
const workerSessions = new PostgresWorkerSessionRepository(pool);
const completionRepository =
new PostgresRemoteWorkerCompletionRepository(pool);
const completionRepository = new PostgresRemoteWorkerCompletionRepository(
pool,
);
const secretProvider = dependencies.secretProvider;
const readLogRange = dependencies.artifactStore.readLogRange;
return Object.freeze({
offers: new ClusterRemoteWorkerOfferClaimService(
new PostgresClusterDispatchSource(pool),
@@ -99,5 +95,12 @@ export function createClusterWorkerRuntimePort(
leaseControl: new ClusterRemoteWorkerLeaseControlService(
new PostgresRemoteWorkerLeaseControlRepository(pool),
),
...(readLogRange === undefined
? {}
: {
runAttemptLogRead: Object.freeze({
read: readLogRange.bind(dependencies.artifactStore),
}),
}),
});
}
@@ -0,0 +1,176 @@
import {
InvalidRunAttemptLogReadError,
RunAttemptLogReadService,
RunAttemptLogReadUnavailableError,
type RunAttemptLogRangeReader,
type RunAttemptLogReadResult,
} from '@qinglong/runtime-core/run-attempt-log-read';
import type { RunRepositoryReader } from '@qinglong/runtime-core/run-repository';
import type { ClusterControlAdmissionResponse } from '../transport/httpSurface';
import type {
ClusterControlAuthorizedOperationRequest,
ClusterControlRouteDefinition,
ClusterControlRouteParameters,
} from '../transport/routeRegistry';
export const CLUSTER_CONTROL_RUN_ATTEMPT_LOG_READ_ROUTE = Object.freeze({
method: 'GET' as const,
path: '/api/v3/projects/{projectId}/runs/{runId}/attempts/{attemptId}/log',
operationId: 'run.log.read',
permission: 'artifact.read',
projectParameter: 'projectId',
allowedQuery: Object.freeze(['length', 'offset']),
});
const DEFAULT_READ_BYTES = 64 * 1024;
const MAXIMUM_READ_BYTES = 256 * 1024;
function response(
statusCode: number,
body: Readonly<Record<string, unknown>>,
): ClusterControlAdmissionResponse {
return Object.freeze({ statusCode, body: Object.freeze(body) });
}
function parseQuery(
query: Readonly<Record<string, readonly string[]>>,
): Readonly<{ offset: number; length: number }> {
const offsetValues = query.offset;
const lengthValues = query.length;
if (
(offsetValues !== undefined && offsetValues.length !== 1) ||
(lengthValues !== undefined && lengthValues.length !== 1)
) {
throw new TypeError();
}
const rawOffset = offsetValues?.[0];
const offset = rawOffset === undefined ? 0 : Number(rawOffset);
const rawLength = lengthValues?.[0];
const length =
rawLength === undefined ? DEFAULT_READ_BYTES : Number(rawLength);
if (
!Number.isSafeInteger(offset) ||
offset < 0 ||
(rawOffset !== undefined && String(offset) !== rawOffset) ||
!Number.isSafeInteger(length) ||
length < 1 ||
length > MAXIMUM_READ_BYTES ||
(rawLength !== undefined && String(length) !== rawLength)
) {
throw new TypeError();
}
return Object.freeze({ offset, length });
}
function validateQuery(
query: Readonly<Record<string, readonly string[]>>,
): void {
parseQuery(query);
}
function projection(
result: Extract<RunAttemptLogReadResult, { readonly status: 'available' }>,
): Readonly<Record<string, unknown>> {
return Object.freeze({
schema: 'qinglong/run-attempt-log-read-result@v1',
status: 'available',
projectId: result.projectId,
runId: result.runId,
attemptId: result.attemptId,
range: Object.freeze({
start: result.start,
endExclusive: result.endExclusive,
totalBytes: result.totalBytes,
...(result.nextOffset === undefined
? {}
: { nextOffset: result.nextOffset }),
}),
encoding: 'base64',
content: Buffer.from(
result.content.buffer,
result.content.byteOffset,
result.content.byteLength,
).toString('base64'),
truncation: result.truncation,
});
}
export function createClusterControlRunAttemptLogReadRoute(
runs: Pick<RunRepositoryReader, 'findRunById' | 'findAttemptById'>,
reader?: RunAttemptLogRangeReader,
): Readonly<ClusterControlRouteDefinition> {
if (
!runs ||
typeof runs.findRunById !== 'function' ||
typeof runs.findAttemptById !== 'function' ||
(reader !== undefined && typeof reader.read !== 'function')
) {
throw new TypeError(
'Cluster-control Run Attempt log read dependencies are invalid',
);
}
const service =
reader === undefined
? undefined
: new RunAttemptLogReadService(runs, reader, {
executorType: 'remote_worker',
artifactIdPattern: /^wlog-[a-f0-9]{30}$/,
maximumReadBytes: MAXIMUM_READ_BYTES,
activeMissingIsPending: true,
});
return Object.freeze({
...CLUSTER_CONTROL_RUN_ATTEMPT_LOG_READ_ROUTE,
validateQuery,
async handle(
authorized: ClusterControlAuthorizedOperationRequest,
parameters: ClusterControlRouteParameters,
) {
if (authorized.request.body !== null) {
return response(400, { code: 'invalid_request_body' });
}
if (authorized.projectId === null || service === undefined) {
return response(503, { code: 'artifact_unavailable' });
}
let range;
try {
range = parseQuery(authorized.request.query);
} catch {
return response(400, { code: 'invalid_run_log_read_query' });
}
try {
const result = await service.read({
projectId: authorized.projectId,
runId: parameters.runId!,
attemptId: parameters.attemptId!,
range,
signal: authorized.request.signal,
});
if (result.status === 'not_found') {
return response(404, { code: 'artifact_not_found' });
}
if (result.status === 'pending') {
return response(202, {
schema: 'qinglong/run-attempt-log-read-result@v1',
status: 'pending',
projectId: result.projectId,
runId: result.runId,
attemptId: result.attemptId,
});
}
if (result.status === 'missing') {
return response(503, { code: 'artifact_unavailable' });
}
return response(200, projection(result));
} catch (error) {
if (error instanceof InvalidRunAttemptLogReadError) {
return response(400, { code: 'invalid_run_log_read_request' });
}
if (error instanceof RunAttemptLogReadUnavailableError) {
return response(503, { code: 'artifact_unavailable' });
}
return response(503, { code: 'artifact_unavailable' });
}
},
});
}
@@ -308,6 +308,13 @@ export function createClusterControlAdmissionPipeline(
now,
);
if (decision.effect === 'deny') {
if (route.permission === 'artifact.read') {
throw securityError(
404,
'artifact_not_found',
'Cluster-control Artifact is not available',
);
}
throw securityError(
403,
'forbidden',
@@ -315,6 +322,13 @@ export function createClusterControlAdmissionPipeline(
);
}
if (decision.effect === 'require_approval') {
if (route.permission === 'artifact.read') {
throw securityError(
404,
'artifact_not_found',
'Cluster-control Artifact is not available',
);
}
throw securityError(
403,
'approval_required',
@@ -573,6 +573,9 @@ test('injects reviewed Worker operations without exposing the runtime Pool', asy
async inspect() {
throw new Error('not invoked during assembly');
},
async readLogRange() {
throw new Error('not invoked during assembly');
},
};
const result = await bootstrapClusterControlRuntime(
bootstrapOptions(events, {
@@ -596,6 +599,10 @@ test('injects reviewed Worker operations without exposing the runtime Pool', asy
typeof input.workerRuntime.leaseControl.control,
'function',
);
assert.equal(
typeof input.workerRuntime.runAttemptLogRead.read,
'function',
);
return activationStack(events);
},
}),
@@ -143,6 +143,9 @@ function fixture(overrides = {}) {
},
];
},
async findAttemptById() {
return null;
},
},
trustedToolStorage: {
stepRuns: {
@@ -190,7 +193,8 @@ function fixture(overrides = {}) {
taskDefinitions: {
async findCurrentTaskDefinition(projectId, taskId) {
events.push(`task-get:${projectId}:${taskId}`);
return projectId === currentTask.projectId && taskId === currentTask.taskId
return projectId === currentTask.projectId &&
taskId === currentTask.taskId
? currentTask
: null;
},
@@ -328,6 +332,7 @@ test('production composition exposes the reviewed Run and Workflow routes', asyn
'run.list',
'run.events.list',
'run.steps.list',
'run.log.read',
'run.cancel',
'workflow.read',
'workflow.run.read',
@@ -401,6 +406,15 @@ test('production composition exposes the reviewed Run and Workflow routes', asyn
body: { steps: [], hasMore: false, next: null },
});
const log = await invoke(
stack,
metadata('/api/v3/projects/project-1/runs/run-1/attempts/attempt-1/log'),
);
assert.deepEqual(log, {
statusCode: 503,
body: { code: 'artifact_unavailable' },
});
const cancellation = await invoke(
stack,
metadata('/api/v3/projects/project-1/runs/run-1/cancellation', 'POST', {
@@ -414,6 +428,7 @@ test('production composition exposes the reviewed Run and Workflow routes', asyn
assert.equal(events.includes('audit:run.get:allowed'), true);
assert.equal(events.includes('audit:run.events.list:allowed'), true);
assert.equal(events.includes('audit:run.steps.list:allowed'), true);
assert.equal(events.includes('audit:run.log.read:allowed'), true);
assert.equal(events.includes('audit:run.cancel:allowed'), true);
const workflows = await invoke(
@@ -539,6 +554,69 @@ test('production composition fails closed for an unreviewed route', async () =>
);
});
test('wires the production Worker object reader into the Project-scoped log route', async () => {
const { input } = fixture();
const run = await input.runs.findRunById('run-1');
const logArtifactId = `wlog-${'a'.repeat(30)}`;
const stack = createProductionClusterControlApplicationStack({
...input,
runs: {
...input.runs,
async findRunById() {
return { ...run, status: 'running' };
},
async findAttemptById() {
return {
id: 'attempt-1',
runId: 'run-1',
attempt: 1,
status: 'running',
executorType: 'remote_worker',
logArtifactId,
callbackSequence: 0,
createdAtMs: 1,
};
},
},
workerRuntime: {
offers: { claimNext() {} },
activation: {
acknowledgeStarting() {},
acknowledgeRunning() {},
failStart() {},
},
artifacts: { upload() {} },
completion: { complete() {} },
leaseControl: { control() {} },
runAttemptLogRead: {
async read(identity, range) {
assert.equal(identity.logArtifactId, logArtifactId);
assert.deepEqual(range, { offset: 1, length: 4 });
return {
status: 'available',
content: Buffer.from('prod'),
start: 1,
endExclusive: 5,
totalBytes: 5,
truncation: { truncated: false },
};
},
},
},
});
const result = await invoke(
stack,
metadata(
'/api/v3/projects/project-1/runs/run-1/attempts/attempt-1/log',
'GET',
null,
{ offset: ['1'], length: ['4'] },
),
);
assert.equal(result.statusCode, 200);
assert.equal(Buffer.from(result.body.content, 'base64').toString(), 'prod');
});
test('optionally exposes Prompt execution behind shared admission and policy', async () => {
const { events, input } = fixture();
let command;
@@ -0,0 +1,251 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
createClusterControlAdmissionPipeline,
} = require('@qinglong/cluster-control/admission');
const {
createClusterControlRouteRegistry,
} = require('@qinglong/cluster-control/routes');
const {
CLUSTER_CONTROL_RUN_ATTEMPT_LOG_READ_ROUTE,
createClusterControlRunAttemptLogReadRoute,
} = require('../dist/run/runAttemptLogReadRoute.js');
const PRINCIPAL = Object.freeze({
subject: Object.freeze({ type: 'user', id: 'usr_viewer' }),
authenticationId: 'session:viewer',
authenticatedAtMs: 9_000,
expiresAtMs: 11_000,
assurance: 'single_factor',
});
function run(overrides = {}) {
return {
id: 'run_123',
projectId: 'prj_default',
taskId: 'task_1',
taskRevision: 'revision_1',
triggerType: 'task_start',
executionOrigin: 'manual',
executionOwner: 'runtime',
status: 'running',
version: 2,
eventSequence: 2,
priority: 0,
createdAtMs: 1,
...overrides,
};
}
function attempt(overrides = {}) {
return {
id: 'attempt_123',
runId: 'run_123',
attempt: 1,
status: 'running',
executorType: 'remote_worker',
logArtifactId: `wlog-${'a'.repeat(30)}`,
callbackSequence: 0,
createdAtMs: 1,
...overrides,
};
}
function metadata(query = {}) {
return Object.freeze({
requestId: 'request-log-read',
method: 'GET',
path: '/api/v3/projects/prj_default/runs/run_123/attempts/attempt_123/log',
query: Object.freeze(query),
headers: Object.freeze({ authorization: 'Bearer opaque' }),
signal: new AbortController().signal,
});
}
function pipeline(options = {}) {
const events = options.events ?? [];
const repository = options.repository ?? {
async findRunById() {
events.push('run');
return run();
},
async findAttemptById() {
events.push('attempt');
return attempt();
},
};
const reader = options.reader ?? {
async read(identity, range) {
events.push(`storage:${range.offset}:${range.length}`);
return {
status: 'available',
content: Buffer.from('cluster-log'),
start: range.offset,
endExclusive: range.offset + 11,
totalBytes: range.offset + 20,
nextOffset: range.offset + 11,
truncation: { truncated: false },
};
},
};
return createClusterControlAdmissionPipeline({
routes: createClusterControlRouteRegistry([
createClusterControlRunAttemptLogReadRoute(repository, reader),
]),
authenticator: {
authenticate() {
events.push('authenticate');
return PRINCIPAL;
},
},
policy: {
authorize(request) {
events.push(`authorize:${request.permission}`);
return options.effect
? { effect: options.effect, reasons: ['masked'], fence: null }
: {
effect: 'allow',
reasons: ['role_grant'],
fence: { projectVersion: 2, bindingVersion: 3 },
};
},
},
audit: {
record(record) {
events.push(`audit:${record.outcome}`);
},
},
now: () => 10_000,
});
}
async function invoke(value, query = {}) {
const prepared = await value.prepare(metadata(query));
return prepared.handle(null);
}
test('publishes the immutable Artifact-scoped route contract', () => {
assert.deepEqual(CLUSTER_CONTROL_RUN_ATTEMPT_LOG_READ_ROUTE, {
method: 'GET',
path: '/api/v3/projects/{projectId}/runs/{runId}/attempts/{attemptId}/log',
operationId: 'run.log.read',
permission: 'artifact.read',
projectParameter: 'projectId',
allowedQuery: ['length', 'offset'],
});
});
test('authorizes and audits before metadata and one bounded range read', async () => {
const events = [];
const result = await invoke(pipeline({ events }), {
offset: ['4'],
length: ['16'],
});
assert.equal(result.statusCode, 200);
assert.equal(result.body.schema, 'qinglong/run-attempt-log-read-result@v1');
assert.equal(
Buffer.from(result.body.content, 'base64').toString(),
'cluster-log',
);
assert.deepEqual(result.body.range, {
start: 4,
endExclusive: 15,
totalBytes: 24,
nextOffset: 15,
});
assert.deepEqual(events, [
'authenticate',
'authorize:artifact.read',
'audit:allowed',
'run',
'attempt',
'storage:4:16',
]);
});
test('uses the Cluster default window and rejects unbounded query values', async () => {
const events = [];
assert.equal((await invoke(pipeline({ events }))).statusCode, 200);
assert.equal(events.at(-1), `storage:0:${64 * 1024}`);
for (const query of [
{ offset: ['-1'] },
{ offset: ['04'] },
{ length: ['0'] },
{ length: [String(256 * 1024 + 1)] },
{ length: ['1', '2'] },
]) {
await assert.rejects(
pipeline().prepare(metadata(query)),
(error) =>
error.statusCode === 400 && error.code === 'invalid_route_query',
);
}
});
test('masks deny and approval without reading Run or object storage', async () => {
for (const effect of ['deny', 'require_approval']) {
let touched = false;
await assert.rejects(
pipeline({
effect,
repository: {
async findRunById() {
touched = true;
return run();
},
async findAttemptById() {
touched = true;
return attempt();
},
},
reader: {
async read() {
touched = true;
return { status: 'missing' };
},
},
}).prepare(metadata()),
(error) =>
error.statusCode === 404 && error.code === 'artifact_not_found',
);
assert.equal(touched, false);
}
});
test('returns pending during upload and fails closed without an object reader', async () => {
const pending = pipeline({
reader: {
async read() {
return { status: 'missing' };
},
},
});
assert.equal((await invoke(pending)).statusCode, 202);
const unavailableRoute = createClusterControlRunAttemptLogReadRoute({
async findRunById() {
return run();
},
async findAttemptById() {
return attempt();
},
});
const prepared = await createClusterControlAdmissionPipeline({
routes: createClusterControlRouteRegistry([unavailableRoute]),
authenticator: { authenticate: () => PRINCIPAL },
policy: {
authorize: () => ({
effect: 'allow',
reasons: ['role_grant'],
fence: { projectVersion: 1, bindingVersion: 1 },
}),
},
audit: { record() {} },
now: () => 10_000,
}).prepare(metadata());
assert.deepEqual(await prepared.handle(null), {
statusCode: 503,
body: { code: 'artifact_unavailable' },
});
});
@@ -19,80 +19,111 @@ const endpoint = process.env.QL3_TEST_S3_ENDPOINT;
const accessKeyId = process.env.QL3_TEST_S3_ACCESS_KEY_ID;
const secretAccessKey = process.env.QL3_TEST_S3_SECRET_ACCESS_KEY;
test('real S3-compatible service preserves immutable Artifact evidence', {
skip: endpoint && accessKeyId && secretAccessKey
? false
: 'requires QL3_TEST_S3_ENDPOINT and credentials',
}, async () => {
const client = new S3Client({
endpoint,
region: 'us-east-1',
forcePathStyle: true,
credentials: { accessKeyId, secretAccessKey },
});
const bucket = `ql3-artifact-${process.pid}-${Date.now()}`.slice(0, 63);
const command = Object.freeze({
projectId: 'project-s3-integration',
runId: 'run-s3-integration',
attemptId: 'attempt-s3-integration',
logArtifactId: `wlog-${'c'.repeat(30)}`,
byteLength: 17,
truncated: true,
});
const content = Buffer.from('real object bytes');
const body = (value) => Object.freeze({
async *[Symbol.asyncIterator]() {
yield value.subarray(0, 4);
yield value.subarray(4);
},
});
try {
await client.send(new CreateBucketCommand({ Bucket: bucket }));
const store = new S3ClusterRemoteWorkerArtifactStore({
client,
bucket,
prefix: 'qinglong/integration',
encryption: { mode: 's3' },
test(
'real S3-compatible service preserves immutable Artifact evidence',
{
skip:
endpoint && accessKeyId && secretAccessKey
? false
: 'requires QL3_TEST_S3_ENDPOINT and credentials',
},
async () => {
const client = new S3Client({
endpoint,
region: 'us-east-1',
forcePathStyle: true,
credentials: { accessKeyId, secretAccessKey },
});
const stored = await store.put(command, body(content));
assert.equal(stored.status, 'stored');
assert.equal(
stored.sha256,
createHash('sha256').update(content).digest('hex'),
);
const replay = await store.put(command, body(content));
assert.equal(replay.status, 'already_stored');
await assert.rejects(
store.put(command, body(Buffer.from('REAL OBJECT BYTES'))),
(error) =>
error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
error.reason === 'integrity_mismatch',
);
const objects = await client.send(new ListObjectsV2Command({
Bucket: bucket,
Prefix: 'qinglong/integration/',
}));
assert.equal(objects.KeyCount, 1);
assert.match(objects.Contents[0].Key, /\/objects\//);
} finally {
const bucket = `ql3-artifact-${process.pid}-${Date.now()}`.slice(0, 63);
const command = Object.freeze({
projectId: 'project-s3-integration',
runId: 'run-s3-integration',
attemptId: 'attempt-s3-integration',
logArtifactId: `wlog-${'c'.repeat(30)}`,
byteLength: 17,
truncated: true,
});
const content = Buffer.from('real object bytes');
const body = (value) =>
Object.freeze({
async *[Symbol.asyncIterator]() {
yield value.subarray(0, 4);
yield value.subarray(4);
},
});
try {
const objects = await client.send(new ListObjectsV2Command({
Bucket: bucket,
}));
if (objects.Contents?.length) {
await client.send(new DeleteObjectsCommand({
await client.send(new CreateBucketCommand({ Bucket: bucket }));
const store = new S3ClusterRemoteWorkerArtifactStore({
client,
bucket,
prefix: 'qinglong/integration',
encryption: { mode: 's3' },
});
const stored = await store.put(command, body(content));
assert.equal(stored.status, 'stored');
assert.equal(
stored.sha256,
createHash('sha256').update(content).digest('hex'),
);
const replay = await store.put(command, body(content));
assert.equal(replay.status, 'already_stored');
const range = await store.readLogRange(
{
projectId: command.projectId,
runId: command.runId,
attemptId: command.attemptId,
logArtifactId: command.logArtifactId,
},
{
offset: 5,
length: 6,
},
);
assert.equal(range.status, 'available');
assert.equal(Buffer.from(range.content).toString(), 'object');
assert.equal(range.start, 5);
assert.equal(range.endExclusive, 11);
assert.equal(range.totalBytes, content.byteLength);
assert.equal(range.nextOffset, 11);
assert.deepEqual(range.truncation, { truncated: true });
await assert.rejects(
store.put(command, body(Buffer.from('REAL OBJECT BYTES'))),
(error) =>
error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
error.reason === 'integrity_mismatch',
);
const objects = await client.send(
new ListObjectsV2Command({
Bucket: bucket,
Delete: {
Objects: objects.Contents.map(({ Key }) => ({ Key })),
Quiet: true,
},
}));
Prefix: 'qinglong/integration/',
}),
);
assert.equal(objects.KeyCount, 1);
assert.match(objects.Contents[0].Key, /\/objects\//);
} finally {
try {
const objects = await client.send(
new ListObjectsV2Command({
Bucket: bucket,
}),
);
if (objects.Contents?.length) {
await client.send(
new DeleteObjectsCommand({
Bucket: bucket,
Delete: {
Objects: objects.Contents.map(({ Key }) => ({ Key })),
Quiet: true,
},
}),
);
}
await client.send(new DeleteBucketCommand({ Bucket: bucket }));
} catch {
// Preserve the integration assertion; the ephemeral container is removed.
}
await client.send(new DeleteBucketCommand({ Bucket: bucket }));
} catch {
// Preserve the integration assertion; the ephemeral container is removed.
client.destroy();
}
client.destroy();
}
});
},
);
@@ -6,6 +6,7 @@ const { test } = require('node:test');
const {
CopyObjectCommand,
DeleteObjectCommand,
GetObjectCommand,
HeadObjectCommand,
PutObjectCommand,
} = require('@aws-sdk/client-s3');
@@ -57,19 +58,58 @@ class MemoryS3Client {
const object = this.objects.get(input.Key);
if (!object) throw notFound();
const metadata = { ...object.metadata };
if (this.options.corruptFinalMetadata && input.Key.includes('/objects/')) {
if (
this.options.corruptFinalMetadata &&
input.Key.includes('/objects/')
) {
metadata['ql3-content-sha256'] = '0'.repeat(64);
}
return {
ContentLength: object.content.byteLength,
ContentType: object.contentType,
ChecksumSHA256: this.options.corruptFinalChecksum &&
input.Key.includes('/objects/')
? Buffer.alloc(32, 9).toString('base64')
: checksum(object.content),
ETag: `"${checksum(object.content).slice(0, 32)}"`,
ChecksumSHA256:
this.options.corruptFinalChecksum && input.Key.includes('/objects/')
? Buffer.alloc(32, 9).toString('base64')
: checksum(object.content),
Metadata: metadata,
};
}
if (command instanceof GetObjectCommand) {
const object = this.objects.get(input.Key);
if (!object || this.options.rangeNotFound) throw notFound();
const eTag = `"${checksum(object.content).slice(0, 32)}"`;
assert.equal(input.IfMatch, eTag);
const match = /^bytes=(\d+)-(\d+)$/.exec(input.Range);
assert.ok(match);
const start = Number(match[1]);
const end = Number(match[2]);
let content = object.content.subarray(start, end + 1);
if (this.options.shortRangeBody) content = content.subarray(0, -1);
if (this.options.oversizedRangeBody) {
content = Buffer.concat([content, Buffer.from('x')]);
}
const metadata = { ...object.metadata };
if (this.options.corruptRangeMetadata) {
metadata['ql3-run-sha256'] = '0'.repeat(64);
}
return {
ContentLength: end - start + 1,
ContentRange: this.options.corruptContentRange
? `bytes ${start}-${end}/${object.content.byteLength + 1}`
: `bytes ${start}-${end}/${object.content.byteLength}`,
ContentType: object.contentType,
ETag: this.options.corruptRangeETag ? '"other"' : eTag,
Metadata: metadata,
Body: {
async *[Symbol.asyncIterator]() {
const split = Math.min(2, content.byteLength);
if (split > 0) yield content.subarray(0, split);
if (split < content.byteLength) yield content.subarray(split);
},
},
};
}
if (command instanceof PutObjectCommand) {
assert.equal(input.IfNoneMatch, '*');
assert.equal(input.ChecksumAlgorithm, 'SHA256');
@@ -183,14 +223,25 @@ test('streams to a checksummed temporary object then conditionally promotes it',
],
);
const key = permanentKey(client);
assert.match(key, /^tenant-a\/worker-artifacts\/objects\/[a-f0-9]{2}\/[a-f0-9]{64}$/);
assert.match(
key,
/^tenant-a\/worker-artifacts\/objects\/[a-f0-9]{2}\/[a-f0-9]{64}$/,
);
assert.equal(key.includes(COMMAND.runId), false);
assert.equal(client.objects.size, 1);
const copy = client.commands.find((command) => command instanceof CopyObjectCommand);
const copy = client.commands.find(
(command) => command instanceof CopyObjectCommand,
);
assert.equal(copy.input.Metadata['ql3-content-sha256'], CONTENT_SHA256);
assert.equal(JSON.stringify(copy.input.Metadata).includes(COMMAND.projectId), false);
assert.equal(JSON.stringify(copy.input.Metadata).includes(COMMAND.runId), false);
assert.equal(
JSON.stringify(copy.input.Metadata).includes(COMMAND.projectId),
false,
);
assert.equal(
JSON.stringify(copy.input.Metadata).includes(COMMAND.runId),
false,
);
const inspected = await adapter.inspect(LOOKUP);
assert.deepEqual(inspected, { ...receipt, status: 'already_stored' });
@@ -219,6 +270,78 @@ test('exact replay consumes and hashes the whole body without another write', as
);
});
test('reads only one ETag-fenced immutable byte range and stable end snapshot', async () => {
const client = new MemoryS3Client();
const adapter = store(client);
await adapter.put(COMMAND, chunks());
client.commands.length = 0;
const result = await adapter.readLogRange(LOOKUP, { offset: 2, length: 4 });
assert.equal(result.status, 'available');
assert.equal(Buffer.from(result.content).toString(), 'llo ');
assert.deepEqual(
{
start: result.start,
endExclusive: result.endExclusive,
totalBytes: result.totalBytes,
nextOffset: result.nextOffset,
truncation: result.truncation,
},
{
start: 2,
endExclusive: 6,
totalBytes: 11,
nextOffset: 6,
truncation: { truncated: false },
},
);
assert.deepEqual(
client.commands.map((command) => command.constructor.name),
['HeadObjectCommand', 'GetObjectCommand'],
);
assert.equal(client.commands[1].input.Range, 'bytes=2-5');
client.commands.length = 0;
const ended = await adapter.readLogRange(LOOKUP, {
offset: 999,
length: 4,
});
assert.equal(ended.status, 'available');
assert.equal(ended.content.byteLength, 0);
assert.equal(ended.start, 11);
assert.equal(ended.totalBytes, 11);
assert.deepEqual(
client.commands.map((command) => command.constructor.name),
['HeadObjectCommand'],
);
});
test('maps absent objects and fails closed on range evidence drift', async () => {
const absent = new MemoryS3Client();
assert.deepEqual(
await store(absent).readLogRange(LOOKUP, { offset: 0, length: 1 }),
{ status: 'missing' },
);
for (const option of [
'corruptContentRange',
'corruptRangeETag',
'corruptRangeMetadata',
'shortRangeBody',
'oversizedRangeBody',
]) {
const client = new MemoryS3Client();
const adapter = store(client);
await adapter.put(COMMAND, chunks());
client.options[option] = true;
await assert.rejects(
adapter.readLogRange(LOOKUP, { offset: 0, length: 4 }),
(error) =>
error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
error.reason === 'integrity_mismatch',
);
}
});
test('resolves a concurrent conditional-copy winner by immutable inspect', async () => {
const client = new MemoryS3Client({ raceOnCopy: true });
const receipt = await store(client).put(COMMAND, chunks());
@@ -270,38 +393,40 @@ test('temporary cleanup failure is diagnostic and never reverses promotion', asy
},
}).put(COMMAND, chunks());
assert.equal(receipt.status, 'stored');
assert.deepEqual(diagnostics, [[
'delete unavailable',
'temporary_object_cleanup',
]]);
assert.deepEqual(diagnostics, [
['delete unavailable', 'temporary_object_cleanup'],
]);
assert.equal(client.objects.size, 2);
});
test('requires exact bucket, prefix, encryption and temporary ID configuration', async () => {
const client = new MemoryS3Client();
assert.throws(
() => new S3ClusterRemoteWorkerArtifactStore({
client,
bucket: 'Invalid_Bucket',
encryption: { mode: 's3' },
}),
() =>
new S3ClusterRemoteWorkerArtifactStore({
client,
bucket: 'Invalid_Bucket',
encryption: { mode: 's3' },
}),
/bucket is invalid/,
);
assert.throws(
() => new S3ClusterRemoteWorkerArtifactStore({
client,
bucket: 'valid-bucket',
prefix: '../escape',
encryption: { mode: 's3' },
}),
() =>
new S3ClusterRemoteWorkerArtifactStore({
client,
bucket: 'valid-bucket',
prefix: '../escape',
encryption: { mode: 's3' },
}),
/prefix is invalid/,
);
assert.throws(
() => new S3ClusterRemoteWorkerArtifactStore({
client,
bucket: 'valid-bucket',
encryption: { mode: 'kms' },
}),
() =>
new S3ClusterRemoteWorkerArtifactStore({
client,
bucket: 'valid-bucket',
encryption: { mode: 'kms' },
}),
/encryption is invalid/,
);
await assert.rejects(
@@ -355,8 +480,7 @@ test('propagates KMS and expected-owner fences to both sides of promotion', asyn
test('never deletes a colliding temporary object it cannot prove it owns', async () => {
const client = new MemoryS3Client();
const temporaryKey =
`tenant-a/worker-artifacts/temporary/${TEMPORARY_ID}`;
const temporaryKey = `tenant-a/worker-artifacts/temporary/${TEMPORARY_ID}`;
client.objects.set(temporaryKey, {
content: Buffer.from('other operation'),
contentType: 'application/octet-stream',
@@ -22,6 +22,7 @@ import type { LocalApiRunListRoute } from '../run/runListRoute';
import type { LocalApiRunReadRoute } from '../run/runReadRoute';
import type { LocalApiRunStepListRoute } from '../run/runStepListRoute';
import type { LocalApiRunCancellationRoute } from '../run/runCancellationRoute';
import type { LocalApiRunAttemptLogReadRoute } from '../run/runAttemptLogReadRoute';
import type { LocalApiTaskListRoute } from '../task/taskListRoute';
import type { LocalApiTaskReadRoute } from '../task/taskReadRoute';
import type { LocalApiTaskStartRoute } from '../task/taskStartRoute';
@@ -50,6 +51,14 @@ export type LocalApiAdmissionOperation =
runId: string;
input: Readonly<BoundedRunStepListInput>;
}>
| Readonly<{
operationId: 'run.log.read';
projectId: string;
runId: string;
attemptId: string;
offset: number;
length: number;
}>
| Readonly<{
operationId: 'run.cancel';
projectId: string;
@@ -99,6 +108,7 @@ export interface LocalApiAdmissionOptions {
readonly runEventListRoute: LocalApiRunEventListRoute;
readonly runStepListRoute: LocalApiRunStepListRoute;
readonly runCancellationRoute: LocalApiRunCancellationRoute;
readonly runAttemptLogReadRoute: LocalApiRunAttemptLogReadRoute;
readonly taskListRoute: LocalApiTaskListRoute;
readonly taskReadRoute: LocalApiTaskReadRoute;
readonly taskStartRoute: LocalApiTaskStartRoute;
@@ -176,6 +186,7 @@ export function createLocalApiAdmission(
typeof options.runEventListRoute?.handle !== 'function' ||
typeof options.runStepListRoute?.handle !== 'function' ||
typeof options.runCancellationRoute?.handle !== 'function' ||
typeof options.runAttemptLogReadRoute?.handle !== 'function' ||
typeof options.taskListRoute?.handle !== 'function' ||
typeof options.taskReadRoute?.handle !== 'function' ||
typeof options.taskStartRoute?.handle !== 'function' ||
@@ -236,12 +247,14 @@ export function createLocalApiAdmission(
request.operation.projectId,
request.operation.operationId === 'run.cancel'
? 'run.stop'
: request.operation.operationId === 'run.log.read'
? 'artifact.read'
: request.operation.operationId === 'task.start'
? 'run.start'
? 'run.start'
: request.operation.operationId === 'task.list' ||
request.operation.operationId === 'task.get'
? 'task.read'
: 'run.read',
request.operation.operationId === 'task.get'
? 'task.read'
: 'run.read',
),
);
} catch {
@@ -279,9 +292,15 @@ export function createLocalApiAdmission(
),
);
if (auditFailure) return auditFailure;
if (decision.effect === 'deny') return response(403, 'forbidden');
if (decision.effect === 'deny') {
return request.operation.operationId === 'run.log.read'
? response(404, 'artifact_not_found')
: response(403, 'forbidden');
}
if (decision.effect === 'require_approval') {
return response(403, 'approval_required');
return request.operation.operationId === 'run.log.read'
? response(404, 'artifact_not_found')
: response(403, 'approval_required');
}
if (request.signal.aborted) return response(503, 'request_unavailable');
try {
@@ -337,6 +356,16 @@ export function createLocalApiAdmission(
principal: authenticated.principal,
policyFence: decision.fence,
});
case 'run.log.read':
if (body !== null) return response(400, 'invalid_request_body');
return options.runAttemptLogReadRoute.handle({
projectId: request.operation.projectId,
runId: request.operation.runId,
attemptId: request.operation.attemptId,
offset: request.operation.offset,
length: request.operation.length,
signal: request.signal,
});
case 'task.list':
if (body !== null) return response(400, 'invalid_request_body');
return options.taskListRoute.handle({
@@ -15,6 +15,7 @@ import { createLocalApiRunReadRoute } from '../run/runReadRoute';
import { createLocalApiRunEventListRoute } from '../run/runEventListRoute';
import { createLocalApiRunStepListRoute } from '../run/runStepListRoute';
import { createLocalApiRunCancellationRoute } from '../run/runCancellationRoute';
import { createLocalApiRunAttemptLogReadRoute } from '../run/runAttemptLogReadRoute';
import { createLocalApiTaskListRoute } from '../task/taskListRoute';
import { createLocalApiTaskReadRoute } from '../task/taskReadRoute';
import { createLocalApiTaskStartRoute } from '../task/taskStartRoute';
@@ -109,6 +110,9 @@ export function createLocalApiProductSurface(
authority.runCancellation,
options.randomUuid ?? randomUUID,
);
const runAttemptLogReadRoute = createLocalApiRunAttemptLogReadRoute(
authority.runAttemptLogRead,
);
const taskListRoute = createLocalApiTaskListRoute(
authority.taskDefinitions,
);
@@ -128,6 +132,7 @@ export function createLocalApiProductSurface(
runEventListRoute,
runStepListRoute,
runCancellationRoute,
runAttemptLogReadRoute,
taskListRoute,
taskReadRoute,
taskStartRoute,
@@ -0,0 +1,111 @@
import {
InvalidRunAttemptLogReadError,
RunAttemptLogReadUnavailableError,
type RunAttemptLogReadRequest,
type RunAttemptLogReadResult,
} from '@qinglong/runtime-core/run-attempt-log-read';
import type { LocalApiResponse } from '../transport/contract';
export interface LocalApiRunAttemptLogReadCapability {
read(
request: Readonly<RunAttemptLogReadRequest>,
): Promise<RunAttemptLogReadResult>;
}
export interface LocalApiRunAttemptLogReadRequest {
readonly projectId: string;
readonly runId: string;
readonly attemptId: string;
readonly offset: number;
readonly length: number;
readonly signal?: AbortSignal;
}
export interface LocalApiRunAttemptLogReadRoute {
handle(
request: Readonly<LocalApiRunAttemptLogReadRequest>,
): Promise<LocalApiResponse>;
}
function response(
statusCode: number,
body: Readonly<Record<string, unknown>>,
): LocalApiResponse {
return Object.freeze({ statusCode, body: Object.freeze(body) });
}
function projection(
result: Extract<RunAttemptLogReadResult, { readonly status: 'available' }>,
): Readonly<Record<string, unknown>> {
return Object.freeze({
schema: 'qinglong/run-attempt-log-read-result@v1',
status: 'available',
projectId: result.projectId,
runId: result.runId,
attemptId: result.attemptId,
range: Object.freeze({
start: result.start,
endExclusive: result.endExclusive,
totalBytes: result.totalBytes,
...(result.nextOffset === undefined
? {}
: { nextOffset: result.nextOffset }),
}),
encoding: 'base64',
content: Buffer.from(
result.content.buffer,
result.content.byteOffset,
result.content.byteLength,
).toString('base64'),
truncation: result.truncation,
});
}
export function createLocalApiRunAttemptLogReadRoute(
capability: LocalApiRunAttemptLogReadCapability,
): Readonly<LocalApiRunAttemptLogReadRoute> {
if (!capability || typeof capability.read !== 'function') {
throw new TypeError('Local API Run Attempt log read capability is invalid');
}
return Object.freeze({
async handle(request: Readonly<LocalApiRunAttemptLogReadRequest>) {
try {
const result = await capability.read({
projectId: request.projectId,
runId: request.runId,
attemptId: request.attemptId,
range: Object.freeze({
offset: request.offset,
length: request.length,
}),
...(request.signal === undefined ? {} : { signal: request.signal }),
});
if (result.status === 'not_found') {
return response(404, { code: 'artifact_not_found' });
}
if (result.status === 'pending') {
return response(202, {
schema: 'qinglong/run-attempt-log-read-result@v1',
status: 'pending',
projectId: result.projectId,
runId: result.runId,
attemptId: result.attemptId,
});
}
if (result.status === 'missing') {
return response(503, { code: 'artifact_unavailable' });
}
return response(200, projection(result));
} catch (error) {
if (error instanceof InvalidRunAttemptLogReadError) {
return response(400, { code: 'invalid_run_log_read_request' });
}
if (error instanceof RunAttemptLogReadUnavailableError) {
return response(503, { code: 'artifact_unavailable' });
}
return response(503, { code: 'artifact_unavailable' });
}
},
});
}
@@ -28,6 +28,8 @@ const RUN_STEP_LIST_ROUTE_PATTERN =
/^\/api\/v3\/projects\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/runs\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/steps$/;
const RUN_CANCELLATION_ROUTE_PATTERN =
/^\/api\/v3\/projects\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/runs\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/cancellation$/;
const RUN_ATTEMPT_LOG_READ_ROUTE_PATTERN =
/^\/api\/v3\/projects\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/runs\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/attempts\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/log$/;
const TASK_LIST_ROUTE_PATTERN =
/^\/api\/v3\/projects\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/tasks$/;
const TASK_READ_ROUTE_PATTERN =
@@ -44,6 +46,7 @@ type LocalApiRouteResolution =
| 'invalid_run_list_query'
| 'invalid_run_event_list_query'
| 'invalid_run_step_list_query'
| 'invalid_run_log_read_query'
| 'invalid_task_list_query';
}>;
@@ -340,10 +343,7 @@ function parseTaskListQuery(
}
const name = field.slice(0, separator);
const value = field.slice(separator + 1);
if (
values.has(name) ||
(name !== 'limit' && name !== 'after_task_id')
) {
if (values.has(name) || (name !== 'limit' && name !== 'after_task_id')) {
throw new TypeError();
}
values.set(name, value);
@@ -363,13 +363,58 @@ function parseTaskListQuery(
}
return Object.freeze({
...(limit === undefined ? {} : { limit }),
...(taskId === undefined
? {}
: { after: Object.freeze({ taskId }) }),
...(taskId === undefined ? {} : { after: Object.freeze({ taskId }) }),
});
}
function route(request: IncomingMessage): LocalApiRouteResolution | null {
function parseRunAttemptLogReadQuery(
rawQuery: string | undefined,
profile: LocalApplicationProfile,
): Readonly<{ offset: number; length: number }> {
const defaultLength = profile === 'edge' ? 16 * 1024 : 32 * 1024;
if (rawQuery === undefined) {
return Object.freeze({ offset: 0, length: defaultLength });
}
if (rawQuery.length === 0) throw new TypeError();
const values = new Map<string, string>();
for (const field of rawQuery.split('&')) {
const separator = field.indexOf('=');
if (
separator < 1 ||
separator !== field.lastIndexOf('=') ||
separator === field.length - 1
) {
throw new TypeError();
}
const name = field.slice(0, separator);
const value = field.slice(separator + 1);
if (values.has(name) || (name !== 'offset' && name !== 'length')) {
throw new TypeError();
}
values.set(name, value);
}
const rawOffset = values.get('offset');
const offset = rawOffset === undefined ? 0 : Number(rawOffset);
const rawLength = values.get('length');
const length = rawLength === undefined ? defaultLength : Number(rawLength);
if (
!Number.isSafeInteger(offset) ||
offset < 0 ||
(rawOffset !== undefined && String(offset) !== rawOffset) ||
!Number.isSafeInteger(length) ||
length < 1 ||
length > 32 * 1024 ||
(rawLength !== undefined && String(length) !== rawLength)
) {
throw new TypeError();
}
return Object.freeze({ offset, length });
}
function route(
request: IncomingMessage,
profile: LocalApplicationProfile,
): LocalApiRouteResolution | null {
const rawUrl = request.url;
if (
typeof rawUrl !== 'string' ||
@@ -403,6 +448,20 @@ function route(request: IncomingMessage): LocalApiRouteResolution | null {
: null;
}
if (request.method !== 'GET') return null;
const runAttemptLogReadMatch = RUN_ATTEMPT_LOG_READ_ROUTE_PATTERN.exec(path);
if (runAttemptLogReadMatch) {
try {
return Object.freeze({
operationId: 'run.log.read',
projectId: runAttemptLogReadMatch[1]!,
runId: runAttemptLogReadMatch[2]!,
attemptId: runAttemptLogReadMatch[3]!,
...parseRunAttemptLogReadQuery(rawQuery, profile),
});
} catch {
return Object.freeze({ errorCode: 'invalid_run_log_read_query' });
}
}
const taskReadMatch = TASK_READ_ROUTE_PATTERN.exec(path);
if (taskReadMatch) {
return rawQuery === undefined
@@ -552,7 +611,7 @@ export async function startLocalApiHttpSurface(
send(response, requestId, errorResponse(503, 'server_overloaded'));
return;
}
const resolvedRoute = route(request);
const resolvedRoute = route(request, options.profile);
if (!resolvedRoute) {
send(response, requestId, errorResponse(404, 'route_not_found'));
return;
@@ -608,9 +667,9 @@ export async function startLocalApiHttpSurface(
error instanceof RangeError
? 'request_body_too_large'
: error instanceof Error &&
error.message === 'request_unavailable'
? 'request_unavailable'
: 'invalid_request_body';
error.message === 'request_unavailable'
? 'request_unavailable'
: 'invalid_request_body';
send(
response,
requestId,
@@ -618,8 +677,8 @@ export async function startLocalApiHttpSurface(
code === 'request_body_too_large'
? 413
: code === 'request_unavailable'
? 503
: 400,
? 503
: 400,
code,
),
);
@@ -101,6 +101,14 @@ function fixture(overrides = {}) {
return { statusCode: 202, body: { status: 'accepted' } };
},
},
runAttemptLogReadRoute: {
async handle(value) {
events.push(
`log:${value.projectId}:${value.runId}:${value.attemptId}:${value.offset}:${value.length}`,
);
return { statusCode: 200, body: { status: 'available' } };
},
},
taskListRoute: {
async handle(value) {
events.push(`tasks:${value.projectId}:${value.input.limit ?? 32}`);
@@ -148,6 +156,51 @@ test('authenticates, authorizes, durably audits and re-confirms before reading',
]);
});
test('uses artifact.read and masks denied or approval-fenced log existence', async () => {
const operation = Object.freeze({
operationId: 'run.log.read',
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_123',
offset: 4,
length: 16,
});
const allowed = fixture();
assert.deepEqual(await execute(allowed.admission, request({ operation })), {
statusCode: 200,
body: { status: 'available' },
});
assert.deepEqual(allowed.events, [
'authenticate',
'authorize:artifact.read:prj_default',
'audit:allowed:run.log.read',
'confirm',
'log:prj_default:run_123:attempt_123:4:16',
]);
for (const effect of ['deny', 'require_approval']) {
let routed = false;
const denied = fixture({
policy: {
async authorize() {
return { effect, reasons: ['masked'], fence: null };
},
},
runAttemptLogReadRoute: {
async handle() {
routed = true;
throw new Error('must not route');
},
},
});
assert.deepEqual(await execute(denied.admission, request({ operation })), {
statusCode: 404,
body: { code: 'artifact_not_found' },
});
assert.equal(routed, false);
}
});
test('uses the same admission chain with a route-owned run.list audit identity', async () => {
const { admission, events } = fixture();
assert.deepEqual(
+135 -93
View File
@@ -74,60 +74,71 @@ test('serves only the fixed canonical loopback Run route and drains idempotently
host: '127.0.0.1',
port,
admission: preparedAdmission(async (value, body) => {
observed.push(value);
if (
value.operation.operationId === 'run.cancel' ||
value.operation.operationId === 'task.start'
) {
return { statusCode: 202, body: { accepted: body } };
}
if (value.operation.operationId === 'run.get') {
return {
statusCode: 200,
body: { run: { id: value.operation.runId } },
};
}
if (value.operation.operationId === 'run.events.list') {
return {
statusCode: 200,
body: {
events: [],
hasMore: false,
nextAfterSequence: value.operation.input.afterSequence ?? 0,
},
};
}
if (value.operation.operationId === 'run.steps.list') {
return {
statusCode: 200,
body: {
steps: [],
hasMore: false,
next: value.operation.input.after ?? null,
},
};
}
if (value.operation.operationId === 'task.list') {
return {
statusCode: 200,
body: {
tasks: [],
hasMore: false,
input: value.operation.input,
},
};
}
if (value.operation.operationId === 'task.get') {
return {
statusCode: 200,
body: { task: { taskId: value.operation.taskId } },
};
}
observed.push(value);
if (
value.operation.operationId === 'run.cancel' ||
value.operation.operationId === 'task.start'
) {
return { statusCode: 202, body: { accepted: body } };
}
if (value.operation.operationId === 'run.get') {
return {
statusCode: 200,
body: { runs: [], hasMore: false, input: value.operation.input },
body: { run: { id: value.operation.runId } },
};
}),
}
if (value.operation.operationId === 'run.events.list') {
return {
statusCode: 200,
body: {
events: [],
hasMore: false,
nextAfterSequence: value.operation.input.afterSequence ?? 0,
},
};
}
if (value.operation.operationId === 'run.steps.list') {
return {
statusCode: 200,
body: {
steps: [],
hasMore: false,
next: value.operation.input.after ?? null,
},
};
}
if (value.operation.operationId === 'run.log.read') {
return {
statusCode: 200,
body: {
range: {
offset: value.operation.offset,
length: value.operation.length,
},
},
};
}
if (value.operation.operationId === 'task.list') {
return {
statusCode: 200,
body: {
tasks: [],
hasMore: false,
input: value.operation.input,
},
};
}
if (value.operation.operationId === 'task.get') {
return {
statusCode: 200,
body: { task: { taskId: value.operation.taskId } },
};
}
return {
statusCode: 200,
body: { runs: [], hasMore: false, input: value.operation.input },
};
}),
randomUuid: () => '019f70c0-0000-4000-8000-000000000003',
});
t.after(() => surface.stopAndDrain());
@@ -212,10 +223,7 @@ test('serves only the fixed canonical loopback Run route and drains idempotently
input: { after: { taskId: 'task_100' }, limit: 8 },
});
const task = await request(
port,
'/api/v3/projects/prj_default/tasks/task_1',
);
const task = await request(port, '/api/v3/projects/prj_default/tasks/task_1');
assert.deepEqual(task.body, { task: { taskId: 'task_1' } });
assert.deepEqual(observed[5].operation, {
operationId: 'task.get',
@@ -275,6 +283,28 @@ test('serves only the fixed canonical loopback Run route and drains idempotently
taskId: 'task_1',
});
const log = await request(
port,
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?offset=4&length=32',
);
assert.deepEqual(log.body, { range: { offset: 4, length: 32 } });
assert.deepEqual(observed[8].operation, {
operationId: 'run.log.read',
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_1',
offset: 4,
length: 32,
});
const defaultLog = await request(
port,
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log',
);
assert.deepEqual(defaultLog.body, {
range: { offset: 0, length: 16 * 1024 },
});
for (const invalidPath of [
'/api/v3/projects/prj_default/runs/run_123?expanded=true',
'/api/v3/projects/prj_default/runs/run%5f123',
@@ -299,6 +329,18 @@ test('serves only the fixed canonical loopback Run route and drains idempotently
assert.equal(invalid.statusCode, 400);
assert.deepEqual(invalid.body, { code: 'invalid_run_list_query' });
}
for (const invalidQuery of [
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?',
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?offset=-1',
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?offset=04',
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?length=0',
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?length=32769',
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?unknown=1',
]) {
const invalid = await request(port, invalidQuery);
assert.equal(invalid.statusCode, 400);
assert.deepEqual(invalid.body, { code: 'invalid_run_log_read_query' });
}
for (const invalidQuery of [
'/api/v3/projects/prj_default/tasks?',
'/api/v3/projects/prj_default/tasks?limit=08',
@@ -332,7 +374,7 @@ test('serves only the fixed canonical loopback Run route and drains idempotently
assert.equal(invalid.statusCode, 400);
assert.deepEqual(invalid.body, { code: 'invalid_run_step_list_query' });
}
assert.equal(observed.length, 8);
assert.equal(observed.length, 10);
assert.deepEqual(
await Promise.all([surface.stopAndDrain(), surface.stopAndDrain()]),
['stopped', 'stopped'],
@@ -347,9 +389,9 @@ test('rejects GET bodies without invoking the prepared route handler', async (t)
host: '127.0.0.1',
port,
admission: preparedAdmission(async () => {
handlers += 1;
return { statusCode: 200, body: {} };
}),
handlers += 1;
return { statusCode: 200, body: {} };
}),
});
t.after(() => surface.stopAndDrain());
const response = await request(
@@ -477,15 +519,15 @@ test('serves the reviewed worst-case 64-item Run list inside the fixed response
host: '127.0.0.1',
port,
admission: preparedAdmission(async () => {
return {
statusCode: 200,
body: {
runs: Object.freeze(Array.from({ length: 64 }, () => item)),
hasMore: true,
next: { createdAtMs: Number.MAX_SAFE_INTEGER, runId: id128 },
},
};
}),
return {
statusCode: 200,
body: {
runs: Object.freeze(Array.from({ length: 64 }, () => item)),
hasMore: true,
next: { createdAtMs: Number.MAX_SAFE_INTEGER, runId: id128 },
},
};
}),
});
t.after(() => surface.stopAndDrain());
const response = await request(
@@ -545,15 +587,15 @@ test('serves the reviewed worst-case 64-item RunEvent list inside the fixed resp
host: '127.0.0.1',
port,
admission: preparedAdmission(async () => {
return {
statusCode: 200,
body: {
events: Object.freeze(Array.from({ length: 64 }, () => event)),
hasMore: true,
nextAfterSequence: event.sequence,
},
};
}),
return {
statusCode: 200,
body: {
events: Object.freeze(Array.from({ length: 64 }, () => event)),
hasMore: true,
nextAfterSequence: event.sequence,
},
};
}),
});
t.after(() => surface.stopAndDrain());
const response = await request(
@@ -589,15 +631,15 @@ test('serves the reviewed worst-case 64-item Run Step list inside the fixed resp
host: '127.0.0.1',
port,
admission: preparedAdmission(async () => {
return {
statusCode: 200,
body: {
steps: Object.freeze(Array.from({ length: 64 }, () => item)),
hasMore: true,
next: { stepKey: id128, stepRunId: id128 },
},
};
}),
return {
statusCode: 200,
body: {
steps: Object.freeze(Array.from({ length: 64 }, () => item)),
hasMore: true,
next: { stepKey: id128, stepRunId: id128 },
},
};
}),
});
t.after(() => surface.stopAndDrain());
const response = await request(
@@ -621,13 +663,13 @@ test('bounds Edge admission concurrency and drains accepted work', async (t) =>
host: '127.0.0.1',
port,
admission: preparedAdmission(async (value) => {
admissions += 1;
await barrier;
return {
statusCode: 200,
body: { run: { id: value.operation.runId } },
};
}),
admissions += 1;
await barrier;
return {
statusCode: 200,
body: { run: { id: value.operation.runId } },
};
}),
});
t.after(() => surface.stopAndDrain());
const accepted = Array.from({ length: 4 }, () =>
@@ -0,0 +1,109 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
RunAttemptLogReadUnavailableError,
} = require('@qinglong/runtime-core/run-attempt-log-read');
const {
createLocalApiRunAttemptLogReadRoute,
} = require('../dist/run/runAttemptLogReadRoute.js');
function request(overrides = {}) {
return {
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_123',
offset: 2,
length: 16,
...overrides,
};
}
test('projects an available byte range as bounded base64 JSON', async () => {
const route = createLocalApiRunAttemptLogReadRoute({
async read(value) {
assert.deepEqual(value.range, { offset: 2, length: 16 });
return {
status: 'available',
projectId: value.projectId,
runId: value.runId,
attemptId: value.attemptId,
logArtifactId: `local-${'a'.repeat(30)}`,
content: Buffer.from('hello'),
start: 2,
endExclusive: 7,
totalBytes: 9,
nextOffset: 7,
truncation: { truncated: 'unknown' },
};
},
});
assert.deepEqual(await route.handle(request()), {
statusCode: 200,
body: {
schema: 'qinglong/run-attempt-log-read-result@v1',
status: 'available',
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_123',
range: { start: 2, endExclusive: 7, totalBytes: 9, nextOffset: 7 },
encoding: 'base64',
content: Buffer.from('hello').toString('base64'),
truncation: { truncated: 'unknown' },
},
});
});
test('maps pending, masked absence, missing storage and unavailable evidence', async () => {
const cases = [
[
{
status: 'pending',
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_123',
},
{
statusCode: 202,
body: {
schema: 'qinglong/run-attempt-log-read-result@v1',
status: 'pending',
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_123',
},
},
],
[
{ status: 'not_found' },
{ statusCode: 404, body: { code: 'artifact_not_found' } },
],
[
{
status: 'missing',
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_123',
logArtifactId: `local-${'a'.repeat(30)}`,
},
{ statusCode: 503, body: { code: 'artifact_unavailable' } },
],
];
for (const [result, expected] of cases) {
const route = createLocalApiRunAttemptLogReadRoute({
async read() {
return result;
},
});
assert.deepEqual(await route.handle(request()), expected);
}
const unavailable = createLocalApiRunAttemptLogReadRoute({
async read() {
throw new RunAttemptLogReadUnavailableError();
},
});
assert.deepEqual(await unavailable.handle(request()), {
statusCode: 503,
body: { code: 'artifact_unavailable' },
});
});
@@ -24,6 +24,9 @@ const {
const {
compileLocalCommandTaskDefinition,
} = require('@qinglong/runtime-core/task-definition-execution-compiler');
const {
RunAttemptLogReadService,
} = require('@qinglong/runtime-core/run-attempt-log-read');
const {
createBuiltInTaskSpecSemanticRegistry,
} = require('@qinglong/runtime-core/task-spec-semantic');
@@ -34,11 +37,16 @@ const {
const {
createLocalApiProductSurface,
} = require('../dist/application-runtime/localApiProductSurface.js');
const {
LocalRunAttemptLogRangeReader,
} = require('../../ql3-local-execution/dist/artifact-read/localRunAttemptLogRangeReader.js');
const NOW = 1_800_000_000_000;
const PEPPER_KEY_ID = 'local-api-pepper-v1';
const CREDENTIAL_ID = 'local-api-owner';
const RUN_ID = 'run_local_api_1';
const ATTEMPT_ID = 'attempt_local_api_1';
const LOG_ARTIFACT_ID = `local-${'a'.repeat(30)}`;
const SECRET = Buffer.alloc(32, 81).toString('base64url');
const PEPPER = Buffer.alloc(32, 82).toString('base64url');
const TOKEN = formatApiCredentialToken(CREDENTIAL_ID, SECRET);
@@ -206,15 +214,18 @@ function seed(databasePath, materialDigest) {
enabled: true,
occurredAtMs: NOW - 200,
};
const taskDefinition = createTaskDefinitionRecord({
...taskCommand,
spec: taskSemantics.normalize({
projectId: taskCommand.projectId,
taskId: taskCommand.taskId,
kind: taskCommand.kind,
spec: taskCommand.spec,
}),
}, NOW - 200);
const taskDefinition = createTaskDefinitionRecord(
{
...taskCommand,
spec: taskSemantics.normalize({
projectId: taskCommand.projectId,
taskId: taskCommand.taskId,
kind: taskCommand.kind,
spec: taskCommand.spec,
}),
},
NOW - 200,
);
const taskExecution = compileLocalCommandTaskDefinition(
taskDefinition,
taskSemantics,
@@ -298,6 +309,15 @@ function seed(databasePath, materialDigest) {
'manual', 'runtime', 'running', 1, 1, 0, ?)`,
)
.run(RUN_ID, NOW - 100);
client
.prepare(
`INSERT INTO "RunAttempts" (
"id", "run_id", "attempt", "status", "executor_type",
"log_artifact_id", "callback_sequence", "created_at_ms",
"started_at_ms"
) VALUES (?, ?, 1, 'running', 'local_process', ?, 0, ?, ?)`,
)
.run(ATTEMPT_ID, RUN_ID, LOG_ARTIFACT_ID, NOW - 90, NOW - 80);
client
.prepare(
`INSERT INTO "StepRuns" (
@@ -357,6 +377,15 @@ test('serves an authenticated Run through one real SQLite authority and durable
fs.chmodSync(root, 0o700);
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
const databasePath = path.join(root, 'qinglong3.sqlite');
const artifactRoot = path.join(root, 'artifacts');
const artifactShard = path.join(artifactRoot, 'aa');
fs.mkdirSync(artifactShard, { recursive: true, mode: 0o700 });
fs.chmodSync(artifactRoot, 0o700);
fs.chmodSync(artifactShard, 0o700);
const logContent = Buffer.from('local-api-log-line\n', 'utf8');
const logPath = path.join(artifactShard, `${LOG_ARTIFACT_ID}.log`);
fs.writeFileSync(logPath, logContent, { mode: 0o600 });
fs.chmodSync(logPath, 0o600);
const keyringDirectory = path.join(root, 'owner-pepper');
fs.mkdirSync(keyringDirectory, { mode: 0o700 });
const summary = provisionLocalOwnerPepperKey({
@@ -398,6 +427,15 @@ test('serves an authenticated Run through one real SQLite authority and durable
stepRuns: await runtime.stepRunReader(),
runCancellation: await runtime.runCancellationRepository(),
taskStart: await runtime.taskStartRepository(),
runAttemptLogRead: new RunAttemptLogReadService(
runtime.runRepository,
new LocalRunAttemptLogRangeReader(artifactRoot),
{
executorType: 'local_process',
artifactIdPattern: /^local-[a-f0-9]{30}$/,
maximumReadBytes: 32 * 1024,
},
),
taskDefinitions: runtime.taskDefinitions,
apiCredentials: runtime.apiCredentials,
ownerPepper: runtime.ownerPepper,
@@ -572,12 +610,29 @@ test('serves an authenticated Run through one real SQLite authority and durable
});
assert.equal(JSON.stringify(steps).includes('private'), false);
const log = await request(
port,
`Bearer ${TOKEN}`,
`/api/v3/projects/default/runs/${RUN_ID}/attempts/${ATTEMPT_ID}/log?offset=0&length=8`,
);
assert.equal(log.statusCode, 200);
assert.equal(log.body.schema, 'qinglong/run-attempt-log-read-result@v1');
assert.equal(log.body.status, 'available');
assert.equal(log.body.encoding, 'base64');
assert.equal(Buffer.from(log.body.content, 'base64').toString(), 'local-ap');
assert.deepEqual(log.body.range, {
start: 0,
endExclusive: 8,
totalBytes: logContent.byteLength,
nextOffset: 8,
});
assert.deepEqual(log.body.truncation, { truncated: 'unknown' });
const cancellationBody = JSON.stringify({
schema: 'qinglong/run-cancellation@v1',
mutationId: 'cancel-local-api-1',
});
const cancellationPath =
`/api/v3/projects/default/runs/${RUN_ID}/cancellation`;
const cancellationPath = `/api/v3/projects/default/runs/${RUN_ID}/cancellation`;
const cancellationOptions = {
method: 'POST',
headers: {
@@ -623,7 +678,7 @@ test('serves an authenticated Run through one real SQLite authority and durable
WHERE operation_id IN (
'run.get', 'run.list', 'run.events.list', 'run.steps.list',
'run.cancel', 'task.get', 'task.list'
, 'task.start'
, 'task.start', 'run.log.read'
)
ORDER BY operation_id, outcome`,
)
@@ -636,6 +691,7 @@ test('serves an authenticated Run through one real SQLite authority and durable
'run.get:allowed',
'run.get:authentication_rejected',
'run.list:allowed',
'run.log.read:allowed',
'run.steps.list:allowed',
'task.get:allowed',
'task.get:allowed',
@@ -455,11 +455,28 @@ export async function bootstrapLocalApplication(
});
if (options.productSurface) {
const [stepRuns, runCancellation, taskStart] = await Promise.all([
const [
stepRuns,
runCancellation,
taskStart,
{ LocalRunAttemptLogRangeReader },
{ RunAttemptLogReadService },
] = await Promise.all([
storage.stepRunReader(),
storage.runCancellationRepository(),
storage.taskStartRepository(),
import('@qinglong/local-execution/artifact-read'),
import('@qinglong/runtime-core/run-attempt-log-read'),
]);
const runAttemptLogRead = new RunAttemptLogReadService(
storage.runs,
new LocalRunAttemptLogRangeReader(options.artifactRoot),
{
executorType: 'local_process',
artifactIdPattern: /^local-[a-f0-9]{30}$/,
maximumReadBytes: 32 * 1024,
},
);
productSurfaceLifecycle = await options.productSurface.start(
Object.freeze({
profile: options.profile,
@@ -467,6 +484,7 @@ export async function bootstrapLocalApplication(
stepRuns,
runCancellation,
taskStart,
runAttemptLogRead,
taskDefinitions: storage.taskDefinitions,
apiCredentials: storage.apiCredentials,
ownerPepper: storage.ownerPepper,
@@ -18,6 +18,10 @@ import type { PluginPackageRecoveryCycleResult } from '@qinglong/runtime-core/pl
import type { PluginPackageAutomationPublicationRecoveryCycleResult } from '@qinglong/runtime-core/plugin-package-automation-publication';
import type { PluginPackageTaskPublicationRecoveryCycleResult } from '@qinglong/runtime-core/plugin-package-task-publication';
import type { ProjectToolDefinitionSnapshotRecoveryCycleResult } from '@qinglong/runtime-core/project-tool-definition-snapshot';
import type {
RunAttemptLogReadRequest,
RunAttemptLogReadResult,
} from '@qinglong/runtime-core/run-attempt-log-read';
export type LocalApplicationProfile = 'edge' | 'standalone';
@@ -59,6 +63,11 @@ export interface LocalApplicationProductSurfaceAuthority {
ReadyFreshStorage['taskDefinitions'],
'findCurrentTaskDefinition' | 'listTaskDefinitions'
>;
readonly runAttemptLogRead: Readonly<{
read(
request: Readonly<RunAttemptLogReadRequest>,
): Promise<RunAttemptLogReadResult>;
}>;
readonly apiCredentials: ReadyFreshStorage['apiCredentials'];
readonly ownerPepper: ReadyFreshStorage['ownerPepper'];
readonly projectPolicy: ReadyFreshStorage['projectPolicy'];
@@ -88,8 +88,7 @@ const RECEIPT_TOKEN = 'A'.repeat(32);
const CLEANUP_RUN_ID = '019f70c0-0000-7000-8000-000000000011';
const CLEANUP_ATTEMPT_ID = '019f70c0-0000-7000-8000-000000000012';
const WORKFLOW_CANCELLATION_CREDENTIAL_ID = 'application-workflow-owner';
const WORKFLOW_CANCELLATION_PEPPER_KEY_ID =
'application-workflow-owner-v1';
const WORKFLOW_CANCELLATION_PEPPER_KEY_ID = 'application-workflow-owner-v1';
const WORKFLOW_CANCELLATION_PEPPER_BYTES = Buffer.alloc(32, 141);
const WORKFLOW_CANCELLATION_SECRET = Buffer.alloc(32, 142).toString(
'base64url',
@@ -322,8 +321,7 @@ function promptResourceConfiguration() {
const prefix = 'private router model output:';
const minimumOutputBytes = Buffer.byteLength(prefix, 'utf8');
const rawOutputBytes =
process.env.QL3_PROMPT_RESOURCE_OUTPUT_BYTES ??
String(minimumOutputBytes);
process.env.QL3_PROMPT_RESOURCE_OUTPUT_BYTES ?? String(minimumOutputBytes);
const outputBytes = Number(rawOutputBytes);
if (
!Number.isSafeInteger(outputBytes) ||
@@ -338,8 +336,7 @@ function promptResourceConfiguration() {
profile,
output: prefix + 'x'.repeat(outputBytes - Buffer.byteLength(prefix)),
outputBytes,
resourceProbe:
process.env.QL3_PROMPT_RESOURCE_OUTPUT_BYTES !== undefined,
resourceProbe: process.env.QL3_PROMPT_RESOURCE_OUTPUT_BYTES !== undefined,
});
}
@@ -1263,7 +1260,10 @@ test('executes one active Package Prompt through local AI composition with conte
promptOutputKeys: {
async active() {
metrics.keyLoads += 1;
return { keyId: 'edge-prompt-output-key-1', key: Buffer.alloc(32, 7) };
return {
keyId: 'edge-prompt-output-key-1',
key: Buffer.alloc(32, 7),
};
},
async resolve(keyId) {
metrics.keyResolutions += 1;
@@ -1274,7 +1274,10 @@ test('executes one active Package Prompt through local AI composition with conte
promptOutputRead: {
authorizer: {
async authorize(request) {
assert.equal(request.projectId, 'project-application-prompt-resource');
assert.equal(
request.projectId,
'project-application-prompt-resource',
);
assert.equal(request.principal.subject.id, 'edge-resource-owner');
return { effect: 'allow' };
},
@@ -1376,7 +1379,10 @@ test('executes one active Package Prompt through local AI composition with conte
assert.equal(recoveredByRequest.result.text, metrics.privateOutput);
assert.deepEqual(recoveredByRequest.reference, durableFirst.outputArtifact);
assert.equal(metrics.keyResolutions, 2);
assert.equal(metrics.lastResolvedKey.every((byte) => byte === 0), true);
assert.equal(
metrics.lastResolvedKey.every((byte) => byte === 0),
true,
);
const durableStorageAfter = sqliteStorageSnapshot(value.targetPath);
const durableStorageGrowth = sqliteStorageGrowth(
durableStorageBefore,
@@ -1450,9 +1456,8 @@ test('executes one active Package Prompt through local AI composition with conte
)
.get(durableFirst.admission.invocationId),
},
integrityCheck: reader
.prepare('PRAGMA integrity_check')
.get().integrity_check,
integrityCheck: reader.prepare('PRAGMA integrity_check').get()
.integrity_check,
};
} finally {
reader.close();
@@ -1532,8 +1537,7 @@ test('executes one active Package Prompt through local AI composition with conte
databaseLogicalWriteAmplificationPermille,
databaseAllocatedWriteAmplificationPermille,
walWriteAmplificationPermille,
journalMode:
resource.profile === 'edge' ? 'delete' : 'wal',
journalMode: resource.profile === 'edge' ? 'delete' : 'wal',
providerCalls: metrics.providerCalls,
keyLoads: metrics.keyLoads,
keyResolutions: metrics.keyResolutions,
@@ -1700,6 +1704,7 @@ test('starts an optional product surface after recovery and drains it before own
typeof authority.taskDefinitions.listTaskDefinitions,
'function',
);
assert.equal(typeof authority.runAttemptLogRead.read, 'function');
assert.equal(typeof authority.apiCredentials.resolve, 'function');
assert.equal(typeof authority.ownerPepper.resolveKey, 'function');
assert.equal(typeof authority.projectPolicy.resolve, 'function');
@@ -28,6 +28,11 @@
"require": "./dist/dispatch/index.js",
"default": "./dist/dispatch/index.js"
},
"./artifact-read": {
"types": "./dist/artifact-read/index.d.ts",
"require": "./dist/artifact-read/index.js",
"default": "./dist/artifact-read/index.js"
},
"./scheduler": {
"types": "./dist/scheduler/index.d.ts",
"require": "./dist/scheduler/index.js",
@@ -0,0 +1 @@
export * from './localRunAttemptLogRangeReader';
@@ -0,0 +1,285 @@
import { constants, type Stats } from 'node:fs';
import fs, { type FileHandle } from 'node:fs/promises';
import path from 'node:path';
import {
normalizeRunAttemptLogReadRange,
type RunAttemptLogRangeReader,
type RunAttemptLogRangeReadResult,
type RunAttemptLogReadIdentity,
type RunAttemptLogReadRange,
type RunAttemptLogTruncationView,
} from '@qinglong/runtime-core/run-attempt-log-read';
const LOCAL_ARTIFACT_ID = /^local-[a-f0-9]{30}$/;
const MAXIMUM_ARTIFACT_BYTES = 1024 * 1024 * 1024;
const MAXIMUM_FACT_BYTES = 1024;
export class LocalRunAttemptLogRangeReadError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'unsafe_path'
| 'integrity_mismatch',
options?: ErrorOptions,
) {
super(`Local Run Attempt log range read failed: ${reason}`, options);
this.name = 'LocalRunAttemptLogRangeReadError';
}
}
function isCode(error: unknown, code: string): boolean {
return (
!!error &&
typeof error === 'object' &&
'code' in error &&
(error as { code?: unknown }).code === code
);
}
function currentUid(): number | undefined {
return typeof process.getuid === 'function' ? process.getuid() : undefined;
}
function root(value: string): string {
if (
typeof value !== 'string' ||
!path.isAbsolute(value) ||
path.parse(value).root === value ||
value.includes('\0') ||
Buffer.byteLength(value, 'utf8') > 4096
) {
throw new LocalRunAttemptLogRangeReadError('invalid_configuration');
}
return path.resolve(value);
}
function identity(
value: Readonly<RunAttemptLogReadIdentity>,
): Readonly<RunAttemptLogReadIdentity> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
!LOCAL_ARTIFACT_ID.test(value.logArtifactId)
) {
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
}
return value;
}
function assertOwnedDirectory(stat: Stats): void {
const uid = currentUid();
if (
!stat.isDirectory() ||
stat.isSymbolicLink() ||
(stat.mode & 0o777) !== 0o700 ||
(uid !== undefined && stat.uid !== uid)
) {
throw new LocalRunAttemptLogRangeReadError('unsafe_path');
}
}
function assertOwnedFile(stat: Stats): void {
const uid = currentUid();
if (
!stat.isFile() ||
stat.nlink !== 1 ||
(stat.mode & 0o777) !== 0o600 ||
(uid !== undefined && stat.uid !== uid) ||
!Number.isSafeInteger(stat.size) ||
stat.size < 0 ||
stat.size > MAXIMUM_ARTIFACT_BYTES
) {
throw new LocalRunAttemptLogRangeReadError('unsafe_path');
}
}
async function optionalPrivateDirectory(directory: string): Promise<boolean> {
try {
assertOwnedDirectory(await fs.lstat(directory));
return true;
} catch (error) {
if (isCode(error, 'ENOENT')) return false;
if (error instanceof LocalRunAttemptLogRangeReadError) throw error;
throw new LocalRunAttemptLogRangeReadError('unsafe_path', { cause: error });
}
}
async function openPrivateFile(
filePath: string,
): Promise<FileHandle | undefined> {
try {
return await fs.open(
filePath,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
} catch (error) {
if (isCode(error, 'ENOENT')) return undefined;
throw new LocalRunAttemptLogRangeReadError('unsafe_path', { cause: error });
}
}
function exactFact(
value: unknown,
expected: Readonly<RunAttemptLogReadIdentity>,
): Readonly<RunAttemptLogTruncationView> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
}
const fact = value as Record<string, unknown>;
const keys = Object.keys(fact).sort();
if (
keys.join(',') !==
'attemptId,logArtifactId,maximumBytes,observedAtMs,quotaReached,runId,schemaVersion' ||
fact.schemaVersion !== 1 ||
fact.runId !== expected.runId ||
fact.attemptId !== expected.attemptId ||
fact.logArtifactId !== expected.logArtifactId ||
!Number.isSafeInteger(fact.maximumBytes) ||
Number(fact.maximumBytes) < 64 * 1024 ||
Number(fact.maximumBytes) > MAXIMUM_ARTIFACT_BYTES ||
typeof fact.quotaReached !== 'boolean' ||
!Number.isSafeInteger(fact.observedAtMs) ||
Number(fact.observedAtMs) < 0
) {
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
}
return Object.freeze({
truncated: fact.quotaReached,
maximumBytes: fact.maximumBytes as number,
observedAtMs: fact.observedAtMs as number,
});
}
async function readTruncationFact(
directory: string,
expected: Readonly<RunAttemptLogReadIdentity>,
signal?: AbortSignal,
): Promise<Readonly<RunAttemptLogTruncationView>> {
if (signal?.aborted) throw signal.reason;
const factPath = path.join(
directory,
`.${expected.logArtifactId}.log.truncated.json`,
);
const handle = await openPrivateFile(factPath);
if (!handle) return Object.freeze({ truncated: 'unknown' as const });
try {
const before = await handle.stat();
assertOwnedFile(before);
if (before.size < 2 || before.size > MAXIMUM_FACT_BYTES) {
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
}
const content = Buffer.allocUnsafe(before.size);
let read = 0;
while (read < content.byteLength) {
if (signal?.aborted) throw signal.reason;
const result = await handle.read(
content,
read,
content.byteLength - read,
read,
);
if (result.bytesRead < 1) {
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
}
read += result.bytesRead;
}
const after = await handle.stat();
assertOwnedFile(after);
if (
after.dev !== before.dev ||
after.ino !== before.ino ||
after.size !== before.size
) {
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
}
try {
const text = new TextDecoder('utf-8', { fatal: true }).decode(content);
return exactFact(JSON.parse(text), expected);
} catch (error) {
if (error instanceof LocalRunAttemptLogRangeReadError) throw error;
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch', {
cause: error,
});
} finally {
content.fill(0);
}
} finally {
await handle.close().catch(() => undefined);
}
}
export class LocalRunAttemptLogRangeReader implements RunAttemptLogRangeReader {
private readonly root: string;
constructor(artifactRoot: string) {
this.root = root(artifactRoot);
}
async read(
rawIdentity: Readonly<RunAttemptLogReadIdentity>,
rawRange: Readonly<RunAttemptLogReadRange>,
signal?: AbortSignal,
): Promise<RunAttemptLogRangeReadResult> {
const expected = identity(rawIdentity);
const range = normalizeRunAttemptLogReadRange(rawRange);
if (signal?.aborted) throw signal.reason;
if (!(await optionalPrivateDirectory(this.root))) {
return Object.freeze({ status: 'missing' as const });
}
const directory = path.join(
this.root,
expected.logArtifactId.slice('local-'.length, 'local-'.length + 2),
);
if (!(await optionalPrivateDirectory(directory))) {
return Object.freeze({ status: 'missing' as const });
}
const target = path.join(directory, `${expected.logArtifactId}.log`);
const handle = await openPrivateFile(target);
if (!handle) return Object.freeze({ status: 'missing' as const });
try {
const before = await handle.stat();
assertOwnedFile(before);
const start = Math.min(range.offset, before.size);
const expectedBytes = Math.min(range.length, before.size - start);
const content = Buffer.allocUnsafe(expectedBytes);
let read = 0;
while (read < expectedBytes) {
if (signal?.aborted) throw signal.reason;
const result = await handle.read(
content,
read,
expectedBytes - read,
start + read,
);
if (result.bytesRead < 1) {
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
}
read += result.bytesRead;
}
const after = await handle.stat();
assertOwnedFile(after);
if (
after.dev !== before.dev ||
after.ino !== before.ino ||
after.size < before.size
) {
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
}
const endExclusive = start + content.byteLength;
const truncation = await readTruncationFact(directory, expected, signal);
return Object.freeze({
status: 'available' as const,
content,
start,
endExclusive,
totalBytes: before.size,
...(endExclusive < before.size ? { nextOffset: endExclusive } : {}),
truncation,
});
} finally {
await handle.close().catch(() => undefined);
}
}
}
@@ -0,0 +1,158 @@
const assert = require('node:assert/strict');
const fs = require('node:fs/promises');
const os = require('node:os');
const path = require('node:path');
const { test } = require('node:test');
const {
LocalRunAttemptLogRangeReadError,
LocalRunAttemptLogRangeReader,
} = require('../dist/artifact-read/localRunAttemptLogRangeReader.js');
const artifactId = `local-${'a'.repeat(30)}`;
const identity = Object.freeze({
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_123',
logArtifactId: artifactId,
});
async function fixture(t, content = Buffer.from('0123456789')) {
const parent = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-log-read-'));
t.after(() => fs.rm(parent, { recursive: true, force: true }));
const root = path.join(parent, 'artifacts');
const shard = path.join(root, 'aa');
await fs.mkdir(shard, { recursive: true, mode: 0o700 });
await fs.chmod(root, 0o700);
await fs.chmod(shard, 0o700);
const log = path.join(shard, `${artifactId}.log`);
await fs.writeFile(log, content, { mode: 0o600 });
await fs.chmod(log, 0o600);
return { parent, root, shard, log };
}
async function fact(shard, overrides = {}) {
const file = path.join(shard, `.${artifactId}.log.truncated.json`);
await fs.writeFile(
file,
JSON.stringify({
schemaVersion: 1,
runId: identity.runId,
attemptId: identity.attemptId,
logArtifactId: identity.logArtifactId,
maximumBytes: 64 * 1024,
quotaReached: false,
observedAtMs: 9,
...overrides,
}),
{ mode: 0o600 },
);
await fs.chmod(file, 0o600);
return file;
}
test('reads one bounded private-file snapshot and canonical truncation fact', async (t) => {
const value = await fixture(t);
await fact(value.shard, { quotaReached: true });
const result = await new LocalRunAttemptLogRangeReader(value.root).read(
identity,
{ offset: 2, length: 4 },
);
assert.equal(result.status, 'available');
assert.equal(Buffer.from(result.content).toString(), '2345');
assert.deepEqual(
{
start: result.start,
endExclusive: result.endExclusive,
totalBytes: result.totalBytes,
nextOffset: result.nextOffset,
truncation: result.truncation,
},
{
start: 2,
endExclusive: 6,
totalBytes: 10,
nextOffset: 6,
truncation: {
truncated: true,
maximumBytes: 64 * 1024,
observedAtMs: 9,
},
},
);
});
test('returns unknown truncation and a stable empty range beyond the snapshot', async (t) => {
const value = await fixture(t);
const result = await new LocalRunAttemptLogRangeReader(value.root).read(
identity,
{ offset: 99, length: 4 },
);
assert.equal(result.status, 'available');
assert.equal(result.content.byteLength, 0);
assert.equal(result.start, 10);
assert.equal(result.endExclusive, 10);
assert.equal(result.totalBytes, 10);
assert.equal(result.nextOffset, undefined);
assert.deepEqual(result.truncation, { truncated: 'unknown' });
});
test('treats absent root, shard and log as missing', async (t) => {
const parent = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-log-missing-'));
t.after(() => fs.rm(parent, { recursive: true, force: true }));
const root = path.join(parent, 'artifacts');
const reader = new LocalRunAttemptLogRangeReader(root);
assert.deepEqual(await reader.read(identity, { offset: 0, length: 1 }), {
status: 'missing',
});
await fs.mkdir(root, { mode: 0o700 });
assert.deepEqual(await reader.read(identity, { offset: 0, length: 1 }), {
status: 'missing',
});
const shard = path.join(root, 'aa');
await fs.mkdir(shard, { mode: 0o700 });
assert.deepEqual(await reader.read(identity, { offset: 0, length: 1 }), {
status: 'missing',
});
});
test('fails closed for symlink targets and widened file permissions', async (t) => {
const symlink = await fixture(t);
await fs.rm(symlink.log);
await fs.symlink(path.join(symlink.parent, 'outside'), symlink.log);
await assert.rejects(
new LocalRunAttemptLogRangeReader(symlink.root).read(identity, {
offset: 0,
length: 1,
}),
LocalRunAttemptLogRangeReadError,
);
const widened = await fixture(t);
await fs.chmod(widened.log, 0o644);
await assert.rejects(
new LocalRunAttemptLogRangeReader(widened.root).read(identity, {
offset: 0,
length: 1,
}),
LocalRunAttemptLogRangeReadError,
);
});
test('fails closed for truncation identity drift and an aborted request', async (t) => {
const value = await fixture(t);
await fact(value.shard, { attemptId: 'attempt_other' });
const reader = new LocalRunAttemptLogRangeReader(value.root);
await assert.rejects(
reader.read(identity, { offset: 0, length: 1 }),
(error) =>
error instanceof LocalRunAttemptLogRangeReadError &&
error.reason === 'integrity_mismatch',
);
const abort = new AbortController();
abort.abort(new Error('cancelled'));
await assert.rejects(
reader.read(identity, { offset: 0, length: 1 }, abort.signal),
/cancelled/,
);
});
+8
View File
@@ -232,6 +232,9 @@
],
"bounded-run-step-list-projection": [
"dist/run/projection/boundedRunStepListProjection.d.ts"
],
"run-attempt-log-read": [
"dist/run/log-read/runAttemptLogRead.d.ts"
]
}
},
@@ -281,6 +284,11 @@
"require": "./dist/run/projection/boundedRunStepListProjection.js",
"default": "./dist/run/projection/boundedRunStepListProjection.js"
},
"./run-attempt-log-read": {
"types": "./dist/run/log-read/runAttemptLogRead.d.ts",
"require": "./dist/run/log-read/runAttemptLogRead.js",
"default": "./dist/run/log-read/runAttemptLogRead.js"
},
"./task-definition": {
"types": "./dist/task-definition/taskDefinition.d.ts",
"require": "./dist/task-definition/taskDefinition.js",
@@ -0,0 +1,343 @@
import { RUN_ATTEMPT_STATUSES, type RunAttemptStatus } from '../run';
import type { RunRepositoryReader } from '../runRepository';
export const MAX_RUN_ATTEMPT_LOG_READ_BYTES = 256 * 1024;
const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
const TERMINAL_ATTEMPT_STATUSES = new Set<RunAttemptStatus>([
'succeeded',
'failed',
'cancelled',
'timed_out',
'lost',
]);
export interface RunAttemptLogReadRange {
readonly offset: number;
readonly length: number;
}
export interface RunAttemptLogReadIdentity {
readonly projectId: string;
readonly runId: string;
readonly attemptId: string;
readonly logArtifactId: string;
}
export interface RunAttemptLogTruncationView {
readonly truncated: boolean | 'unknown';
readonly maximumBytes?: number;
readonly observedAtMs?: number;
}
export type RunAttemptLogRangeReadResult =
| Readonly<{ readonly status: 'missing' }>
| Readonly<{
readonly status: 'available';
readonly content: Uint8Array;
readonly start: number;
readonly endExclusive: number;
readonly totalBytes: number;
readonly nextOffset?: number;
readonly truncation: Readonly<RunAttemptLogTruncationView>;
}>;
export interface RunAttemptLogRangeReader {
read(
identity: Readonly<RunAttemptLogReadIdentity>,
range: Readonly<RunAttemptLogReadRange>,
signal?: AbortSignal,
): Promise<RunAttemptLogRangeReadResult>;
}
export interface RunAttemptLogReadRequest {
readonly projectId: string;
readonly runId: string;
readonly attemptId: string;
readonly range: Readonly<RunAttemptLogReadRange>;
readonly signal?: AbortSignal;
}
export type RunAttemptLogReadResult =
| Readonly<{ readonly status: 'not_found' }>
| Readonly<{
readonly status: 'pending';
readonly projectId: string;
readonly runId: string;
readonly attemptId: string;
readonly logArtifactId?: string;
}>
| (Readonly<RunAttemptLogReadIdentity> &
Readonly<{ readonly status: 'missing' }>)
| (Readonly<RunAttemptLogReadIdentity> &
Extract<RunAttemptLogRangeReadResult, { readonly status: 'available' }>);
export interface RunAttemptLogReadServiceOptions {
readonly executorType: 'local_process' | 'remote_worker';
readonly artifactIdPattern: RegExp;
readonly maximumReadBytes: number;
readonly activeMissingIsPending?: boolean;
}
export class InvalidRunAttemptLogReadError extends TypeError {
constructor(message: string) {
super(`Run Attempt log read is invalid: ${message}`);
this.name = 'InvalidRunAttemptLogReadError';
}
}
export class RunAttemptLogReadUnavailableError extends Error {
constructor(options?: ErrorOptions) {
super('Run Attempt log read is unavailable', options);
this.name = 'RunAttemptLogReadUnavailableError';
}
}
function canonicalId(name: string, value: unknown): string {
if (typeof value !== 'string' || !ID_PATTERN.test(value)) {
throw new InvalidRunAttemptLogReadError(`${name} is invalid`);
}
return value;
}
function exactKeys(
value: object,
required: readonly string[],
optional: readonly string[],
name: string,
): void {
const keys = Object.keys(value);
const allowed = new Set([...required, ...optional]);
if (
required.some((key) => !Object.hasOwn(value, key)) ||
keys.some((key) => !allowed.has(key))
) {
throw new InvalidRunAttemptLogReadError(`${name} shape is invalid`);
}
}
export function normalizeRunAttemptLogReadRange(
value: Readonly<RunAttemptLogReadRange>,
maximumReadBytes = MAX_RUN_ATTEMPT_LOG_READ_BYTES,
): Readonly<RunAttemptLogReadRange> {
if (
!Number.isSafeInteger(maximumReadBytes) ||
maximumReadBytes < 1 ||
maximumReadBytes > MAX_RUN_ATTEMPT_LOG_READ_BYTES
) {
throw new InvalidRunAttemptLogReadError('maximum read bytes is invalid');
}
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidRunAttemptLogReadError('range is invalid');
}
exactKeys(value, ['length', 'offset'], [], 'range');
if (!Number.isSafeInteger(value.offset) || value.offset < 0) {
throw new InvalidRunAttemptLogReadError('offset is invalid');
}
if (
!Number.isSafeInteger(value.length) ||
value.length < 1 ||
value.length > maximumReadBytes
) {
throw new InvalidRunAttemptLogReadError('length is invalid');
}
return Object.freeze({ offset: value.offset, length: value.length });
}
function prepareOptions(
options: RunAttemptLogReadServiceOptions,
): Readonly<RunAttemptLogReadServiceOptions> {
if (
!options ||
typeof options !== 'object' ||
Array.isArray(options) ||
(options.executorType !== 'local_process' &&
options.executorType !== 'remote_worker') ||
!(options.artifactIdPattern instanceof RegExp) ||
options.artifactIdPattern.global ||
options.artifactIdPattern.sticky ||
(options.activeMissingIsPending !== undefined &&
typeof options.activeMissingIsPending !== 'boolean')
) {
throw new InvalidRunAttemptLogReadError('service options are invalid');
}
exactKeys(
options,
['artifactIdPattern', 'executorType', 'maximumReadBytes'],
['activeMissingIsPending'],
'service options',
);
normalizeRunAttemptLogReadRange(
{ offset: 0, length: options.maximumReadBytes },
options.maximumReadBytes,
);
return Object.freeze({ ...options });
}
function truncation(
value: Readonly<RunAttemptLogTruncationView>,
): Readonly<RunAttemptLogTruncationView> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
(value.truncated !== true &&
value.truncated !== false &&
value.truncated !== 'unknown') ||
(value.maximumBytes !== undefined &&
(!Number.isSafeInteger(value.maximumBytes) || value.maximumBytes < 1)) ||
(value.observedAtMs !== undefined &&
(!Number.isSafeInteger(value.observedAtMs) || value.observedAtMs < 0)) ||
(value.truncated === 'unknown' &&
(value.maximumBytes !== undefined || value.observedAtMs !== undefined))
) {
throw new RunAttemptLogReadUnavailableError();
}
return Object.freeze({ ...value });
}
function available(
identity: Readonly<RunAttemptLogReadIdentity>,
range: Readonly<RunAttemptLogReadRange>,
result: Extract<
RunAttemptLogRangeReadResult,
{ readonly status: 'available' }
>,
): RunAttemptLogReadResult {
if (
!(result.content instanceof Uint8Array) ||
!Number.isSafeInteger(result.start) ||
!Number.isSafeInteger(result.endExclusive) ||
!Number.isSafeInteger(result.totalBytes) ||
result.start !== Math.min(range.offset, result.totalBytes) ||
result.endExclusive !== result.start + result.content.byteLength ||
result.endExclusive > result.totalBytes ||
result.content.byteLength > range.length ||
(result.nextOffset === undefined) !==
(result.endExclusive === result.totalBytes) ||
(result.nextOffset !== undefined &&
result.nextOffset !== result.endExclusive)
) {
throw new RunAttemptLogReadUnavailableError();
}
return Object.freeze({
status: 'available' as const,
...identity,
content: result.content,
start: result.start,
endExclusive: result.endExclusive,
totalBytes: result.totalBytes,
...(result.nextOffset === undefined
? {}
: { nextOffset: result.nextOffset }),
truncation: truncation(result.truncation),
});
}
export class RunAttemptLogReadService {
private readonly options: Readonly<RunAttemptLogReadServiceOptions>;
constructor(
private readonly runs: Pick<
RunRepositoryReader,
'findRunById' | 'findAttemptById'
>,
private readonly reader: RunAttemptLogRangeReader,
options: RunAttemptLogReadServiceOptions,
) {
if (
!runs ||
typeof runs.findRunById !== 'function' ||
typeof runs.findAttemptById !== 'function' ||
!reader ||
typeof reader.read !== 'function'
) {
throw new InvalidRunAttemptLogReadError('dependencies are invalid');
}
this.options = prepareOptions(options);
}
async read(
request: Readonly<RunAttemptLogReadRequest>,
): Promise<RunAttemptLogReadResult> {
if (!request || typeof request !== 'object' || Array.isArray(request)) {
throw new InvalidRunAttemptLogReadError('request is invalid');
}
exactKeys(
request,
['attemptId', 'projectId', 'range', 'runId'],
['signal'],
'request',
);
const projectId = canonicalId('projectId', request.projectId);
const runId = canonicalId('runId', request.runId);
const attemptId = canonicalId('attemptId', request.attemptId);
const range = normalizeRunAttemptLogReadRange(
request.range,
this.options.maximumReadBytes,
);
if (request.signal?.aborted) {
throw new RunAttemptLogReadUnavailableError({
cause: request.signal.reason,
});
}
try {
const run = await this.runs.findRunById(runId);
if (
!run ||
run.id !== runId ||
run.projectId !== projectId ||
run.executionOwner !== 'runtime'
) {
return Object.freeze({ status: 'not_found' as const });
}
const attempt = await this.runs.findAttemptById(attemptId);
if (
!attempt ||
attempt.id !== attemptId ||
attempt.runId !== runId ||
attempt.executorType !== this.options.executorType ||
!RUN_ATTEMPT_STATUSES.includes(attempt.status)
) {
return Object.freeze({ status: 'not_found' as const });
}
if (attempt.logArtifactId === undefined) {
return Object.freeze({
status: 'pending' as const,
projectId,
runId,
attemptId,
});
}
if (!this.options.artifactIdPattern.test(attempt.logArtifactId)) {
return Object.freeze({ status: 'not_found' as const });
}
const identity = Object.freeze({
projectId,
runId,
attemptId,
logArtifactId: attempt.logArtifactId,
});
const result = await this.reader.read(identity, range, request.signal);
if (result.status === 'missing') {
if (
this.options.activeMissingIsPending === true &&
!TERMINAL_ATTEMPT_STATUSES.has(attempt.status)
) {
return Object.freeze({ status: 'pending' as const, ...identity });
}
return Object.freeze({ status: 'missing' as const, ...identity });
}
return available(identity, range, result);
} catch (error) {
if (
error instanceof InvalidRunAttemptLogReadError ||
error instanceof RunAttemptLogReadUnavailableError
) {
throw error;
}
throw new RunAttemptLogReadUnavailableError({ cause: error });
}
}
}
@@ -0,0 +1,261 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
InvalidRunAttemptLogReadError,
MAX_RUN_ATTEMPT_LOG_READ_BYTES,
RunAttemptLogReadService,
RunAttemptLogReadUnavailableError,
normalizeRunAttemptLogReadRange,
} = require('../dist/run/log-read/runAttemptLogRead.js');
function run(overrides = {}) {
return {
id: 'run_123',
projectId: 'prj_default',
taskId: 'task_1',
taskRevision: 'revision_1',
triggerType: 'task_start',
executionOrigin: 'manual',
executionOwner: 'runtime',
status: 'running',
version: 2,
eventSequence: 2,
priority: 0,
createdAtMs: 1,
...overrides,
};
}
function attempt(overrides = {}) {
return {
id: 'attempt_123',
runId: 'run_123',
attempt: 1,
status: 'running',
executorType: 'local_process',
logArtifactId: `local-${'a'.repeat(30)}`,
callbackSequence: 0,
createdAtMs: 1,
...overrides,
};
}
function service(overrides = {}) {
const calls = [];
const runs = overrides.runs ?? {
async findRunById() {
return run();
},
async findAttemptById() {
return attempt();
},
};
const reader = overrides.reader ?? {
async read(identity, range, signal) {
calls.push({ identity, range, signal });
return {
status: 'available',
content: Buffer.from('log'),
start: range.offset,
endExclusive: range.offset + 3,
totalBytes: range.offset + 5,
nextOffset: range.offset + 3,
truncation: { truncated: false, maximumBytes: 1024, observedAtMs: 9 },
};
},
};
return {
calls,
value: new RunAttemptLogReadService(runs, reader, {
executorType: overrides.executorType ?? 'local_process',
artifactIdPattern: overrides.artifactIdPattern ?? /^local-[a-f0-9]{30}$/,
maximumReadBytes: overrides.maximumReadBytes ?? 32 * 1024,
...(overrides.activeMissingIsPending === undefined
? {}
: { activeMissingIsPending: overrides.activeMissingIsPending }),
}),
};
}
function request(overrides = {}) {
return {
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_123',
range: { offset: 4, length: 16 },
...overrides,
};
}
test('normalizes only bounded safe ranges', () => {
assert.deepEqual(normalizeRunAttemptLogReadRange({ offset: 0, length: 1 }), {
offset: 0,
length: 1,
});
assert.deepEqual(
normalizeRunAttemptLogReadRange({
offset: Number.MAX_SAFE_INTEGER,
length: MAX_RUN_ATTEMPT_LOG_READ_BYTES,
}),
{ offset: Number.MAX_SAFE_INTEGER, length: MAX_RUN_ATTEMPT_LOG_READ_BYTES },
);
for (const range of [
{ offset: -1, length: 1 },
{ offset: 0.5, length: 1 },
{ offset: 0, length: 0 },
{ offset: 0, length: MAX_RUN_ATTEMPT_LOG_READ_BYTES + 1 },
]) {
assert.throws(
() => normalizeRunAttemptLogReadRange(range),
InvalidRunAttemptLogReadError,
);
}
});
test('validates Project, Run, Attempt, owner and executor before storage access', async () => {
const cases = [
{ run: null },
{ run: run({ projectId: 'prj_other' }) },
{ run: run({ executionOwner: 'legacy' }) },
{ attempt: null },
{ attempt: attempt({ runId: 'run_other' }) },
{ attempt: attempt({ executorType: 'remote_worker' }) },
{ attempt: attempt({ logArtifactId: `wlog-${'a'.repeat(30)}` }) },
];
for (const values of cases) {
let reads = 0;
const { value } = service({
runs: {
async findRunById() {
return values.run === undefined ? run() : values.run;
},
async findAttemptById() {
return values.attempt === undefined ? attempt() : values.attempt;
},
},
reader: {
async read() {
reads += 1;
return { status: 'missing' };
},
},
});
assert.deepEqual(await value.read(request()), { status: 'not_found' });
assert.equal(reads, 0);
}
});
test('returns pending before Artifact binding and for active remote publication lag', async () => {
const unbound = service({
runs: {
async findRunById() {
return run();
},
async findAttemptById() {
return attempt({ logArtifactId: undefined });
},
},
});
assert.deepEqual(await unbound.value.read(request()), {
status: 'pending',
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_123',
});
assert.equal(unbound.calls.length, 0);
const remote = service({
executorType: 'remote_worker',
artifactIdPattern: /^wlog-[a-f0-9]{30}$/,
activeMissingIsPending: true,
runs: {
async findRunById() {
return run();
},
async findAttemptById() {
return attempt({
executorType: 'remote_worker',
logArtifactId: `wlog-${'b'.repeat(30)}`,
});
},
},
reader: {
async read() {
return { status: 'missing' };
},
},
});
assert.equal((await remote.value.read(request())).status, 'pending');
});
test('returns a validated bounded snapshot without copying storage bytes', async () => {
const content = Buffer.from('log');
const abort = new AbortController();
const { value, calls } = service({
reader: {
async read(identity, range, signal) {
assert.equal(signal, abort.signal);
return {
status: 'available',
content,
start: 4,
endExclusive: 7,
totalBytes: 9,
nextOffset: 7,
truncation: {
truncated: true,
maximumBytes: 64 * 1024,
observedAtMs: 10,
},
};
},
},
});
const result = await value.read(request({ signal: abort.signal }));
assert.equal(result.status, 'available');
assert.equal(result.content, content);
assert.equal(result.nextOffset, 7);
assert.deepEqual(result.truncation, {
truncated: true,
maximumBytes: 64 * 1024,
observedAtMs: 10,
});
assert.equal(calls.length, 0);
});
test('fails closed on malformed storage results and dependency failures', async () => {
const malformed = service({
reader: {
async read() {
return {
status: 'available',
content: Buffer.from('too-long'),
start: 4,
endExclusive: 12,
totalBytes: 9,
truncation: { truncated: 'unknown' },
};
},
},
});
await assert.rejects(
malformed.value.read(request()),
RunAttemptLogReadUnavailableError,
);
const failed = service({
runs: {
async findRunById() {
throw new Error('database detail');
},
async findAttemptById() {
return null;
},
},
});
await assert.rejects(
failed.value.read(request()),
RunAttemptLogReadUnavailableError,
);
});