mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 00:17:47 +08:00
feat(ql3): add profile-aware run log reads
This commit is contained in:
@@ -24,6 +24,7 @@ import { createClusterControlRunReadRoute } from '../run/runReadRoute';
|
||||
import { createClusterControlRunListRoute } from '../run/runListRoute';
|
||||
import { createClusterControlRunEventListRoute } from '../run/runEventListRoute';
|
||||
import { createClusterControlRunStepListRoute } from '../run/runStepListRoute';
|
||||
import { createClusterControlRunAttemptLogReadRoute } from '../run/runAttemptLogReadRoute';
|
||||
import { createClusterControlTaskListRoute } from '../task/taskListRoute';
|
||||
import { createClusterControlTaskReadRoute } from '../task/taskReadRoute';
|
||||
import { createClusterControlTaskStartRoute } from '../task/taskStartRoute';
|
||||
@@ -69,6 +70,7 @@ export const PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS = Object.freeze([
|
||||
'run.list',
|
||||
'run.events.list',
|
||||
'run.steps.list',
|
||||
'run.log.read',
|
||||
'run.cancel',
|
||||
'workflow.read',
|
||||
'workflow.run.read',
|
||||
@@ -195,6 +197,10 @@ export function createProductionClusterControlApplicationStack(
|
||||
input.runs,
|
||||
input.trustedToolStorage.stepRuns,
|
||||
),
|
||||
createClusterControlRunAttemptLogReadRoute(
|
||||
input.runs,
|
||||
input.workerRuntime?.runAttemptLogRead,
|
||||
),
|
||||
createClusterControlRunCancellationRoute(
|
||||
input.runCancellation,
|
||||
createEventId,
|
||||
|
||||
@@ -6,12 +6,19 @@ import {
|
||||
ChecksumMode,
|
||||
CopyObjectCommand,
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
MetadataDirective,
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
ServerSideEncryption,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import {
|
||||
normalizeRunAttemptLogReadRange,
|
||||
type RunAttemptLogRangeReadResult,
|
||||
type RunAttemptLogReadIdentity,
|
||||
type RunAttemptLogReadRange,
|
||||
} from '@qinglong/runtime-core/run-attempt-log-read';
|
||||
import {
|
||||
MAX_REMOTE_WORKER_ARTIFACT_BYTES,
|
||||
REMOTE_WORKER_ARTIFACT_CONTENT_TYPE,
|
||||
@@ -31,7 +38,8 @@ const TEMPORARY_METADATA_SCHEMA =
|
||||
const SHA256_PATTERN = /^[a-f0-9]{64}$/;
|
||||
const BUCKET_PATTERN = /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/;
|
||||
const PREFIX_PATTERN = /^[A-Za-z0-9][A-Za-z0-9/_=-]{0,254}$/;
|
||||
const TEMPORARY_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
const TEMPORARY_ID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
|
||||
type S3SendClient = Pick<S3Client, 'send'>;
|
||||
|
||||
@@ -70,8 +78,7 @@ export function createS3ClusterRemoteWorkerArtifactClient(
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
!/^[a-z0-9][a-z0-9-]{0,62}$/.test(options.region) ||
|
||||
(options.endpoint !== undefined &&
|
||||
typeof options.endpoint !== 'string') ||
|
||||
(options.endpoint !== undefined && typeof options.endpoint !== 'string') ||
|
||||
(options.forcePathStyle !== undefined &&
|
||||
typeof options.forcePathStyle !== 'boolean')
|
||||
) {
|
||||
@@ -79,9 +86,7 @@ export function createS3ClusterRemoteWorkerArtifactClient(
|
||||
}
|
||||
return new S3Client({
|
||||
region: options.region,
|
||||
...(options.endpoint === undefined
|
||||
? {}
|
||||
: { endpoint: options.endpoint }),
|
||||
...(options.endpoint === undefined ? {} : { endpoint: options.endpoint }),
|
||||
forcePathStyle: options.forcePathStyle ?? false,
|
||||
});
|
||||
}
|
||||
@@ -116,6 +121,11 @@ type ArtifactAuthority = Readonly<{
|
||||
logArtifactId: string;
|
||||
}>;
|
||||
|
||||
type StoredArtifactHead = Readonly<{
|
||||
receipt: Readonly<RemoteWorkerArtifactReceipt>;
|
||||
eTag?: string;
|
||||
}>;
|
||||
|
||||
type NormalizedStorageCommand = ArtifactAuthority &
|
||||
Readonly<{
|
||||
byteLength: number;
|
||||
@@ -127,7 +137,9 @@ const DIAGNOSTIC_CONTEXT = Object.freeze({
|
||||
});
|
||||
|
||||
function configurationError(message: string): TypeError {
|
||||
return new TypeError(`S3 Remote Worker Artifact store is invalid: ${message}`);
|
||||
return new TypeError(
|
||||
`S3 Remote Worker Artifact store is invalid: ${message}`,
|
||||
);
|
||||
}
|
||||
|
||||
function prepareOptions(
|
||||
@@ -175,14 +187,15 @@ function prepareOptions(
|
||||
throw configurationError('expected bucket owner is invalid');
|
||||
}
|
||||
const encryption = options.encryption;
|
||||
if (!encryption || typeof encryption !== 'object' || Array.isArray(encryption)) {
|
||||
if (
|
||||
!encryption ||
|
||||
typeof encryption !== 'object' ||
|
||||
Array.isArray(encryption)
|
||||
) {
|
||||
throw configurationError('encryption is required');
|
||||
}
|
||||
let preparedEncryption: PreparedOptions['encryption'];
|
||||
if (
|
||||
encryption.mode === 's3' &&
|
||||
Object.keys(encryption).length === 1
|
||||
) {
|
||||
if (encryption.mode === 's3' && Object.keys(encryption).length === 1) {
|
||||
preparedEncryption = Object.freeze({
|
||||
ServerSideEncryption: ServerSideEncryption.AES256,
|
||||
});
|
||||
@@ -383,9 +396,8 @@ function finalMetadata(
|
||||
),
|
||||
'ql3-byte-length': String(command.byteLength),
|
||||
'ql3-content-sha256': sha256,
|
||||
'ql3-truncated': command.truncated === undefined
|
||||
? 'omitted'
|
||||
: String(command.truncated),
|
||||
'ql3-truncated':
|
||||
command.truncated === undefined ? 'omitted' : String(command.truncated),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -460,6 +472,99 @@ function parseStoredReceipt(
|
||||
);
|
||||
}
|
||||
|
||||
function canonicalETag(value: unknown): string {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 3 ||
|
||||
value.length > 256 ||
|
||||
!/^"[^"\u0000-\u001f\u007f]+"$/.test(value)
|
||||
) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function assertRangeMetadata(
|
||||
authority: ArtifactAuthority,
|
||||
receipt: Readonly<RemoteWorkerArtifactReceipt>,
|
||||
output: Readonly<{
|
||||
ContentLength?: number | undefined;
|
||||
ContentRange?: string | undefined;
|
||||
ContentType?: string | undefined;
|
||||
ETag?: string | undefined;
|
||||
Metadata?: Readonly<Record<string, string | undefined>> | undefined;
|
||||
}>,
|
||||
eTag: string,
|
||||
start: number,
|
||||
endExclusive: number,
|
||||
): void {
|
||||
const metadata = output.Metadata;
|
||||
const truncated =
|
||||
receipt.truncated === undefined ? 'omitted' : String(receipt.truncated);
|
||||
if (
|
||||
output.ContentLength !== endExclusive - start ||
|
||||
output.ContentRange !==
|
||||
`bytes ${start}-${endExclusive - 1}/${receipt.byteLength}` ||
|
||||
output.ContentType !== REMOTE_WORKER_ARTIFACT_CONTENT_TYPE ||
|
||||
canonicalETag(output.ETag) !== eTag ||
|
||||
metadataValue(metadata, 'ql3-schema') !== METADATA_SCHEMA ||
|
||||
metadataValue(metadata, 'ql3-project-sha256') !==
|
||||
fieldDigest('project', authority.projectId) ||
|
||||
metadataValue(metadata, 'ql3-run-sha256') !==
|
||||
fieldDigest('run', authority.runId) ||
|
||||
metadataValue(metadata, 'ql3-attempt-sha256') !==
|
||||
fieldDigest('attempt', authority.attemptId) ||
|
||||
metadataValue(metadata, 'ql3-log-artifact-sha256') !==
|
||||
fieldDigest('log-artifact', authority.logArtifactId) ||
|
||||
metadataValue(metadata, 'ql3-byte-length') !== String(receipt.byteLength) ||
|
||||
metadataValue(metadata, 'ql3-content-sha256') !== receipt.sha256 ||
|
||||
metadataValue(metadata, 'ql3-truncated') !== truncated
|
||||
) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
}
|
||||
|
||||
async function readBoundedRangeBody(
|
||||
body: unknown,
|
||||
expectedBytes: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Buffer> {
|
||||
if (
|
||||
!body ||
|
||||
typeof body !== 'object' ||
|
||||
!(Symbol.asyncIterator in body) ||
|
||||
typeof (body as AsyncIterable<Uint8Array>)[Symbol.asyncIterator] !==
|
||||
'function'
|
||||
) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
const content = Buffer.allocUnsafe(expectedBytes);
|
||||
let received = 0;
|
||||
try {
|
||||
for await (const chunk of body as AsyncIterable<Uint8Array>) {
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
if (!(chunk instanceof Uint8Array) || chunk.byteLength === 0) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
if (received + chunk.byteLength > expectedBytes) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength).copy(
|
||||
content,
|
||||
received,
|
||||
);
|
||||
received += chunk.byteLength;
|
||||
}
|
||||
if (received !== expectedBytes) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
return content;
|
||||
} catch (error) {
|
||||
content.fill(0);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function isNotFound(error: unknown): boolean {
|
||||
if (!error || typeof error !== 'object') return false;
|
||||
const value = error as {
|
||||
@@ -467,13 +572,17 @@ function isNotFound(error: unknown): boolean {
|
||||
Code?: unknown;
|
||||
$metadata?: { httpStatusCode?: unknown };
|
||||
};
|
||||
return value.name === 'NotFound' ||
|
||||
return (
|
||||
value.name === 'NotFound' ||
|
||||
value.name === 'NoSuchKey' ||
|
||||
value.Code === 'NoSuchKey' ||
|
||||
value.$metadata?.httpStatusCode === 404;
|
||||
value.$metadata?.httpStatusCode === 404
|
||||
);
|
||||
}
|
||||
|
||||
function requestOptions(signal?: AbortSignal): { abortSignal: AbortSignal } | undefined {
|
||||
function requestOptions(
|
||||
signal?: AbortSignal,
|
||||
): { abortSignal: AbortSignal } | undefined {
|
||||
return signal === undefined ? undefined : { abortSignal: signal };
|
||||
}
|
||||
|
||||
@@ -549,7 +658,8 @@ class ArtifactContentDigest {
|
||||
* objects are never overwritten or deleted by this adapter.
|
||||
*/
|
||||
export class S3ClusterRemoteWorkerArtifactStore
|
||||
implements ClusterRemoteWorkerArtifactStore {
|
||||
implements ClusterRemoteWorkerArtifactStore
|
||||
{
|
||||
private readonly options: PreparedOptions;
|
||||
|
||||
constructor(options: S3ClusterRemoteWorkerArtifactStoreOptions) {
|
||||
@@ -561,6 +671,90 @@ export class S3ClusterRemoteWorkerArtifactStore
|
||||
signal?: AbortSignal,
|
||||
): Promise<Readonly<RemoteWorkerArtifactReceipt> | undefined> {
|
||||
const authority = normalizeLookup(lookup);
|
||||
return (await this.head(authority, signal))?.receipt;
|
||||
}
|
||||
|
||||
async readLogRange(
|
||||
rawIdentity: Readonly<RunAttemptLogReadIdentity>,
|
||||
rawRange: Readonly<RunAttemptLogReadRange>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RunAttemptLogRangeReadResult> {
|
||||
const authority = normalizeLookup(rawIdentity);
|
||||
const range = normalizeRunAttemptLogReadRange(rawRange);
|
||||
const stored = await this.head(authority, signal);
|
||||
if (!stored) return Object.freeze({ status: 'missing' as const });
|
||||
const start = Math.min(range.offset, stored.receipt.byteLength);
|
||||
const endExclusive = Math.min(
|
||||
start + range.length,
|
||||
stored.receipt.byteLength,
|
||||
);
|
||||
const truncation = Object.freeze({
|
||||
truncated: stored.receipt.truncated ?? ('unknown' as const),
|
||||
});
|
||||
if (start === endExclusive) {
|
||||
return Object.freeze({
|
||||
status: 'available' as const,
|
||||
content: Buffer.alloc(0),
|
||||
start,
|
||||
endExclusive,
|
||||
totalBytes: stored.receipt.byteLength,
|
||||
truncation,
|
||||
});
|
||||
}
|
||||
const eTag = canonicalETag(stored.eTag);
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
try {
|
||||
const output = await this.options.client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: finalObjectKey(this.options.prefix, authority),
|
||||
IfMatch: eTag,
|
||||
Range: `bytes=${start}-${endExclusive - 1}`,
|
||||
...(this.options.expectedBucketOwner === undefined
|
||||
? {}
|
||||
: { ExpectedBucketOwner: this.options.expectedBucketOwner }),
|
||||
}),
|
||||
requestOptions(signal),
|
||||
);
|
||||
assertRangeMetadata(
|
||||
authority,
|
||||
stored.receipt,
|
||||
output,
|
||||
eTag,
|
||||
start,
|
||||
endExclusive,
|
||||
);
|
||||
const bytes = await readBoundedRangeBody(
|
||||
output.Body,
|
||||
endExclusive - start,
|
||||
signal,
|
||||
);
|
||||
return Object.freeze({
|
||||
status: 'available' as const,
|
||||
content: bytes,
|
||||
start,
|
||||
endExclusive,
|
||||
totalBytes: stored.receipt.byteLength,
|
||||
...(endExclusive < stored.receipt.byteLength
|
||||
? { nextOffset: endExclusive }
|
||||
: {}),
|
||||
truncation,
|
||||
});
|
||||
} catch (error) {
|
||||
if (isNotFound(error)) {
|
||||
return Object.freeze({ status: 'missing' as const });
|
||||
}
|
||||
if (error instanceof S3ClusterRemoteWorkerArtifactStoreError) throw error;
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('unavailable', {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async head(
|
||||
authority: ArtifactAuthority,
|
||||
signal?: AbortSignal,
|
||||
): Promise<StoredArtifactHead | undefined> {
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
try {
|
||||
const output = await this.options.client.send(
|
||||
@@ -574,7 +768,10 @@ export class S3ClusterRemoteWorkerArtifactStore
|
||||
}),
|
||||
requestOptions(signal),
|
||||
);
|
||||
return parseStoredReceipt(authority, output);
|
||||
return Object.freeze({
|
||||
receipt: parseStoredReceipt(authority, output),
|
||||
...(output.ETag === undefined ? {} : { eTag: output.ETag }),
|
||||
});
|
||||
} catch (error) {
|
||||
if (isNotFound(error)) return undefined;
|
||||
if (error instanceof S3ClusterRemoteWorkerArtifactStoreError) throw error;
|
||||
@@ -604,9 +801,7 @@ export class S3ClusterRemoteWorkerArtifactStore
|
||||
existing.truncated !== command.truncated ||
|
||||
existing.sha256 !== incomingSha256
|
||||
) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError(
|
||||
'integrity_mismatch',
|
||||
);
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
return existing;
|
||||
}
|
||||
@@ -693,13 +888,11 @@ export class S3ClusterRemoteWorkerArtifactStore
|
||||
stored.truncated !== command.truncated ||
|
||||
stored.sha256 !== sha256
|
||||
) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError(
|
||||
'integrity_mismatch',
|
||||
);
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
result = Object.freeze({
|
||||
...stored,
|
||||
status: copied ? 'stored' as const : 'already_stored' as const,
|
||||
status: copied ? ('stored' as const) : ('already_stored' as const),
|
||||
});
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
@@ -770,9 +963,7 @@ export class S3ClusterRemoteWorkerArtifactStore
|
||||
output.Metadata?.['ql3-owner-sha256'] !== ownerSha256 ||
|
||||
canonicalChecksum(output.ChecksumSHA256) !== sha256
|
||||
) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError(
|
||||
'integrity_mismatch',
|
||||
);
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,11 @@ import {
|
||||
type RemoteWorkerCompletionRepository,
|
||||
type RemoteWorkerCompletionResult,
|
||||
} from '@qinglong/runtime-core/remote-worker-completion';
|
||||
import type {
|
||||
RunAttemptLogRangeReadResult,
|
||||
RunAttemptLogReadIdentity,
|
||||
RunAttemptLogReadRange,
|
||||
} from '@qinglong/runtime-core/run-attempt-log-read';
|
||||
|
||||
export interface ClusterRemoteWorkerArtifactStorageCommand {
|
||||
readonly projectId: string;
|
||||
@@ -47,6 +52,12 @@ export interface ClusterRemoteWorkerArtifactStore {
|
||||
lookup: Readonly<ClusterRemoteWorkerArtifactLookup>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Readonly<RemoteWorkerArtifactReceipt> | undefined>;
|
||||
/** Optional during Alpha so upload-only test and alternate stores remain compatible. */
|
||||
readLogRange?(
|
||||
identity: Readonly<RunAttemptLogReadIdentity>,
|
||||
range: Readonly<RunAttemptLogReadRange>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RunAttemptLogRangeReadResult>;
|
||||
}
|
||||
|
||||
export interface ClusterRemoteWorkerArtifactUploadInput {
|
||||
@@ -353,11 +364,13 @@ export class ClusterRemoteWorkerCompletionService {
|
||||
}
|
||||
try {
|
||||
const result = normalizeRemoteWorkerCompletionResult(
|
||||
await this.repository.complete(Object.freeze({
|
||||
...command,
|
||||
attemptEventId: eventId(this.createEventId),
|
||||
runEventId: eventId(this.createEventId),
|
||||
})),
|
||||
await this.repository.complete(
|
||||
Object.freeze({
|
||||
...command,
|
||||
attemptEventId: eventId(this.createEventId),
|
||||
runEventId: eventId(this.createEventId),
|
||||
}),
|
||||
),
|
||||
);
|
||||
if (
|
||||
result.runId !== command.runId ||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// Remote execution owns the least-privilege assembly of Worker-facing runtime capabilities.
|
||||
import type { PostgresPool } from '@qinglong/runtime-core';
|
||||
import type { RemoteWorkerSecretValueProvider } from '@qinglong/runtime-core/remote-secret-delivery';
|
||||
import type { RunAttemptLogRangeReader } from '@qinglong/runtime-core/run-attempt-log-read';
|
||||
import {
|
||||
PostgresClusterDispatchSource,
|
||||
PostgresRemoteRunActivationRepository,
|
||||
@@ -11,23 +12,15 @@ import {
|
||||
PostgresTaskExecutionRevisionSource,
|
||||
PostgresWorkerSessionRepository,
|
||||
} from '@qinglong/cluster-postgres/runtime';
|
||||
import {
|
||||
ClusterRemoteWorkerOfferClaimService,
|
||||
} from './remoteWorkerDispatcher';
|
||||
import {
|
||||
ClusterRemoteRunActivationService,
|
||||
} from './remoteRunActivationService';
|
||||
import {
|
||||
ClusterRemoteWorkerSecretDeliveryService,
|
||||
} from './remoteWorkerSecretDeliveryService';
|
||||
import { ClusterRemoteWorkerOfferClaimService } from './remoteWorkerDispatcher';
|
||||
import { ClusterRemoteRunActivationService } from './remoteRunActivationService';
|
||||
import { ClusterRemoteWorkerSecretDeliveryService } from './remoteWorkerSecretDeliveryService';
|
||||
import {
|
||||
ClusterRemoteWorkerArtifactService,
|
||||
ClusterRemoteWorkerCompletionService,
|
||||
type ClusterRemoteWorkerArtifactStore,
|
||||
} from './remoteWorkerCompletionService';
|
||||
import {
|
||||
ClusterRemoteWorkerLeaseControlService,
|
||||
} from './remoteWorkerLeaseControlService';
|
||||
import { ClusterRemoteWorkerLeaseControlService } from './remoteWorkerLeaseControlService';
|
||||
import type { WorkerIngressPipelineOptions } from '../worker-ingress/workerIngressPipeline';
|
||||
|
||||
export interface ClusterWorkerRuntimeDependencies {
|
||||
@@ -49,6 +42,7 @@ export interface ClusterWorkerRuntimePort {
|
||||
readonly leaseControl: NonNullable<
|
||||
WorkerIngressPipelineOptions['leaseControl']
|
||||
>;
|
||||
readonly runAttemptLogRead?: RunAttemptLogRangeReader;
|
||||
}
|
||||
|
||||
export function createClusterWorkerRuntimePort(
|
||||
@@ -67,9 +61,11 @@ export function createClusterWorkerRuntimePort(
|
||||
}
|
||||
|
||||
const workerSessions = new PostgresWorkerSessionRepository(pool);
|
||||
const completionRepository =
|
||||
new PostgresRemoteWorkerCompletionRepository(pool);
|
||||
const completionRepository = new PostgresRemoteWorkerCompletionRepository(
|
||||
pool,
|
||||
);
|
||||
const secretProvider = dependencies.secretProvider;
|
||||
const readLogRange = dependencies.artifactStore.readLogRange;
|
||||
return Object.freeze({
|
||||
offers: new ClusterRemoteWorkerOfferClaimService(
|
||||
new PostgresClusterDispatchSource(pool),
|
||||
@@ -99,5 +95,12 @@ export function createClusterWorkerRuntimePort(
|
||||
leaseControl: new ClusterRemoteWorkerLeaseControlService(
|
||||
new PostgresRemoteWorkerLeaseControlRepository(pool),
|
||||
),
|
||||
...(readLogRange === undefined
|
||||
? {}
|
||||
: {
|
||||
runAttemptLogRead: Object.freeze({
|
||||
read: readLogRange.bind(dependencies.artifactStore),
|
||||
}),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
import {
|
||||
InvalidRunAttemptLogReadError,
|
||||
RunAttemptLogReadService,
|
||||
RunAttemptLogReadUnavailableError,
|
||||
type RunAttemptLogRangeReader,
|
||||
type RunAttemptLogReadResult,
|
||||
} from '@qinglong/runtime-core/run-attempt-log-read';
|
||||
import type { RunRepositoryReader } from '@qinglong/runtime-core/run-repository';
|
||||
|
||||
import type { ClusterControlAdmissionResponse } from '../transport/httpSurface';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
ClusterControlRouteParameters,
|
||||
} from '../transport/routeRegistry';
|
||||
|
||||
export const CLUSTER_CONTROL_RUN_ATTEMPT_LOG_READ_ROUTE = Object.freeze({
|
||||
method: 'GET' as const,
|
||||
path: '/api/v3/projects/{projectId}/runs/{runId}/attempts/{attemptId}/log',
|
||||
operationId: 'run.log.read',
|
||||
permission: 'artifact.read',
|
||||
projectParameter: 'projectId',
|
||||
allowedQuery: Object.freeze(['length', 'offset']),
|
||||
});
|
||||
|
||||
const DEFAULT_READ_BYTES = 64 * 1024;
|
||||
const MAXIMUM_READ_BYTES = 256 * 1024;
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
function parseQuery(
|
||||
query: Readonly<Record<string, readonly string[]>>,
|
||||
): Readonly<{ offset: number; length: number }> {
|
||||
const offsetValues = query.offset;
|
||||
const lengthValues = query.length;
|
||||
if (
|
||||
(offsetValues !== undefined && offsetValues.length !== 1) ||
|
||||
(lengthValues !== undefined && lengthValues.length !== 1)
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
const rawOffset = offsetValues?.[0];
|
||||
const offset = rawOffset === undefined ? 0 : Number(rawOffset);
|
||||
const rawLength = lengthValues?.[0];
|
||||
const length =
|
||||
rawLength === undefined ? DEFAULT_READ_BYTES : Number(rawLength);
|
||||
if (
|
||||
!Number.isSafeInteger(offset) ||
|
||||
offset < 0 ||
|
||||
(rawOffset !== undefined && String(offset) !== rawOffset) ||
|
||||
!Number.isSafeInteger(length) ||
|
||||
length < 1 ||
|
||||
length > MAXIMUM_READ_BYTES ||
|
||||
(rawLength !== undefined && String(length) !== rawLength)
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
return Object.freeze({ offset, length });
|
||||
}
|
||||
|
||||
function validateQuery(
|
||||
query: Readonly<Record<string, readonly string[]>>,
|
||||
): void {
|
||||
parseQuery(query);
|
||||
}
|
||||
|
||||
function projection(
|
||||
result: Extract<RunAttemptLogReadResult, { readonly status: 'available' }>,
|
||||
): Readonly<Record<string, unknown>> {
|
||||
return Object.freeze({
|
||||
schema: 'qinglong/run-attempt-log-read-result@v1',
|
||||
status: 'available',
|
||||
projectId: result.projectId,
|
||||
runId: result.runId,
|
||||
attemptId: result.attemptId,
|
||||
range: Object.freeze({
|
||||
start: result.start,
|
||||
endExclusive: result.endExclusive,
|
||||
totalBytes: result.totalBytes,
|
||||
...(result.nextOffset === undefined
|
||||
? {}
|
||||
: { nextOffset: result.nextOffset }),
|
||||
}),
|
||||
encoding: 'base64',
|
||||
content: Buffer.from(
|
||||
result.content.buffer,
|
||||
result.content.byteOffset,
|
||||
result.content.byteLength,
|
||||
).toString('base64'),
|
||||
truncation: result.truncation,
|
||||
});
|
||||
}
|
||||
|
||||
export function createClusterControlRunAttemptLogReadRoute(
|
||||
runs: Pick<RunRepositoryReader, 'findRunById' | 'findAttemptById'>,
|
||||
reader?: RunAttemptLogRangeReader,
|
||||
): Readonly<ClusterControlRouteDefinition> {
|
||||
if (
|
||||
!runs ||
|
||||
typeof runs.findRunById !== 'function' ||
|
||||
typeof runs.findAttemptById !== 'function' ||
|
||||
(reader !== undefined && typeof reader.read !== 'function')
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Cluster-control Run Attempt log read dependencies are invalid',
|
||||
);
|
||||
}
|
||||
const service =
|
||||
reader === undefined
|
||||
? undefined
|
||||
: new RunAttemptLogReadService(runs, reader, {
|
||||
executorType: 'remote_worker',
|
||||
artifactIdPattern: /^wlog-[a-f0-9]{30}$/,
|
||||
maximumReadBytes: MAXIMUM_READ_BYTES,
|
||||
activeMissingIsPending: true,
|
||||
});
|
||||
return Object.freeze({
|
||||
...CLUSTER_CONTROL_RUN_ATTEMPT_LOG_READ_ROUTE,
|
||||
validateQuery,
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
if (authorized.request.body !== null) {
|
||||
return response(400, { code: 'invalid_request_body' });
|
||||
}
|
||||
if (authorized.projectId === null || service === undefined) {
|
||||
return response(503, { code: 'artifact_unavailable' });
|
||||
}
|
||||
let range;
|
||||
try {
|
||||
range = parseQuery(authorized.request.query);
|
||||
} catch {
|
||||
return response(400, { code: 'invalid_run_log_read_query' });
|
||||
}
|
||||
try {
|
||||
const result = await service.read({
|
||||
projectId: authorized.projectId,
|
||||
runId: parameters.runId!,
|
||||
attemptId: parameters.attemptId!,
|
||||
range,
|
||||
signal: authorized.request.signal,
|
||||
});
|
||||
if (result.status === 'not_found') {
|
||||
return response(404, { code: 'artifact_not_found' });
|
||||
}
|
||||
if (result.status === 'pending') {
|
||||
return response(202, {
|
||||
schema: 'qinglong/run-attempt-log-read-result@v1',
|
||||
status: 'pending',
|
||||
projectId: result.projectId,
|
||||
runId: result.runId,
|
||||
attemptId: result.attemptId,
|
||||
});
|
||||
}
|
||||
if (result.status === 'missing') {
|
||||
return response(503, { code: 'artifact_unavailable' });
|
||||
}
|
||||
return response(200, projection(result));
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidRunAttemptLogReadError) {
|
||||
return response(400, { code: 'invalid_run_log_read_request' });
|
||||
}
|
||||
if (error instanceof RunAttemptLogReadUnavailableError) {
|
||||
return response(503, { code: 'artifact_unavailable' });
|
||||
}
|
||||
return response(503, { code: 'artifact_unavailable' });
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -308,6 +308,13 @@ export function createClusterControlAdmissionPipeline(
|
||||
now,
|
||||
);
|
||||
if (decision.effect === 'deny') {
|
||||
if (route.permission === 'artifact.read') {
|
||||
throw securityError(
|
||||
404,
|
||||
'artifact_not_found',
|
||||
'Cluster-control Artifact is not available',
|
||||
);
|
||||
}
|
||||
throw securityError(
|
||||
403,
|
||||
'forbidden',
|
||||
@@ -315,6 +322,13 @@ export function createClusterControlAdmissionPipeline(
|
||||
);
|
||||
}
|
||||
if (decision.effect === 'require_approval') {
|
||||
if (route.permission === 'artifact.read') {
|
||||
throw securityError(
|
||||
404,
|
||||
'artifact_not_found',
|
||||
'Cluster-control Artifact is not available',
|
||||
);
|
||||
}
|
||||
throw securityError(
|
||||
403,
|
||||
'approval_required',
|
||||
|
||||
@@ -573,6 +573,9 @@ test('injects reviewed Worker operations without exposing the runtime Pool', asy
|
||||
async inspect() {
|
||||
throw new Error('not invoked during assembly');
|
||||
},
|
||||
async readLogRange() {
|
||||
throw new Error('not invoked during assembly');
|
||||
},
|
||||
};
|
||||
const result = await bootstrapClusterControlRuntime(
|
||||
bootstrapOptions(events, {
|
||||
@@ -596,6 +599,10 @@ test('injects reviewed Worker operations without exposing the runtime Pool', asy
|
||||
typeof input.workerRuntime.leaseControl.control,
|
||||
'function',
|
||||
);
|
||||
assert.equal(
|
||||
typeof input.workerRuntime.runAttemptLogRead.read,
|
||||
'function',
|
||||
);
|
||||
return activationStack(events);
|
||||
},
|
||||
}),
|
||||
|
||||
@@ -143,6 +143,9 @@ function fixture(overrides = {}) {
|
||||
},
|
||||
];
|
||||
},
|
||||
async findAttemptById() {
|
||||
return null;
|
||||
},
|
||||
},
|
||||
trustedToolStorage: {
|
||||
stepRuns: {
|
||||
@@ -190,7 +193,8 @@ function fixture(overrides = {}) {
|
||||
taskDefinitions: {
|
||||
async findCurrentTaskDefinition(projectId, taskId) {
|
||||
events.push(`task-get:${projectId}:${taskId}`);
|
||||
return projectId === currentTask.projectId && taskId === currentTask.taskId
|
||||
return projectId === currentTask.projectId &&
|
||||
taskId === currentTask.taskId
|
||||
? currentTask
|
||||
: null;
|
||||
},
|
||||
@@ -328,6 +332,7 @@ test('production composition exposes the reviewed Run and Workflow routes', asyn
|
||||
'run.list',
|
||||
'run.events.list',
|
||||
'run.steps.list',
|
||||
'run.log.read',
|
||||
'run.cancel',
|
||||
'workflow.read',
|
||||
'workflow.run.read',
|
||||
@@ -401,6 +406,15 @@ test('production composition exposes the reviewed Run and Workflow routes', asyn
|
||||
body: { steps: [], hasMore: false, next: null },
|
||||
});
|
||||
|
||||
const log = await invoke(
|
||||
stack,
|
||||
metadata('/api/v3/projects/project-1/runs/run-1/attempts/attempt-1/log'),
|
||||
);
|
||||
assert.deepEqual(log, {
|
||||
statusCode: 503,
|
||||
body: { code: 'artifact_unavailable' },
|
||||
});
|
||||
|
||||
const cancellation = await invoke(
|
||||
stack,
|
||||
metadata('/api/v3/projects/project-1/runs/run-1/cancellation', 'POST', {
|
||||
@@ -414,6 +428,7 @@ test('production composition exposes the reviewed Run and Workflow routes', asyn
|
||||
assert.equal(events.includes('audit:run.get:allowed'), true);
|
||||
assert.equal(events.includes('audit:run.events.list:allowed'), true);
|
||||
assert.equal(events.includes('audit:run.steps.list:allowed'), true);
|
||||
assert.equal(events.includes('audit:run.log.read:allowed'), true);
|
||||
assert.equal(events.includes('audit:run.cancel:allowed'), true);
|
||||
|
||||
const workflows = await invoke(
|
||||
@@ -539,6 +554,69 @@ test('production composition fails closed for an unreviewed route', async () =>
|
||||
);
|
||||
});
|
||||
|
||||
test('wires the production Worker object reader into the Project-scoped log route', async () => {
|
||||
const { input } = fixture();
|
||||
const run = await input.runs.findRunById('run-1');
|
||||
const logArtifactId = `wlog-${'a'.repeat(30)}`;
|
||||
const stack = createProductionClusterControlApplicationStack({
|
||||
...input,
|
||||
runs: {
|
||||
...input.runs,
|
||||
async findRunById() {
|
||||
return { ...run, status: 'running' };
|
||||
},
|
||||
async findAttemptById() {
|
||||
return {
|
||||
id: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
attempt: 1,
|
||||
status: 'running',
|
||||
executorType: 'remote_worker',
|
||||
logArtifactId,
|
||||
callbackSequence: 0,
|
||||
createdAtMs: 1,
|
||||
};
|
||||
},
|
||||
},
|
||||
workerRuntime: {
|
||||
offers: { claimNext() {} },
|
||||
activation: {
|
||||
acknowledgeStarting() {},
|
||||
acknowledgeRunning() {},
|
||||
failStart() {},
|
||||
},
|
||||
artifacts: { upload() {} },
|
||||
completion: { complete() {} },
|
||||
leaseControl: { control() {} },
|
||||
runAttemptLogRead: {
|
||||
async read(identity, range) {
|
||||
assert.equal(identity.logArtifactId, logArtifactId);
|
||||
assert.deepEqual(range, { offset: 1, length: 4 });
|
||||
return {
|
||||
status: 'available',
|
||||
content: Buffer.from('prod'),
|
||||
start: 1,
|
||||
endExclusive: 5,
|
||||
totalBytes: 5,
|
||||
truncation: { truncated: false },
|
||||
};
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const result = await invoke(
|
||||
stack,
|
||||
metadata(
|
||||
'/api/v3/projects/project-1/runs/run-1/attempts/attempt-1/log',
|
||||
'GET',
|
||||
null,
|
||||
{ offset: ['1'], length: ['4'] },
|
||||
),
|
||||
);
|
||||
assert.equal(result.statusCode, 200);
|
||||
assert.equal(Buffer.from(result.body.content, 'base64').toString(), 'prod');
|
||||
});
|
||||
|
||||
test('optionally exposes Prompt execution behind shared admission and policy', async () => {
|
||||
const { events, input } = fixture();
|
||||
let command;
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
createClusterControlAdmissionPipeline,
|
||||
} = require('@qinglong/cluster-control/admission');
|
||||
const {
|
||||
createClusterControlRouteRegistry,
|
||||
} = require('@qinglong/cluster-control/routes');
|
||||
const {
|
||||
CLUSTER_CONTROL_RUN_ATTEMPT_LOG_READ_ROUTE,
|
||||
createClusterControlRunAttemptLogReadRoute,
|
||||
} = require('../dist/run/runAttemptLogReadRoute.js');
|
||||
|
||||
const PRINCIPAL = Object.freeze({
|
||||
subject: Object.freeze({ type: 'user', id: 'usr_viewer' }),
|
||||
authenticationId: 'session:viewer',
|
||||
authenticatedAtMs: 9_000,
|
||||
expiresAtMs: 11_000,
|
||||
assurance: 'single_factor',
|
||||
});
|
||||
|
||||
function run(overrides = {}) {
|
||||
return {
|
||||
id: 'run_123',
|
||||
projectId: 'prj_default',
|
||||
taskId: 'task_1',
|
||||
taskRevision: 'revision_1',
|
||||
triggerType: 'task_start',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
status: 'running',
|
||||
version: 2,
|
||||
eventSequence: 2,
|
||||
priority: 0,
|
||||
createdAtMs: 1,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function attempt(overrides = {}) {
|
||||
return {
|
||||
id: 'attempt_123',
|
||||
runId: 'run_123',
|
||||
attempt: 1,
|
||||
status: 'running',
|
||||
executorType: 'remote_worker',
|
||||
logArtifactId: `wlog-${'a'.repeat(30)}`,
|
||||
callbackSequence: 0,
|
||||
createdAtMs: 1,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function metadata(query = {}) {
|
||||
return Object.freeze({
|
||||
requestId: 'request-log-read',
|
||||
method: 'GET',
|
||||
path: '/api/v3/projects/prj_default/runs/run_123/attempts/attempt_123/log',
|
||||
query: Object.freeze(query),
|
||||
headers: Object.freeze({ authorization: 'Bearer opaque' }),
|
||||
signal: new AbortController().signal,
|
||||
});
|
||||
}
|
||||
|
||||
function pipeline(options = {}) {
|
||||
const events = options.events ?? [];
|
||||
const repository = options.repository ?? {
|
||||
async findRunById() {
|
||||
events.push('run');
|
||||
return run();
|
||||
},
|
||||
async findAttemptById() {
|
||||
events.push('attempt');
|
||||
return attempt();
|
||||
},
|
||||
};
|
||||
const reader = options.reader ?? {
|
||||
async read(identity, range) {
|
||||
events.push(`storage:${range.offset}:${range.length}`);
|
||||
return {
|
||||
status: 'available',
|
||||
content: Buffer.from('cluster-log'),
|
||||
start: range.offset,
|
||||
endExclusive: range.offset + 11,
|
||||
totalBytes: range.offset + 20,
|
||||
nextOffset: range.offset + 11,
|
||||
truncation: { truncated: false },
|
||||
};
|
||||
},
|
||||
};
|
||||
return createClusterControlAdmissionPipeline({
|
||||
routes: createClusterControlRouteRegistry([
|
||||
createClusterControlRunAttemptLogReadRoute(repository, reader),
|
||||
]),
|
||||
authenticator: {
|
||||
authenticate() {
|
||||
events.push('authenticate');
|
||||
return PRINCIPAL;
|
||||
},
|
||||
},
|
||||
policy: {
|
||||
authorize(request) {
|
||||
events.push(`authorize:${request.permission}`);
|
||||
return options.effect
|
||||
? { effect: options.effect, reasons: ['masked'], fence: null }
|
||||
: {
|
||||
effect: 'allow',
|
||||
reasons: ['role_grant'],
|
||||
fence: { projectVersion: 2, bindingVersion: 3 },
|
||||
};
|
||||
},
|
||||
},
|
||||
audit: {
|
||||
record(record) {
|
||||
events.push(`audit:${record.outcome}`);
|
||||
},
|
||||
},
|
||||
now: () => 10_000,
|
||||
});
|
||||
}
|
||||
|
||||
async function invoke(value, query = {}) {
|
||||
const prepared = await value.prepare(metadata(query));
|
||||
return prepared.handle(null);
|
||||
}
|
||||
|
||||
test('publishes the immutable Artifact-scoped route contract', () => {
|
||||
assert.deepEqual(CLUSTER_CONTROL_RUN_ATTEMPT_LOG_READ_ROUTE, {
|
||||
method: 'GET',
|
||||
path: '/api/v3/projects/{projectId}/runs/{runId}/attempts/{attemptId}/log',
|
||||
operationId: 'run.log.read',
|
||||
permission: 'artifact.read',
|
||||
projectParameter: 'projectId',
|
||||
allowedQuery: ['length', 'offset'],
|
||||
});
|
||||
});
|
||||
|
||||
test('authorizes and audits before metadata and one bounded range read', async () => {
|
||||
const events = [];
|
||||
const result = await invoke(pipeline({ events }), {
|
||||
offset: ['4'],
|
||||
length: ['16'],
|
||||
});
|
||||
assert.equal(result.statusCode, 200);
|
||||
assert.equal(result.body.schema, 'qinglong/run-attempt-log-read-result@v1');
|
||||
assert.equal(
|
||||
Buffer.from(result.body.content, 'base64').toString(),
|
||||
'cluster-log',
|
||||
);
|
||||
assert.deepEqual(result.body.range, {
|
||||
start: 4,
|
||||
endExclusive: 15,
|
||||
totalBytes: 24,
|
||||
nextOffset: 15,
|
||||
});
|
||||
assert.deepEqual(events, [
|
||||
'authenticate',
|
||||
'authorize:artifact.read',
|
||||
'audit:allowed',
|
||||
'run',
|
||||
'attempt',
|
||||
'storage:4:16',
|
||||
]);
|
||||
});
|
||||
|
||||
test('uses the Cluster default window and rejects unbounded query values', async () => {
|
||||
const events = [];
|
||||
assert.equal((await invoke(pipeline({ events }))).statusCode, 200);
|
||||
assert.equal(events.at(-1), `storage:0:${64 * 1024}`);
|
||||
for (const query of [
|
||||
{ offset: ['-1'] },
|
||||
{ offset: ['04'] },
|
||||
{ length: ['0'] },
|
||||
{ length: [String(256 * 1024 + 1)] },
|
||||
{ length: ['1', '2'] },
|
||||
]) {
|
||||
await assert.rejects(
|
||||
pipeline().prepare(metadata(query)),
|
||||
(error) =>
|
||||
error.statusCode === 400 && error.code === 'invalid_route_query',
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('masks deny and approval without reading Run or object storage', async () => {
|
||||
for (const effect of ['deny', 'require_approval']) {
|
||||
let touched = false;
|
||||
await assert.rejects(
|
||||
pipeline({
|
||||
effect,
|
||||
repository: {
|
||||
async findRunById() {
|
||||
touched = true;
|
||||
return run();
|
||||
},
|
||||
async findAttemptById() {
|
||||
touched = true;
|
||||
return attempt();
|
||||
},
|
||||
},
|
||||
reader: {
|
||||
async read() {
|
||||
touched = true;
|
||||
return { status: 'missing' };
|
||||
},
|
||||
},
|
||||
}).prepare(metadata()),
|
||||
(error) =>
|
||||
error.statusCode === 404 && error.code === 'artifact_not_found',
|
||||
);
|
||||
assert.equal(touched, false);
|
||||
}
|
||||
});
|
||||
|
||||
test('returns pending during upload and fails closed without an object reader', async () => {
|
||||
const pending = pipeline({
|
||||
reader: {
|
||||
async read() {
|
||||
return { status: 'missing' };
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.equal((await invoke(pending)).statusCode, 202);
|
||||
|
||||
const unavailableRoute = createClusterControlRunAttemptLogReadRoute({
|
||||
async findRunById() {
|
||||
return run();
|
||||
},
|
||||
async findAttemptById() {
|
||||
return attempt();
|
||||
},
|
||||
});
|
||||
const prepared = await createClusterControlAdmissionPipeline({
|
||||
routes: createClusterControlRouteRegistry([unavailableRoute]),
|
||||
authenticator: { authenticate: () => PRINCIPAL },
|
||||
policy: {
|
||||
authorize: () => ({
|
||||
effect: 'allow',
|
||||
reasons: ['role_grant'],
|
||||
fence: { projectVersion: 1, bindingVersion: 1 },
|
||||
}),
|
||||
},
|
||||
audit: { record() {} },
|
||||
now: () => 10_000,
|
||||
}).prepare(metadata());
|
||||
assert.deepEqual(await prepared.handle(null), {
|
||||
statusCode: 503,
|
||||
body: { code: 'artifact_unavailable' },
|
||||
});
|
||||
});
|
||||
@@ -19,80 +19,111 @@ const endpoint = process.env.QL3_TEST_S3_ENDPOINT;
|
||||
const accessKeyId = process.env.QL3_TEST_S3_ACCESS_KEY_ID;
|
||||
const secretAccessKey = process.env.QL3_TEST_S3_SECRET_ACCESS_KEY;
|
||||
|
||||
test('real S3-compatible service preserves immutable Artifact evidence', {
|
||||
skip: endpoint && accessKeyId && secretAccessKey
|
||||
? false
|
||||
: 'requires QL3_TEST_S3_ENDPOINT and credentials',
|
||||
}, async () => {
|
||||
const client = new S3Client({
|
||||
endpoint,
|
||||
region: 'us-east-1',
|
||||
forcePathStyle: true,
|
||||
credentials: { accessKeyId, secretAccessKey },
|
||||
});
|
||||
const bucket = `ql3-artifact-${process.pid}-${Date.now()}`.slice(0, 63);
|
||||
const command = Object.freeze({
|
||||
projectId: 'project-s3-integration',
|
||||
runId: 'run-s3-integration',
|
||||
attemptId: 'attempt-s3-integration',
|
||||
logArtifactId: `wlog-${'c'.repeat(30)}`,
|
||||
byteLength: 17,
|
||||
truncated: true,
|
||||
});
|
||||
const content = Buffer.from('real object bytes');
|
||||
const body = (value) => Object.freeze({
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield value.subarray(0, 4);
|
||||
yield value.subarray(4);
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
await client.send(new CreateBucketCommand({ Bucket: bucket }));
|
||||
const store = new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket,
|
||||
prefix: 'qinglong/integration',
|
||||
encryption: { mode: 's3' },
|
||||
test(
|
||||
'real S3-compatible service preserves immutable Artifact evidence',
|
||||
{
|
||||
skip:
|
||||
endpoint && accessKeyId && secretAccessKey
|
||||
? false
|
||||
: 'requires QL3_TEST_S3_ENDPOINT and credentials',
|
||||
},
|
||||
async () => {
|
||||
const client = new S3Client({
|
||||
endpoint,
|
||||
region: 'us-east-1',
|
||||
forcePathStyle: true,
|
||||
credentials: { accessKeyId, secretAccessKey },
|
||||
});
|
||||
const stored = await store.put(command, body(content));
|
||||
assert.equal(stored.status, 'stored');
|
||||
assert.equal(
|
||||
stored.sha256,
|
||||
createHash('sha256').update(content).digest('hex'),
|
||||
);
|
||||
const replay = await store.put(command, body(content));
|
||||
assert.equal(replay.status, 'already_stored');
|
||||
await assert.rejects(
|
||||
store.put(command, body(Buffer.from('REAL OBJECT BYTES'))),
|
||||
(error) =>
|
||||
error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
|
||||
error.reason === 'integrity_mismatch',
|
||||
);
|
||||
const objects = await client.send(new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
Prefix: 'qinglong/integration/',
|
||||
}));
|
||||
assert.equal(objects.KeyCount, 1);
|
||||
assert.match(objects.Contents[0].Key, /\/objects\//);
|
||||
} finally {
|
||||
const bucket = `ql3-artifact-${process.pid}-${Date.now()}`.slice(0, 63);
|
||||
const command = Object.freeze({
|
||||
projectId: 'project-s3-integration',
|
||||
runId: 'run-s3-integration',
|
||||
attemptId: 'attempt-s3-integration',
|
||||
logArtifactId: `wlog-${'c'.repeat(30)}`,
|
||||
byteLength: 17,
|
||||
truncated: true,
|
||||
});
|
||||
const content = Buffer.from('real object bytes');
|
||||
const body = (value) =>
|
||||
Object.freeze({
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield value.subarray(0, 4);
|
||||
yield value.subarray(4);
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const objects = await client.send(new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
}));
|
||||
if (objects.Contents?.length) {
|
||||
await client.send(new DeleteObjectsCommand({
|
||||
await client.send(new CreateBucketCommand({ Bucket: bucket }));
|
||||
const store = new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket,
|
||||
prefix: 'qinglong/integration',
|
||||
encryption: { mode: 's3' },
|
||||
});
|
||||
const stored = await store.put(command, body(content));
|
||||
assert.equal(stored.status, 'stored');
|
||||
assert.equal(
|
||||
stored.sha256,
|
||||
createHash('sha256').update(content).digest('hex'),
|
||||
);
|
||||
const replay = await store.put(command, body(content));
|
||||
assert.equal(replay.status, 'already_stored');
|
||||
const range = await store.readLogRange(
|
||||
{
|
||||
projectId: command.projectId,
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
logArtifactId: command.logArtifactId,
|
||||
},
|
||||
{
|
||||
offset: 5,
|
||||
length: 6,
|
||||
},
|
||||
);
|
||||
assert.equal(range.status, 'available');
|
||||
assert.equal(Buffer.from(range.content).toString(), 'object');
|
||||
assert.equal(range.start, 5);
|
||||
assert.equal(range.endExclusive, 11);
|
||||
assert.equal(range.totalBytes, content.byteLength);
|
||||
assert.equal(range.nextOffset, 11);
|
||||
assert.deepEqual(range.truncation, { truncated: true });
|
||||
await assert.rejects(
|
||||
store.put(command, body(Buffer.from('REAL OBJECT BYTES'))),
|
||||
(error) =>
|
||||
error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
|
||||
error.reason === 'integrity_mismatch',
|
||||
);
|
||||
const objects = await client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
Delete: {
|
||||
Objects: objects.Contents.map(({ Key }) => ({ Key })),
|
||||
Quiet: true,
|
||||
},
|
||||
}));
|
||||
Prefix: 'qinglong/integration/',
|
||||
}),
|
||||
);
|
||||
assert.equal(objects.KeyCount, 1);
|
||||
assert.match(objects.Contents[0].Key, /\/objects\//);
|
||||
} finally {
|
||||
try {
|
||||
const objects = await client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
}),
|
||||
);
|
||||
if (objects.Contents?.length) {
|
||||
await client.send(
|
||||
new DeleteObjectsCommand({
|
||||
Bucket: bucket,
|
||||
Delete: {
|
||||
Objects: objects.Contents.map(({ Key }) => ({ Key })),
|
||||
Quiet: true,
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
await client.send(new DeleteBucketCommand({ Bucket: bucket }));
|
||||
} catch {
|
||||
// Preserve the integration assertion; the ephemeral container is removed.
|
||||
}
|
||||
await client.send(new DeleteBucketCommand({ Bucket: bucket }));
|
||||
} catch {
|
||||
// Preserve the integration assertion; the ephemeral container is removed.
|
||||
client.destroy();
|
||||
}
|
||||
client.destroy();
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
@@ -6,6 +6,7 @@ const { test } = require('node:test');
|
||||
const {
|
||||
CopyObjectCommand,
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
PutObjectCommand,
|
||||
} = require('@aws-sdk/client-s3');
|
||||
@@ -57,19 +58,58 @@ class MemoryS3Client {
|
||||
const object = this.objects.get(input.Key);
|
||||
if (!object) throw notFound();
|
||||
const metadata = { ...object.metadata };
|
||||
if (this.options.corruptFinalMetadata && input.Key.includes('/objects/')) {
|
||||
if (
|
||||
this.options.corruptFinalMetadata &&
|
||||
input.Key.includes('/objects/')
|
||||
) {
|
||||
metadata['ql3-content-sha256'] = '0'.repeat(64);
|
||||
}
|
||||
return {
|
||||
ContentLength: object.content.byteLength,
|
||||
ContentType: object.contentType,
|
||||
ChecksumSHA256: this.options.corruptFinalChecksum &&
|
||||
input.Key.includes('/objects/')
|
||||
? Buffer.alloc(32, 9).toString('base64')
|
||||
: checksum(object.content),
|
||||
ETag: `"${checksum(object.content).slice(0, 32)}"`,
|
||||
ChecksumSHA256:
|
||||
this.options.corruptFinalChecksum && input.Key.includes('/objects/')
|
||||
? Buffer.alloc(32, 9).toString('base64')
|
||||
: checksum(object.content),
|
||||
Metadata: metadata,
|
||||
};
|
||||
}
|
||||
if (command instanceof GetObjectCommand) {
|
||||
const object = this.objects.get(input.Key);
|
||||
if (!object || this.options.rangeNotFound) throw notFound();
|
||||
const eTag = `"${checksum(object.content).slice(0, 32)}"`;
|
||||
assert.equal(input.IfMatch, eTag);
|
||||
const match = /^bytes=(\d+)-(\d+)$/.exec(input.Range);
|
||||
assert.ok(match);
|
||||
const start = Number(match[1]);
|
||||
const end = Number(match[2]);
|
||||
let content = object.content.subarray(start, end + 1);
|
||||
if (this.options.shortRangeBody) content = content.subarray(0, -1);
|
||||
if (this.options.oversizedRangeBody) {
|
||||
content = Buffer.concat([content, Buffer.from('x')]);
|
||||
}
|
||||
const metadata = { ...object.metadata };
|
||||
if (this.options.corruptRangeMetadata) {
|
||||
metadata['ql3-run-sha256'] = '0'.repeat(64);
|
||||
}
|
||||
return {
|
||||
ContentLength: end - start + 1,
|
||||
ContentRange: this.options.corruptContentRange
|
||||
? `bytes ${start}-${end}/${object.content.byteLength + 1}`
|
||||
: `bytes ${start}-${end}/${object.content.byteLength}`,
|
||||
ContentType: object.contentType,
|
||||
ETag: this.options.corruptRangeETag ? '"other"' : eTag,
|
||||
Metadata: metadata,
|
||||
Body: {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
const split = Math.min(2, content.byteLength);
|
||||
if (split > 0) yield content.subarray(0, split);
|
||||
if (split < content.byteLength) yield content.subarray(split);
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (command instanceof PutObjectCommand) {
|
||||
assert.equal(input.IfNoneMatch, '*');
|
||||
assert.equal(input.ChecksumAlgorithm, 'SHA256');
|
||||
@@ -183,14 +223,25 @@ test('streams to a checksummed temporary object then conditionally promotes it',
|
||||
],
|
||||
);
|
||||
const key = permanentKey(client);
|
||||
assert.match(key, /^tenant-a\/worker-artifacts\/objects\/[a-f0-9]{2}\/[a-f0-9]{64}$/);
|
||||
assert.match(
|
||||
key,
|
||||
/^tenant-a\/worker-artifacts\/objects\/[a-f0-9]{2}\/[a-f0-9]{64}$/,
|
||||
);
|
||||
assert.equal(key.includes(COMMAND.runId), false);
|
||||
assert.equal(client.objects.size, 1);
|
||||
|
||||
const copy = client.commands.find((command) => command instanceof CopyObjectCommand);
|
||||
const copy = client.commands.find(
|
||||
(command) => command instanceof CopyObjectCommand,
|
||||
);
|
||||
assert.equal(copy.input.Metadata['ql3-content-sha256'], CONTENT_SHA256);
|
||||
assert.equal(JSON.stringify(copy.input.Metadata).includes(COMMAND.projectId), false);
|
||||
assert.equal(JSON.stringify(copy.input.Metadata).includes(COMMAND.runId), false);
|
||||
assert.equal(
|
||||
JSON.stringify(copy.input.Metadata).includes(COMMAND.projectId),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
JSON.stringify(copy.input.Metadata).includes(COMMAND.runId),
|
||||
false,
|
||||
);
|
||||
|
||||
const inspected = await adapter.inspect(LOOKUP);
|
||||
assert.deepEqual(inspected, { ...receipt, status: 'already_stored' });
|
||||
@@ -219,6 +270,78 @@ test('exact replay consumes and hashes the whole body without another write', as
|
||||
);
|
||||
});
|
||||
|
||||
test('reads only one ETag-fenced immutable byte range and stable end snapshot', async () => {
|
||||
const client = new MemoryS3Client();
|
||||
const adapter = store(client);
|
||||
await adapter.put(COMMAND, chunks());
|
||||
client.commands.length = 0;
|
||||
|
||||
const result = await adapter.readLogRange(LOOKUP, { offset: 2, length: 4 });
|
||||
assert.equal(result.status, 'available');
|
||||
assert.equal(Buffer.from(result.content).toString(), 'llo ');
|
||||
assert.deepEqual(
|
||||
{
|
||||
start: result.start,
|
||||
endExclusive: result.endExclusive,
|
||||
totalBytes: result.totalBytes,
|
||||
nextOffset: result.nextOffset,
|
||||
truncation: result.truncation,
|
||||
},
|
||||
{
|
||||
start: 2,
|
||||
endExclusive: 6,
|
||||
totalBytes: 11,
|
||||
nextOffset: 6,
|
||||
truncation: { truncated: false },
|
||||
},
|
||||
);
|
||||
assert.deepEqual(
|
||||
client.commands.map((command) => command.constructor.name),
|
||||
['HeadObjectCommand', 'GetObjectCommand'],
|
||||
);
|
||||
assert.equal(client.commands[1].input.Range, 'bytes=2-5');
|
||||
|
||||
client.commands.length = 0;
|
||||
const ended = await adapter.readLogRange(LOOKUP, {
|
||||
offset: 999,
|
||||
length: 4,
|
||||
});
|
||||
assert.equal(ended.status, 'available');
|
||||
assert.equal(ended.content.byteLength, 0);
|
||||
assert.equal(ended.start, 11);
|
||||
assert.equal(ended.totalBytes, 11);
|
||||
assert.deepEqual(
|
||||
client.commands.map((command) => command.constructor.name),
|
||||
['HeadObjectCommand'],
|
||||
);
|
||||
});
|
||||
|
||||
test('maps absent objects and fails closed on range evidence drift', async () => {
|
||||
const absent = new MemoryS3Client();
|
||||
assert.deepEqual(
|
||||
await store(absent).readLogRange(LOOKUP, { offset: 0, length: 1 }),
|
||||
{ status: 'missing' },
|
||||
);
|
||||
for (const option of [
|
||||
'corruptContentRange',
|
||||
'corruptRangeETag',
|
||||
'corruptRangeMetadata',
|
||||
'shortRangeBody',
|
||||
'oversizedRangeBody',
|
||||
]) {
|
||||
const client = new MemoryS3Client();
|
||||
const adapter = store(client);
|
||||
await adapter.put(COMMAND, chunks());
|
||||
client.options[option] = true;
|
||||
await assert.rejects(
|
||||
adapter.readLogRange(LOOKUP, { offset: 0, length: 4 }),
|
||||
(error) =>
|
||||
error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
|
||||
error.reason === 'integrity_mismatch',
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('resolves a concurrent conditional-copy winner by immutable inspect', async () => {
|
||||
const client = new MemoryS3Client({ raceOnCopy: true });
|
||||
const receipt = await store(client).put(COMMAND, chunks());
|
||||
@@ -270,38 +393,40 @@ test('temporary cleanup failure is diagnostic and never reverses promotion', asy
|
||||
},
|
||||
}).put(COMMAND, chunks());
|
||||
assert.equal(receipt.status, 'stored');
|
||||
assert.deepEqual(diagnostics, [[
|
||||
'delete unavailable',
|
||||
'temporary_object_cleanup',
|
||||
]]);
|
||||
assert.deepEqual(diagnostics, [
|
||||
['delete unavailable', 'temporary_object_cleanup'],
|
||||
]);
|
||||
assert.equal(client.objects.size, 2);
|
||||
});
|
||||
|
||||
test('requires exact bucket, prefix, encryption and temporary ID configuration', async () => {
|
||||
const client = new MemoryS3Client();
|
||||
assert.throws(
|
||||
() => new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket: 'Invalid_Bucket',
|
||||
encryption: { mode: 's3' },
|
||||
}),
|
||||
() =>
|
||||
new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket: 'Invalid_Bucket',
|
||||
encryption: { mode: 's3' },
|
||||
}),
|
||||
/bucket is invalid/,
|
||||
);
|
||||
assert.throws(
|
||||
() => new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket: 'valid-bucket',
|
||||
prefix: '../escape',
|
||||
encryption: { mode: 's3' },
|
||||
}),
|
||||
() =>
|
||||
new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket: 'valid-bucket',
|
||||
prefix: '../escape',
|
||||
encryption: { mode: 's3' },
|
||||
}),
|
||||
/prefix is invalid/,
|
||||
);
|
||||
assert.throws(
|
||||
() => new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket: 'valid-bucket',
|
||||
encryption: { mode: 'kms' },
|
||||
}),
|
||||
() =>
|
||||
new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket: 'valid-bucket',
|
||||
encryption: { mode: 'kms' },
|
||||
}),
|
||||
/encryption is invalid/,
|
||||
);
|
||||
await assert.rejects(
|
||||
@@ -355,8 +480,7 @@ test('propagates KMS and expected-owner fences to both sides of promotion', asyn
|
||||
|
||||
test('never deletes a colliding temporary object it cannot prove it owns', async () => {
|
||||
const client = new MemoryS3Client();
|
||||
const temporaryKey =
|
||||
`tenant-a/worker-artifacts/temporary/${TEMPORARY_ID}`;
|
||||
const temporaryKey = `tenant-a/worker-artifacts/temporary/${TEMPORARY_ID}`;
|
||||
client.objects.set(temporaryKey, {
|
||||
content: Buffer.from('other operation'),
|
||||
contentType: 'application/octet-stream',
|
||||
|
||||
@@ -22,6 +22,7 @@ import type { LocalApiRunListRoute } from '../run/runListRoute';
|
||||
import type { LocalApiRunReadRoute } from '../run/runReadRoute';
|
||||
import type { LocalApiRunStepListRoute } from '../run/runStepListRoute';
|
||||
import type { LocalApiRunCancellationRoute } from '../run/runCancellationRoute';
|
||||
import type { LocalApiRunAttemptLogReadRoute } from '../run/runAttemptLogReadRoute';
|
||||
import type { LocalApiTaskListRoute } from '../task/taskListRoute';
|
||||
import type { LocalApiTaskReadRoute } from '../task/taskReadRoute';
|
||||
import type { LocalApiTaskStartRoute } from '../task/taskStartRoute';
|
||||
@@ -50,6 +51,14 @@ export type LocalApiAdmissionOperation =
|
||||
runId: string;
|
||||
input: Readonly<BoundedRunStepListInput>;
|
||||
}>
|
||||
| Readonly<{
|
||||
operationId: 'run.log.read';
|
||||
projectId: string;
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
offset: number;
|
||||
length: number;
|
||||
}>
|
||||
| Readonly<{
|
||||
operationId: 'run.cancel';
|
||||
projectId: string;
|
||||
@@ -99,6 +108,7 @@ export interface LocalApiAdmissionOptions {
|
||||
readonly runEventListRoute: LocalApiRunEventListRoute;
|
||||
readonly runStepListRoute: LocalApiRunStepListRoute;
|
||||
readonly runCancellationRoute: LocalApiRunCancellationRoute;
|
||||
readonly runAttemptLogReadRoute: LocalApiRunAttemptLogReadRoute;
|
||||
readonly taskListRoute: LocalApiTaskListRoute;
|
||||
readonly taskReadRoute: LocalApiTaskReadRoute;
|
||||
readonly taskStartRoute: LocalApiTaskStartRoute;
|
||||
@@ -176,6 +186,7 @@ export function createLocalApiAdmission(
|
||||
typeof options.runEventListRoute?.handle !== 'function' ||
|
||||
typeof options.runStepListRoute?.handle !== 'function' ||
|
||||
typeof options.runCancellationRoute?.handle !== 'function' ||
|
||||
typeof options.runAttemptLogReadRoute?.handle !== 'function' ||
|
||||
typeof options.taskListRoute?.handle !== 'function' ||
|
||||
typeof options.taskReadRoute?.handle !== 'function' ||
|
||||
typeof options.taskStartRoute?.handle !== 'function' ||
|
||||
@@ -236,12 +247,14 @@ export function createLocalApiAdmission(
|
||||
request.operation.projectId,
|
||||
request.operation.operationId === 'run.cancel'
|
||||
? 'run.stop'
|
||||
: request.operation.operationId === 'run.log.read'
|
||||
? 'artifact.read'
|
||||
: request.operation.operationId === 'task.start'
|
||||
? 'run.start'
|
||||
? 'run.start'
|
||||
: request.operation.operationId === 'task.list' ||
|
||||
request.operation.operationId === 'task.get'
|
||||
? 'task.read'
|
||||
: 'run.read',
|
||||
request.operation.operationId === 'task.get'
|
||||
? 'task.read'
|
||||
: 'run.read',
|
||||
),
|
||||
);
|
||||
} catch {
|
||||
@@ -279,9 +292,15 @@ export function createLocalApiAdmission(
|
||||
),
|
||||
);
|
||||
if (auditFailure) return auditFailure;
|
||||
if (decision.effect === 'deny') return response(403, 'forbidden');
|
||||
if (decision.effect === 'deny') {
|
||||
return request.operation.operationId === 'run.log.read'
|
||||
? response(404, 'artifact_not_found')
|
||||
: response(403, 'forbidden');
|
||||
}
|
||||
if (decision.effect === 'require_approval') {
|
||||
return response(403, 'approval_required');
|
||||
return request.operation.operationId === 'run.log.read'
|
||||
? response(404, 'artifact_not_found')
|
||||
: response(403, 'approval_required');
|
||||
}
|
||||
if (request.signal.aborted) return response(503, 'request_unavailable');
|
||||
try {
|
||||
@@ -337,6 +356,16 @@ export function createLocalApiAdmission(
|
||||
principal: authenticated.principal,
|
||||
policyFence: decision.fence,
|
||||
});
|
||||
case 'run.log.read':
|
||||
if (body !== null) return response(400, 'invalid_request_body');
|
||||
return options.runAttemptLogReadRoute.handle({
|
||||
projectId: request.operation.projectId,
|
||||
runId: request.operation.runId,
|
||||
attemptId: request.operation.attemptId,
|
||||
offset: request.operation.offset,
|
||||
length: request.operation.length,
|
||||
signal: request.signal,
|
||||
});
|
||||
case 'task.list':
|
||||
if (body !== null) return response(400, 'invalid_request_body');
|
||||
return options.taskListRoute.handle({
|
||||
|
||||
@@ -15,6 +15,7 @@ import { createLocalApiRunReadRoute } from '../run/runReadRoute';
|
||||
import { createLocalApiRunEventListRoute } from '../run/runEventListRoute';
|
||||
import { createLocalApiRunStepListRoute } from '../run/runStepListRoute';
|
||||
import { createLocalApiRunCancellationRoute } from '../run/runCancellationRoute';
|
||||
import { createLocalApiRunAttemptLogReadRoute } from '../run/runAttemptLogReadRoute';
|
||||
import { createLocalApiTaskListRoute } from '../task/taskListRoute';
|
||||
import { createLocalApiTaskReadRoute } from '../task/taskReadRoute';
|
||||
import { createLocalApiTaskStartRoute } from '../task/taskStartRoute';
|
||||
@@ -109,6 +110,9 @@ export function createLocalApiProductSurface(
|
||||
authority.runCancellation,
|
||||
options.randomUuid ?? randomUUID,
|
||||
);
|
||||
const runAttemptLogReadRoute = createLocalApiRunAttemptLogReadRoute(
|
||||
authority.runAttemptLogRead,
|
||||
);
|
||||
const taskListRoute = createLocalApiTaskListRoute(
|
||||
authority.taskDefinitions,
|
||||
);
|
||||
@@ -128,6 +132,7 @@ export function createLocalApiProductSurface(
|
||||
runEventListRoute,
|
||||
runStepListRoute,
|
||||
runCancellationRoute,
|
||||
runAttemptLogReadRoute,
|
||||
taskListRoute,
|
||||
taskReadRoute,
|
||||
taskStartRoute,
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import {
|
||||
InvalidRunAttemptLogReadError,
|
||||
RunAttemptLogReadUnavailableError,
|
||||
type RunAttemptLogReadRequest,
|
||||
type RunAttemptLogReadResult,
|
||||
} from '@qinglong/runtime-core/run-attempt-log-read';
|
||||
|
||||
import type { LocalApiResponse } from '../transport/contract';
|
||||
|
||||
export interface LocalApiRunAttemptLogReadCapability {
|
||||
read(
|
||||
request: Readonly<RunAttemptLogReadRequest>,
|
||||
): Promise<RunAttemptLogReadResult>;
|
||||
}
|
||||
|
||||
export interface LocalApiRunAttemptLogReadRequest {
|
||||
readonly projectId: string;
|
||||
readonly runId: string;
|
||||
readonly attemptId: string;
|
||||
readonly offset: number;
|
||||
readonly length: number;
|
||||
readonly signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface LocalApiRunAttemptLogReadRoute {
|
||||
handle(
|
||||
request: Readonly<LocalApiRunAttemptLogReadRequest>,
|
||||
): Promise<LocalApiResponse>;
|
||||
}
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): LocalApiResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
function projection(
|
||||
result: Extract<RunAttemptLogReadResult, { readonly status: 'available' }>,
|
||||
): Readonly<Record<string, unknown>> {
|
||||
return Object.freeze({
|
||||
schema: 'qinglong/run-attempt-log-read-result@v1',
|
||||
status: 'available',
|
||||
projectId: result.projectId,
|
||||
runId: result.runId,
|
||||
attemptId: result.attemptId,
|
||||
range: Object.freeze({
|
||||
start: result.start,
|
||||
endExclusive: result.endExclusive,
|
||||
totalBytes: result.totalBytes,
|
||||
...(result.nextOffset === undefined
|
||||
? {}
|
||||
: { nextOffset: result.nextOffset }),
|
||||
}),
|
||||
encoding: 'base64',
|
||||
content: Buffer.from(
|
||||
result.content.buffer,
|
||||
result.content.byteOffset,
|
||||
result.content.byteLength,
|
||||
).toString('base64'),
|
||||
truncation: result.truncation,
|
||||
});
|
||||
}
|
||||
|
||||
export function createLocalApiRunAttemptLogReadRoute(
|
||||
capability: LocalApiRunAttemptLogReadCapability,
|
||||
): Readonly<LocalApiRunAttemptLogReadRoute> {
|
||||
if (!capability || typeof capability.read !== 'function') {
|
||||
throw new TypeError('Local API Run Attempt log read capability is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
async handle(request: Readonly<LocalApiRunAttemptLogReadRequest>) {
|
||||
try {
|
||||
const result = await capability.read({
|
||||
projectId: request.projectId,
|
||||
runId: request.runId,
|
||||
attemptId: request.attemptId,
|
||||
range: Object.freeze({
|
||||
offset: request.offset,
|
||||
length: request.length,
|
||||
}),
|
||||
...(request.signal === undefined ? {} : { signal: request.signal }),
|
||||
});
|
||||
if (result.status === 'not_found') {
|
||||
return response(404, { code: 'artifact_not_found' });
|
||||
}
|
||||
if (result.status === 'pending') {
|
||||
return response(202, {
|
||||
schema: 'qinglong/run-attempt-log-read-result@v1',
|
||||
status: 'pending',
|
||||
projectId: result.projectId,
|
||||
runId: result.runId,
|
||||
attemptId: result.attemptId,
|
||||
});
|
||||
}
|
||||
if (result.status === 'missing') {
|
||||
return response(503, { code: 'artifact_unavailable' });
|
||||
}
|
||||
return response(200, projection(result));
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidRunAttemptLogReadError) {
|
||||
return response(400, { code: 'invalid_run_log_read_request' });
|
||||
}
|
||||
if (error instanceof RunAttemptLogReadUnavailableError) {
|
||||
return response(503, { code: 'artifact_unavailable' });
|
||||
}
|
||||
return response(503, { code: 'artifact_unavailable' });
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -28,6 +28,8 @@ const RUN_STEP_LIST_ROUTE_PATTERN =
|
||||
/^\/api\/v3\/projects\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/runs\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/steps$/;
|
||||
const RUN_CANCELLATION_ROUTE_PATTERN =
|
||||
/^\/api\/v3\/projects\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/runs\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/cancellation$/;
|
||||
const RUN_ATTEMPT_LOG_READ_ROUTE_PATTERN =
|
||||
/^\/api\/v3\/projects\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/runs\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/attempts\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/log$/;
|
||||
const TASK_LIST_ROUTE_PATTERN =
|
||||
/^\/api\/v3\/projects\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/tasks$/;
|
||||
const TASK_READ_ROUTE_PATTERN =
|
||||
@@ -44,6 +46,7 @@ type LocalApiRouteResolution =
|
||||
| 'invalid_run_list_query'
|
||||
| 'invalid_run_event_list_query'
|
||||
| 'invalid_run_step_list_query'
|
||||
| 'invalid_run_log_read_query'
|
||||
| 'invalid_task_list_query';
|
||||
}>;
|
||||
|
||||
@@ -340,10 +343,7 @@ function parseTaskListQuery(
|
||||
}
|
||||
const name = field.slice(0, separator);
|
||||
const value = field.slice(separator + 1);
|
||||
if (
|
||||
values.has(name) ||
|
||||
(name !== 'limit' && name !== 'after_task_id')
|
||||
) {
|
||||
if (values.has(name) || (name !== 'limit' && name !== 'after_task_id')) {
|
||||
throw new TypeError();
|
||||
}
|
||||
values.set(name, value);
|
||||
@@ -363,13 +363,58 @@ function parseTaskListQuery(
|
||||
}
|
||||
return Object.freeze({
|
||||
...(limit === undefined ? {} : { limit }),
|
||||
...(taskId === undefined
|
||||
? {}
|
||||
: { after: Object.freeze({ taskId }) }),
|
||||
...(taskId === undefined ? {} : { after: Object.freeze({ taskId }) }),
|
||||
});
|
||||
}
|
||||
|
||||
function route(request: IncomingMessage): LocalApiRouteResolution | null {
|
||||
function parseRunAttemptLogReadQuery(
|
||||
rawQuery: string | undefined,
|
||||
profile: LocalApplicationProfile,
|
||||
): Readonly<{ offset: number; length: number }> {
|
||||
const defaultLength = profile === 'edge' ? 16 * 1024 : 32 * 1024;
|
||||
if (rawQuery === undefined) {
|
||||
return Object.freeze({ offset: 0, length: defaultLength });
|
||||
}
|
||||
if (rawQuery.length === 0) throw new TypeError();
|
||||
const values = new Map<string, string>();
|
||||
for (const field of rawQuery.split('&')) {
|
||||
const separator = field.indexOf('=');
|
||||
if (
|
||||
separator < 1 ||
|
||||
separator !== field.lastIndexOf('=') ||
|
||||
separator === field.length - 1
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
const name = field.slice(0, separator);
|
||||
const value = field.slice(separator + 1);
|
||||
if (values.has(name) || (name !== 'offset' && name !== 'length')) {
|
||||
throw new TypeError();
|
||||
}
|
||||
values.set(name, value);
|
||||
}
|
||||
const rawOffset = values.get('offset');
|
||||
const offset = rawOffset === undefined ? 0 : Number(rawOffset);
|
||||
const rawLength = values.get('length');
|
||||
const length = rawLength === undefined ? defaultLength : Number(rawLength);
|
||||
if (
|
||||
!Number.isSafeInteger(offset) ||
|
||||
offset < 0 ||
|
||||
(rawOffset !== undefined && String(offset) !== rawOffset) ||
|
||||
!Number.isSafeInteger(length) ||
|
||||
length < 1 ||
|
||||
length > 32 * 1024 ||
|
||||
(rawLength !== undefined && String(length) !== rawLength)
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
return Object.freeze({ offset, length });
|
||||
}
|
||||
|
||||
function route(
|
||||
request: IncomingMessage,
|
||||
profile: LocalApplicationProfile,
|
||||
): LocalApiRouteResolution | null {
|
||||
const rawUrl = request.url;
|
||||
if (
|
||||
typeof rawUrl !== 'string' ||
|
||||
@@ -403,6 +448,20 @@ function route(request: IncomingMessage): LocalApiRouteResolution | null {
|
||||
: null;
|
||||
}
|
||||
if (request.method !== 'GET') return null;
|
||||
const runAttemptLogReadMatch = RUN_ATTEMPT_LOG_READ_ROUTE_PATTERN.exec(path);
|
||||
if (runAttemptLogReadMatch) {
|
||||
try {
|
||||
return Object.freeze({
|
||||
operationId: 'run.log.read',
|
||||
projectId: runAttemptLogReadMatch[1]!,
|
||||
runId: runAttemptLogReadMatch[2]!,
|
||||
attemptId: runAttemptLogReadMatch[3]!,
|
||||
...parseRunAttemptLogReadQuery(rawQuery, profile),
|
||||
});
|
||||
} catch {
|
||||
return Object.freeze({ errorCode: 'invalid_run_log_read_query' });
|
||||
}
|
||||
}
|
||||
const taskReadMatch = TASK_READ_ROUTE_PATTERN.exec(path);
|
||||
if (taskReadMatch) {
|
||||
return rawQuery === undefined
|
||||
@@ -552,7 +611,7 @@ export async function startLocalApiHttpSurface(
|
||||
send(response, requestId, errorResponse(503, 'server_overloaded'));
|
||||
return;
|
||||
}
|
||||
const resolvedRoute = route(request);
|
||||
const resolvedRoute = route(request, options.profile);
|
||||
if (!resolvedRoute) {
|
||||
send(response, requestId, errorResponse(404, 'route_not_found'));
|
||||
return;
|
||||
@@ -608,9 +667,9 @@ export async function startLocalApiHttpSurface(
|
||||
error instanceof RangeError
|
||||
? 'request_body_too_large'
|
||||
: error instanceof Error &&
|
||||
error.message === 'request_unavailable'
|
||||
? 'request_unavailable'
|
||||
: 'invalid_request_body';
|
||||
error.message === 'request_unavailable'
|
||||
? 'request_unavailable'
|
||||
: 'invalid_request_body';
|
||||
send(
|
||||
response,
|
||||
requestId,
|
||||
@@ -618,8 +677,8 @@ export async function startLocalApiHttpSurface(
|
||||
code === 'request_body_too_large'
|
||||
? 413
|
||||
: code === 'request_unavailable'
|
||||
? 503
|
||||
: 400,
|
||||
? 503
|
||||
: 400,
|
||||
code,
|
||||
),
|
||||
);
|
||||
|
||||
@@ -101,6 +101,14 @@ function fixture(overrides = {}) {
|
||||
return { statusCode: 202, body: { status: 'accepted' } };
|
||||
},
|
||||
},
|
||||
runAttemptLogReadRoute: {
|
||||
async handle(value) {
|
||||
events.push(
|
||||
`log:${value.projectId}:${value.runId}:${value.attemptId}:${value.offset}:${value.length}`,
|
||||
);
|
||||
return { statusCode: 200, body: { status: 'available' } };
|
||||
},
|
||||
},
|
||||
taskListRoute: {
|
||||
async handle(value) {
|
||||
events.push(`tasks:${value.projectId}:${value.input.limit ?? 32}`);
|
||||
@@ -148,6 +156,51 @@ test('authenticates, authorizes, durably audits and re-confirms before reading',
|
||||
]);
|
||||
});
|
||||
|
||||
test('uses artifact.read and masks denied or approval-fenced log existence', async () => {
|
||||
const operation = Object.freeze({
|
||||
operationId: 'run.log.read',
|
||||
projectId: 'prj_default',
|
||||
runId: 'run_123',
|
||||
attemptId: 'attempt_123',
|
||||
offset: 4,
|
||||
length: 16,
|
||||
});
|
||||
const allowed = fixture();
|
||||
assert.deepEqual(await execute(allowed.admission, request({ operation })), {
|
||||
statusCode: 200,
|
||||
body: { status: 'available' },
|
||||
});
|
||||
assert.deepEqual(allowed.events, [
|
||||
'authenticate',
|
||||
'authorize:artifact.read:prj_default',
|
||||
'audit:allowed:run.log.read',
|
||||
'confirm',
|
||||
'log:prj_default:run_123:attempt_123:4:16',
|
||||
]);
|
||||
|
||||
for (const effect of ['deny', 'require_approval']) {
|
||||
let routed = false;
|
||||
const denied = fixture({
|
||||
policy: {
|
||||
async authorize() {
|
||||
return { effect, reasons: ['masked'], fence: null };
|
||||
},
|
||||
},
|
||||
runAttemptLogReadRoute: {
|
||||
async handle() {
|
||||
routed = true;
|
||||
throw new Error('must not route');
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.deepEqual(await execute(denied.admission, request({ operation })), {
|
||||
statusCode: 404,
|
||||
body: { code: 'artifact_not_found' },
|
||||
});
|
||||
assert.equal(routed, false);
|
||||
}
|
||||
});
|
||||
|
||||
test('uses the same admission chain with a route-owned run.list audit identity', async () => {
|
||||
const { admission, events } = fixture();
|
||||
assert.deepEqual(
|
||||
|
||||
@@ -74,60 +74,71 @@ test('serves only the fixed canonical loopback Run route and drains idempotently
|
||||
host: '127.0.0.1',
|
||||
port,
|
||||
admission: preparedAdmission(async (value, body) => {
|
||||
observed.push(value);
|
||||
if (
|
||||
value.operation.operationId === 'run.cancel' ||
|
||||
value.operation.operationId === 'task.start'
|
||||
) {
|
||||
return { statusCode: 202, body: { accepted: body } };
|
||||
}
|
||||
if (value.operation.operationId === 'run.get') {
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: { run: { id: value.operation.runId } },
|
||||
};
|
||||
}
|
||||
if (value.operation.operationId === 'run.events.list') {
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: {
|
||||
events: [],
|
||||
hasMore: false,
|
||||
nextAfterSequence: value.operation.input.afterSequence ?? 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
if (value.operation.operationId === 'run.steps.list') {
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: {
|
||||
steps: [],
|
||||
hasMore: false,
|
||||
next: value.operation.input.after ?? null,
|
||||
},
|
||||
};
|
||||
}
|
||||
if (value.operation.operationId === 'task.list') {
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: {
|
||||
tasks: [],
|
||||
hasMore: false,
|
||||
input: value.operation.input,
|
||||
},
|
||||
};
|
||||
}
|
||||
if (value.operation.operationId === 'task.get') {
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: { task: { taskId: value.operation.taskId } },
|
||||
};
|
||||
}
|
||||
observed.push(value);
|
||||
if (
|
||||
value.operation.operationId === 'run.cancel' ||
|
||||
value.operation.operationId === 'task.start'
|
||||
) {
|
||||
return { statusCode: 202, body: { accepted: body } };
|
||||
}
|
||||
if (value.operation.operationId === 'run.get') {
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: { runs: [], hasMore: false, input: value.operation.input },
|
||||
body: { run: { id: value.operation.runId } },
|
||||
};
|
||||
}),
|
||||
}
|
||||
if (value.operation.operationId === 'run.events.list') {
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: {
|
||||
events: [],
|
||||
hasMore: false,
|
||||
nextAfterSequence: value.operation.input.afterSequence ?? 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
if (value.operation.operationId === 'run.steps.list') {
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: {
|
||||
steps: [],
|
||||
hasMore: false,
|
||||
next: value.operation.input.after ?? null,
|
||||
},
|
||||
};
|
||||
}
|
||||
if (value.operation.operationId === 'run.log.read') {
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: {
|
||||
range: {
|
||||
offset: value.operation.offset,
|
||||
length: value.operation.length,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (value.operation.operationId === 'task.list') {
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: {
|
||||
tasks: [],
|
||||
hasMore: false,
|
||||
input: value.operation.input,
|
||||
},
|
||||
};
|
||||
}
|
||||
if (value.operation.operationId === 'task.get') {
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: { task: { taskId: value.operation.taskId } },
|
||||
};
|
||||
}
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: { runs: [], hasMore: false, input: value.operation.input },
|
||||
};
|
||||
}),
|
||||
randomUuid: () => '019f70c0-0000-4000-8000-000000000003',
|
||||
});
|
||||
t.after(() => surface.stopAndDrain());
|
||||
@@ -212,10 +223,7 @@ test('serves only the fixed canonical loopback Run route and drains idempotently
|
||||
input: { after: { taskId: 'task_100' }, limit: 8 },
|
||||
});
|
||||
|
||||
const task = await request(
|
||||
port,
|
||||
'/api/v3/projects/prj_default/tasks/task_1',
|
||||
);
|
||||
const task = await request(port, '/api/v3/projects/prj_default/tasks/task_1');
|
||||
assert.deepEqual(task.body, { task: { taskId: 'task_1' } });
|
||||
assert.deepEqual(observed[5].operation, {
|
||||
operationId: 'task.get',
|
||||
@@ -275,6 +283,28 @@ test('serves only the fixed canonical loopback Run route and drains idempotently
|
||||
taskId: 'task_1',
|
||||
});
|
||||
|
||||
const log = await request(
|
||||
port,
|
||||
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?offset=4&length=32',
|
||||
);
|
||||
assert.deepEqual(log.body, { range: { offset: 4, length: 32 } });
|
||||
assert.deepEqual(observed[8].operation, {
|
||||
operationId: 'run.log.read',
|
||||
projectId: 'prj_default',
|
||||
runId: 'run_123',
|
||||
attemptId: 'attempt_1',
|
||||
offset: 4,
|
||||
length: 32,
|
||||
});
|
||||
|
||||
const defaultLog = await request(
|
||||
port,
|
||||
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log',
|
||||
);
|
||||
assert.deepEqual(defaultLog.body, {
|
||||
range: { offset: 0, length: 16 * 1024 },
|
||||
});
|
||||
|
||||
for (const invalidPath of [
|
||||
'/api/v3/projects/prj_default/runs/run_123?expanded=true',
|
||||
'/api/v3/projects/prj_default/runs/run%5f123',
|
||||
@@ -299,6 +329,18 @@ test('serves only the fixed canonical loopback Run route and drains idempotently
|
||||
assert.equal(invalid.statusCode, 400);
|
||||
assert.deepEqual(invalid.body, { code: 'invalid_run_list_query' });
|
||||
}
|
||||
for (const invalidQuery of [
|
||||
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?',
|
||||
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?offset=-1',
|
||||
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?offset=04',
|
||||
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?length=0',
|
||||
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?length=32769',
|
||||
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?unknown=1',
|
||||
]) {
|
||||
const invalid = await request(port, invalidQuery);
|
||||
assert.equal(invalid.statusCode, 400);
|
||||
assert.deepEqual(invalid.body, { code: 'invalid_run_log_read_query' });
|
||||
}
|
||||
for (const invalidQuery of [
|
||||
'/api/v3/projects/prj_default/tasks?',
|
||||
'/api/v3/projects/prj_default/tasks?limit=08',
|
||||
@@ -332,7 +374,7 @@ test('serves only the fixed canonical loopback Run route and drains idempotently
|
||||
assert.equal(invalid.statusCode, 400);
|
||||
assert.deepEqual(invalid.body, { code: 'invalid_run_step_list_query' });
|
||||
}
|
||||
assert.equal(observed.length, 8);
|
||||
assert.equal(observed.length, 10);
|
||||
assert.deepEqual(
|
||||
await Promise.all([surface.stopAndDrain(), surface.stopAndDrain()]),
|
||||
['stopped', 'stopped'],
|
||||
@@ -347,9 +389,9 @@ test('rejects GET bodies without invoking the prepared route handler', async (t)
|
||||
host: '127.0.0.1',
|
||||
port,
|
||||
admission: preparedAdmission(async () => {
|
||||
handlers += 1;
|
||||
return { statusCode: 200, body: {} };
|
||||
}),
|
||||
handlers += 1;
|
||||
return { statusCode: 200, body: {} };
|
||||
}),
|
||||
});
|
||||
t.after(() => surface.stopAndDrain());
|
||||
const response = await request(
|
||||
@@ -477,15 +519,15 @@ test('serves the reviewed worst-case 64-item Run list inside the fixed response
|
||||
host: '127.0.0.1',
|
||||
port,
|
||||
admission: preparedAdmission(async () => {
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: {
|
||||
runs: Object.freeze(Array.from({ length: 64 }, () => item)),
|
||||
hasMore: true,
|
||||
next: { createdAtMs: Number.MAX_SAFE_INTEGER, runId: id128 },
|
||||
},
|
||||
};
|
||||
}),
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: {
|
||||
runs: Object.freeze(Array.from({ length: 64 }, () => item)),
|
||||
hasMore: true,
|
||||
next: { createdAtMs: Number.MAX_SAFE_INTEGER, runId: id128 },
|
||||
},
|
||||
};
|
||||
}),
|
||||
});
|
||||
t.after(() => surface.stopAndDrain());
|
||||
const response = await request(
|
||||
@@ -545,15 +587,15 @@ test('serves the reviewed worst-case 64-item RunEvent list inside the fixed resp
|
||||
host: '127.0.0.1',
|
||||
port,
|
||||
admission: preparedAdmission(async () => {
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: {
|
||||
events: Object.freeze(Array.from({ length: 64 }, () => event)),
|
||||
hasMore: true,
|
||||
nextAfterSequence: event.sequence,
|
||||
},
|
||||
};
|
||||
}),
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: {
|
||||
events: Object.freeze(Array.from({ length: 64 }, () => event)),
|
||||
hasMore: true,
|
||||
nextAfterSequence: event.sequence,
|
||||
},
|
||||
};
|
||||
}),
|
||||
});
|
||||
t.after(() => surface.stopAndDrain());
|
||||
const response = await request(
|
||||
@@ -589,15 +631,15 @@ test('serves the reviewed worst-case 64-item Run Step list inside the fixed resp
|
||||
host: '127.0.0.1',
|
||||
port,
|
||||
admission: preparedAdmission(async () => {
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: {
|
||||
steps: Object.freeze(Array.from({ length: 64 }, () => item)),
|
||||
hasMore: true,
|
||||
next: { stepKey: id128, stepRunId: id128 },
|
||||
},
|
||||
};
|
||||
}),
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: {
|
||||
steps: Object.freeze(Array.from({ length: 64 }, () => item)),
|
||||
hasMore: true,
|
||||
next: { stepKey: id128, stepRunId: id128 },
|
||||
},
|
||||
};
|
||||
}),
|
||||
});
|
||||
t.after(() => surface.stopAndDrain());
|
||||
const response = await request(
|
||||
@@ -621,13 +663,13 @@ test('bounds Edge admission concurrency and drains accepted work', async (t) =>
|
||||
host: '127.0.0.1',
|
||||
port,
|
||||
admission: preparedAdmission(async (value) => {
|
||||
admissions += 1;
|
||||
await barrier;
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: { run: { id: value.operation.runId } },
|
||||
};
|
||||
}),
|
||||
admissions += 1;
|
||||
await barrier;
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: { run: { id: value.operation.runId } },
|
||||
};
|
||||
}),
|
||||
});
|
||||
t.after(() => surface.stopAndDrain());
|
||||
const accepted = Array.from({ length: 4 }, () =>
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
RunAttemptLogReadUnavailableError,
|
||||
} = require('@qinglong/runtime-core/run-attempt-log-read');
|
||||
const {
|
||||
createLocalApiRunAttemptLogReadRoute,
|
||||
} = require('../dist/run/runAttemptLogReadRoute.js');
|
||||
|
||||
function request(overrides = {}) {
|
||||
return {
|
||||
projectId: 'prj_default',
|
||||
runId: 'run_123',
|
||||
attemptId: 'attempt_123',
|
||||
offset: 2,
|
||||
length: 16,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('projects an available byte range as bounded base64 JSON', async () => {
|
||||
const route = createLocalApiRunAttemptLogReadRoute({
|
||||
async read(value) {
|
||||
assert.deepEqual(value.range, { offset: 2, length: 16 });
|
||||
return {
|
||||
status: 'available',
|
||||
projectId: value.projectId,
|
||||
runId: value.runId,
|
||||
attemptId: value.attemptId,
|
||||
logArtifactId: `local-${'a'.repeat(30)}`,
|
||||
content: Buffer.from('hello'),
|
||||
start: 2,
|
||||
endExclusive: 7,
|
||||
totalBytes: 9,
|
||||
nextOffset: 7,
|
||||
truncation: { truncated: 'unknown' },
|
||||
};
|
||||
},
|
||||
});
|
||||
assert.deepEqual(await route.handle(request()), {
|
||||
statusCode: 200,
|
||||
body: {
|
||||
schema: 'qinglong/run-attempt-log-read-result@v1',
|
||||
status: 'available',
|
||||
projectId: 'prj_default',
|
||||
runId: 'run_123',
|
||||
attemptId: 'attempt_123',
|
||||
range: { start: 2, endExclusive: 7, totalBytes: 9, nextOffset: 7 },
|
||||
encoding: 'base64',
|
||||
content: Buffer.from('hello').toString('base64'),
|
||||
truncation: { truncated: 'unknown' },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test('maps pending, masked absence, missing storage and unavailable evidence', async () => {
|
||||
const cases = [
|
||||
[
|
||||
{
|
||||
status: 'pending',
|
||||
projectId: 'prj_default',
|
||||
runId: 'run_123',
|
||||
attemptId: 'attempt_123',
|
||||
},
|
||||
{
|
||||
statusCode: 202,
|
||||
body: {
|
||||
schema: 'qinglong/run-attempt-log-read-result@v1',
|
||||
status: 'pending',
|
||||
projectId: 'prj_default',
|
||||
runId: 'run_123',
|
||||
attemptId: 'attempt_123',
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
{ status: 'not_found' },
|
||||
{ statusCode: 404, body: { code: 'artifact_not_found' } },
|
||||
],
|
||||
[
|
||||
{
|
||||
status: 'missing',
|
||||
projectId: 'prj_default',
|
||||
runId: 'run_123',
|
||||
attemptId: 'attempt_123',
|
||||
logArtifactId: `local-${'a'.repeat(30)}`,
|
||||
},
|
||||
{ statusCode: 503, body: { code: 'artifact_unavailable' } },
|
||||
],
|
||||
];
|
||||
for (const [result, expected] of cases) {
|
||||
const route = createLocalApiRunAttemptLogReadRoute({
|
||||
async read() {
|
||||
return result;
|
||||
},
|
||||
});
|
||||
assert.deepEqual(await route.handle(request()), expected);
|
||||
}
|
||||
const unavailable = createLocalApiRunAttemptLogReadRoute({
|
||||
async read() {
|
||||
throw new RunAttemptLogReadUnavailableError();
|
||||
},
|
||||
});
|
||||
assert.deepEqual(await unavailable.handle(request()), {
|
||||
statusCode: 503,
|
||||
body: { code: 'artifact_unavailable' },
|
||||
});
|
||||
});
|
||||
@@ -24,6 +24,9 @@ const {
|
||||
const {
|
||||
compileLocalCommandTaskDefinition,
|
||||
} = require('@qinglong/runtime-core/task-definition-execution-compiler');
|
||||
const {
|
||||
RunAttemptLogReadService,
|
||||
} = require('@qinglong/runtime-core/run-attempt-log-read');
|
||||
const {
|
||||
createBuiltInTaskSpecSemanticRegistry,
|
||||
} = require('@qinglong/runtime-core/task-spec-semantic');
|
||||
@@ -34,11 +37,16 @@ const {
|
||||
const {
|
||||
createLocalApiProductSurface,
|
||||
} = require('../dist/application-runtime/localApiProductSurface.js');
|
||||
const {
|
||||
LocalRunAttemptLogRangeReader,
|
||||
} = require('../../ql3-local-execution/dist/artifact-read/localRunAttemptLogRangeReader.js');
|
||||
|
||||
const NOW = 1_800_000_000_000;
|
||||
const PEPPER_KEY_ID = 'local-api-pepper-v1';
|
||||
const CREDENTIAL_ID = 'local-api-owner';
|
||||
const RUN_ID = 'run_local_api_1';
|
||||
const ATTEMPT_ID = 'attempt_local_api_1';
|
||||
const LOG_ARTIFACT_ID = `local-${'a'.repeat(30)}`;
|
||||
const SECRET = Buffer.alloc(32, 81).toString('base64url');
|
||||
const PEPPER = Buffer.alloc(32, 82).toString('base64url');
|
||||
const TOKEN = formatApiCredentialToken(CREDENTIAL_ID, SECRET);
|
||||
@@ -206,15 +214,18 @@ function seed(databasePath, materialDigest) {
|
||||
enabled: true,
|
||||
occurredAtMs: NOW - 200,
|
||||
};
|
||||
const taskDefinition = createTaskDefinitionRecord({
|
||||
...taskCommand,
|
||||
spec: taskSemantics.normalize({
|
||||
projectId: taskCommand.projectId,
|
||||
taskId: taskCommand.taskId,
|
||||
kind: taskCommand.kind,
|
||||
spec: taskCommand.spec,
|
||||
}),
|
||||
}, NOW - 200);
|
||||
const taskDefinition = createTaskDefinitionRecord(
|
||||
{
|
||||
...taskCommand,
|
||||
spec: taskSemantics.normalize({
|
||||
projectId: taskCommand.projectId,
|
||||
taskId: taskCommand.taskId,
|
||||
kind: taskCommand.kind,
|
||||
spec: taskCommand.spec,
|
||||
}),
|
||||
},
|
||||
NOW - 200,
|
||||
);
|
||||
const taskExecution = compileLocalCommandTaskDefinition(
|
||||
taskDefinition,
|
||||
taskSemantics,
|
||||
@@ -298,6 +309,15 @@ function seed(databasePath, materialDigest) {
|
||||
'manual', 'runtime', 'running', 1, 1, 0, ?)`,
|
||||
)
|
||||
.run(RUN_ID, NOW - 100);
|
||||
client
|
||||
.prepare(
|
||||
`INSERT INTO "RunAttempts" (
|
||||
"id", "run_id", "attempt", "status", "executor_type",
|
||||
"log_artifact_id", "callback_sequence", "created_at_ms",
|
||||
"started_at_ms"
|
||||
) VALUES (?, ?, 1, 'running', 'local_process', ?, 0, ?, ?)`,
|
||||
)
|
||||
.run(ATTEMPT_ID, RUN_ID, LOG_ARTIFACT_ID, NOW - 90, NOW - 80);
|
||||
client
|
||||
.prepare(
|
||||
`INSERT INTO "StepRuns" (
|
||||
@@ -357,6 +377,15 @@ test('serves an authenticated Run through one real SQLite authority and durable
|
||||
fs.chmodSync(root, 0o700);
|
||||
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
|
||||
const databasePath = path.join(root, 'qinglong3.sqlite');
|
||||
const artifactRoot = path.join(root, 'artifacts');
|
||||
const artifactShard = path.join(artifactRoot, 'aa');
|
||||
fs.mkdirSync(artifactShard, { recursive: true, mode: 0o700 });
|
||||
fs.chmodSync(artifactRoot, 0o700);
|
||||
fs.chmodSync(artifactShard, 0o700);
|
||||
const logContent = Buffer.from('local-api-log-line\n', 'utf8');
|
||||
const logPath = path.join(artifactShard, `${LOG_ARTIFACT_ID}.log`);
|
||||
fs.writeFileSync(logPath, logContent, { mode: 0o600 });
|
||||
fs.chmodSync(logPath, 0o600);
|
||||
const keyringDirectory = path.join(root, 'owner-pepper');
|
||||
fs.mkdirSync(keyringDirectory, { mode: 0o700 });
|
||||
const summary = provisionLocalOwnerPepperKey({
|
||||
@@ -398,6 +427,15 @@ test('serves an authenticated Run through one real SQLite authority and durable
|
||||
stepRuns: await runtime.stepRunReader(),
|
||||
runCancellation: await runtime.runCancellationRepository(),
|
||||
taskStart: await runtime.taskStartRepository(),
|
||||
runAttemptLogRead: new RunAttemptLogReadService(
|
||||
runtime.runRepository,
|
||||
new LocalRunAttemptLogRangeReader(artifactRoot),
|
||||
{
|
||||
executorType: 'local_process',
|
||||
artifactIdPattern: /^local-[a-f0-9]{30}$/,
|
||||
maximumReadBytes: 32 * 1024,
|
||||
},
|
||||
),
|
||||
taskDefinitions: runtime.taskDefinitions,
|
||||
apiCredentials: runtime.apiCredentials,
|
||||
ownerPepper: runtime.ownerPepper,
|
||||
@@ -572,12 +610,29 @@ test('serves an authenticated Run through one real SQLite authority and durable
|
||||
});
|
||||
assert.equal(JSON.stringify(steps).includes('private'), false);
|
||||
|
||||
const log = await request(
|
||||
port,
|
||||
`Bearer ${TOKEN}`,
|
||||
`/api/v3/projects/default/runs/${RUN_ID}/attempts/${ATTEMPT_ID}/log?offset=0&length=8`,
|
||||
);
|
||||
assert.equal(log.statusCode, 200);
|
||||
assert.equal(log.body.schema, 'qinglong/run-attempt-log-read-result@v1');
|
||||
assert.equal(log.body.status, 'available');
|
||||
assert.equal(log.body.encoding, 'base64');
|
||||
assert.equal(Buffer.from(log.body.content, 'base64').toString(), 'local-ap');
|
||||
assert.deepEqual(log.body.range, {
|
||||
start: 0,
|
||||
endExclusive: 8,
|
||||
totalBytes: logContent.byteLength,
|
||||
nextOffset: 8,
|
||||
});
|
||||
assert.deepEqual(log.body.truncation, { truncated: 'unknown' });
|
||||
|
||||
const cancellationBody = JSON.stringify({
|
||||
schema: 'qinglong/run-cancellation@v1',
|
||||
mutationId: 'cancel-local-api-1',
|
||||
});
|
||||
const cancellationPath =
|
||||
`/api/v3/projects/default/runs/${RUN_ID}/cancellation`;
|
||||
const cancellationPath = `/api/v3/projects/default/runs/${RUN_ID}/cancellation`;
|
||||
const cancellationOptions = {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
@@ -623,7 +678,7 @@ test('serves an authenticated Run through one real SQLite authority and durable
|
||||
WHERE operation_id IN (
|
||||
'run.get', 'run.list', 'run.events.list', 'run.steps.list',
|
||||
'run.cancel', 'task.get', 'task.list'
|
||||
, 'task.start'
|
||||
, 'task.start', 'run.log.read'
|
||||
)
|
||||
ORDER BY operation_id, outcome`,
|
||||
)
|
||||
@@ -636,6 +691,7 @@ test('serves an authenticated Run through one real SQLite authority and durable
|
||||
'run.get:allowed',
|
||||
'run.get:authentication_rejected',
|
||||
'run.list:allowed',
|
||||
'run.log.read:allowed',
|
||||
'run.steps.list:allowed',
|
||||
'task.get:allowed',
|
||||
'task.get:allowed',
|
||||
|
||||
@@ -455,11 +455,28 @@ export async function bootstrapLocalApplication(
|
||||
});
|
||||
|
||||
if (options.productSurface) {
|
||||
const [stepRuns, runCancellation, taskStart] = await Promise.all([
|
||||
const [
|
||||
stepRuns,
|
||||
runCancellation,
|
||||
taskStart,
|
||||
{ LocalRunAttemptLogRangeReader },
|
||||
{ RunAttemptLogReadService },
|
||||
] = await Promise.all([
|
||||
storage.stepRunReader(),
|
||||
storage.runCancellationRepository(),
|
||||
storage.taskStartRepository(),
|
||||
import('@qinglong/local-execution/artifact-read'),
|
||||
import('@qinglong/runtime-core/run-attempt-log-read'),
|
||||
]);
|
||||
const runAttemptLogRead = new RunAttemptLogReadService(
|
||||
storage.runs,
|
||||
new LocalRunAttemptLogRangeReader(options.artifactRoot),
|
||||
{
|
||||
executorType: 'local_process',
|
||||
artifactIdPattern: /^local-[a-f0-9]{30}$/,
|
||||
maximumReadBytes: 32 * 1024,
|
||||
},
|
||||
);
|
||||
productSurfaceLifecycle = await options.productSurface.start(
|
||||
Object.freeze({
|
||||
profile: options.profile,
|
||||
@@ -467,6 +484,7 @@ export async function bootstrapLocalApplication(
|
||||
stepRuns,
|
||||
runCancellation,
|
||||
taskStart,
|
||||
runAttemptLogRead,
|
||||
taskDefinitions: storage.taskDefinitions,
|
||||
apiCredentials: storage.apiCredentials,
|
||||
ownerPepper: storage.ownerPepper,
|
||||
|
||||
@@ -18,6 +18,10 @@ import type { PluginPackageRecoveryCycleResult } from '@qinglong/runtime-core/pl
|
||||
import type { PluginPackageAutomationPublicationRecoveryCycleResult } from '@qinglong/runtime-core/plugin-package-automation-publication';
|
||||
import type { PluginPackageTaskPublicationRecoveryCycleResult } from '@qinglong/runtime-core/plugin-package-task-publication';
|
||||
import type { ProjectToolDefinitionSnapshotRecoveryCycleResult } from '@qinglong/runtime-core/project-tool-definition-snapshot';
|
||||
import type {
|
||||
RunAttemptLogReadRequest,
|
||||
RunAttemptLogReadResult,
|
||||
} from '@qinglong/runtime-core/run-attempt-log-read';
|
||||
|
||||
export type LocalApplicationProfile = 'edge' | 'standalone';
|
||||
|
||||
@@ -59,6 +63,11 @@ export interface LocalApplicationProductSurfaceAuthority {
|
||||
ReadyFreshStorage['taskDefinitions'],
|
||||
'findCurrentTaskDefinition' | 'listTaskDefinitions'
|
||||
>;
|
||||
readonly runAttemptLogRead: Readonly<{
|
||||
read(
|
||||
request: Readonly<RunAttemptLogReadRequest>,
|
||||
): Promise<RunAttemptLogReadResult>;
|
||||
}>;
|
||||
readonly apiCredentials: ReadyFreshStorage['apiCredentials'];
|
||||
readonly ownerPepper: ReadyFreshStorage['ownerPepper'];
|
||||
readonly projectPolicy: ReadyFreshStorage['projectPolicy'];
|
||||
|
||||
@@ -88,8 +88,7 @@ const RECEIPT_TOKEN = 'A'.repeat(32);
|
||||
const CLEANUP_RUN_ID = '019f70c0-0000-7000-8000-000000000011';
|
||||
const CLEANUP_ATTEMPT_ID = '019f70c0-0000-7000-8000-000000000012';
|
||||
const WORKFLOW_CANCELLATION_CREDENTIAL_ID = 'application-workflow-owner';
|
||||
const WORKFLOW_CANCELLATION_PEPPER_KEY_ID =
|
||||
'application-workflow-owner-v1';
|
||||
const WORKFLOW_CANCELLATION_PEPPER_KEY_ID = 'application-workflow-owner-v1';
|
||||
const WORKFLOW_CANCELLATION_PEPPER_BYTES = Buffer.alloc(32, 141);
|
||||
const WORKFLOW_CANCELLATION_SECRET = Buffer.alloc(32, 142).toString(
|
||||
'base64url',
|
||||
@@ -322,8 +321,7 @@ function promptResourceConfiguration() {
|
||||
const prefix = 'private router model output:';
|
||||
const minimumOutputBytes = Buffer.byteLength(prefix, 'utf8');
|
||||
const rawOutputBytes =
|
||||
process.env.QL3_PROMPT_RESOURCE_OUTPUT_BYTES ??
|
||||
String(minimumOutputBytes);
|
||||
process.env.QL3_PROMPT_RESOURCE_OUTPUT_BYTES ?? String(minimumOutputBytes);
|
||||
const outputBytes = Number(rawOutputBytes);
|
||||
if (
|
||||
!Number.isSafeInteger(outputBytes) ||
|
||||
@@ -338,8 +336,7 @@ function promptResourceConfiguration() {
|
||||
profile,
|
||||
output: prefix + 'x'.repeat(outputBytes - Buffer.byteLength(prefix)),
|
||||
outputBytes,
|
||||
resourceProbe:
|
||||
process.env.QL3_PROMPT_RESOURCE_OUTPUT_BYTES !== undefined,
|
||||
resourceProbe: process.env.QL3_PROMPT_RESOURCE_OUTPUT_BYTES !== undefined,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1263,7 +1260,10 @@ test('executes one active Package Prompt through local AI composition with conte
|
||||
promptOutputKeys: {
|
||||
async active() {
|
||||
metrics.keyLoads += 1;
|
||||
return { keyId: 'edge-prompt-output-key-1', key: Buffer.alloc(32, 7) };
|
||||
return {
|
||||
keyId: 'edge-prompt-output-key-1',
|
||||
key: Buffer.alloc(32, 7),
|
||||
};
|
||||
},
|
||||
async resolve(keyId) {
|
||||
metrics.keyResolutions += 1;
|
||||
@@ -1274,7 +1274,10 @@ test('executes one active Package Prompt through local AI composition with conte
|
||||
promptOutputRead: {
|
||||
authorizer: {
|
||||
async authorize(request) {
|
||||
assert.equal(request.projectId, 'project-application-prompt-resource');
|
||||
assert.equal(
|
||||
request.projectId,
|
||||
'project-application-prompt-resource',
|
||||
);
|
||||
assert.equal(request.principal.subject.id, 'edge-resource-owner');
|
||||
return { effect: 'allow' };
|
||||
},
|
||||
@@ -1376,7 +1379,10 @@ test('executes one active Package Prompt through local AI composition with conte
|
||||
assert.equal(recoveredByRequest.result.text, metrics.privateOutput);
|
||||
assert.deepEqual(recoveredByRequest.reference, durableFirst.outputArtifact);
|
||||
assert.equal(metrics.keyResolutions, 2);
|
||||
assert.equal(metrics.lastResolvedKey.every((byte) => byte === 0), true);
|
||||
assert.equal(
|
||||
metrics.lastResolvedKey.every((byte) => byte === 0),
|
||||
true,
|
||||
);
|
||||
const durableStorageAfter = sqliteStorageSnapshot(value.targetPath);
|
||||
const durableStorageGrowth = sqliteStorageGrowth(
|
||||
durableStorageBefore,
|
||||
@@ -1450,9 +1456,8 @@ test('executes one active Package Prompt through local AI composition with conte
|
||||
)
|
||||
.get(durableFirst.admission.invocationId),
|
||||
},
|
||||
integrityCheck: reader
|
||||
.prepare('PRAGMA integrity_check')
|
||||
.get().integrity_check,
|
||||
integrityCheck: reader.prepare('PRAGMA integrity_check').get()
|
||||
.integrity_check,
|
||||
};
|
||||
} finally {
|
||||
reader.close();
|
||||
@@ -1532,8 +1537,7 @@ test('executes one active Package Prompt through local AI composition with conte
|
||||
databaseLogicalWriteAmplificationPermille,
|
||||
databaseAllocatedWriteAmplificationPermille,
|
||||
walWriteAmplificationPermille,
|
||||
journalMode:
|
||||
resource.profile === 'edge' ? 'delete' : 'wal',
|
||||
journalMode: resource.profile === 'edge' ? 'delete' : 'wal',
|
||||
providerCalls: metrics.providerCalls,
|
||||
keyLoads: metrics.keyLoads,
|
||||
keyResolutions: metrics.keyResolutions,
|
||||
@@ -1700,6 +1704,7 @@ test('starts an optional product surface after recovery and drains it before own
|
||||
typeof authority.taskDefinitions.listTaskDefinitions,
|
||||
'function',
|
||||
);
|
||||
assert.equal(typeof authority.runAttemptLogRead.read, 'function');
|
||||
assert.equal(typeof authority.apiCredentials.resolve, 'function');
|
||||
assert.equal(typeof authority.ownerPepper.resolveKey, 'function');
|
||||
assert.equal(typeof authority.projectPolicy.resolve, 'function');
|
||||
|
||||
@@ -28,6 +28,11 @@
|
||||
"require": "./dist/dispatch/index.js",
|
||||
"default": "./dist/dispatch/index.js"
|
||||
},
|
||||
"./artifact-read": {
|
||||
"types": "./dist/artifact-read/index.d.ts",
|
||||
"require": "./dist/artifact-read/index.js",
|
||||
"default": "./dist/artifact-read/index.js"
|
||||
},
|
||||
"./scheduler": {
|
||||
"types": "./dist/scheduler/index.d.ts",
|
||||
"require": "./dist/scheduler/index.js",
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export * from './localRunAttemptLogRangeReader';
|
||||
@@ -0,0 +1,285 @@
|
||||
import { constants, type Stats } from 'node:fs';
|
||||
import fs, { type FileHandle } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
import {
|
||||
normalizeRunAttemptLogReadRange,
|
||||
type RunAttemptLogRangeReader,
|
||||
type RunAttemptLogRangeReadResult,
|
||||
type RunAttemptLogReadIdentity,
|
||||
type RunAttemptLogReadRange,
|
||||
type RunAttemptLogTruncationView,
|
||||
} from '@qinglong/runtime-core/run-attempt-log-read';
|
||||
|
||||
const LOCAL_ARTIFACT_ID = /^local-[a-f0-9]{30}$/;
|
||||
const MAXIMUM_ARTIFACT_BYTES = 1024 * 1024 * 1024;
|
||||
const MAXIMUM_FACT_BYTES = 1024;
|
||||
|
||||
export class LocalRunAttemptLogRangeReadError extends Error {
|
||||
constructor(
|
||||
readonly reason:
|
||||
| 'invalid_configuration'
|
||||
| 'unsafe_path'
|
||||
| 'integrity_mismatch',
|
||||
options?: ErrorOptions,
|
||||
) {
|
||||
super(`Local Run Attempt log range read failed: ${reason}`, options);
|
||||
this.name = 'LocalRunAttemptLogRangeReadError';
|
||||
}
|
||||
}
|
||||
|
||||
function isCode(error: unknown, code: string): boolean {
|
||||
return (
|
||||
!!error &&
|
||||
typeof error === 'object' &&
|
||||
'code' in error &&
|
||||
(error as { code?: unknown }).code === code
|
||||
);
|
||||
}
|
||||
|
||||
function currentUid(): number | undefined {
|
||||
return typeof process.getuid === 'function' ? process.getuid() : undefined;
|
||||
}
|
||||
|
||||
function root(value: string): string {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
!path.isAbsolute(value) ||
|
||||
path.parse(value).root === value ||
|
||||
value.includes('\0') ||
|
||||
Buffer.byteLength(value, 'utf8') > 4096
|
||||
) {
|
||||
throw new LocalRunAttemptLogRangeReadError('invalid_configuration');
|
||||
}
|
||||
return path.resolve(value);
|
||||
}
|
||||
|
||||
function identity(
|
||||
value: Readonly<RunAttemptLogReadIdentity>,
|
||||
): Readonly<RunAttemptLogReadIdentity> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
!LOCAL_ARTIFACT_ID.test(value.logArtifactId)
|
||||
) {
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function assertOwnedDirectory(stat: Stats): void {
|
||||
const uid = currentUid();
|
||||
if (
|
||||
!stat.isDirectory() ||
|
||||
stat.isSymbolicLink() ||
|
||||
(stat.mode & 0o777) !== 0o700 ||
|
||||
(uid !== undefined && stat.uid !== uid)
|
||||
) {
|
||||
throw new LocalRunAttemptLogRangeReadError('unsafe_path');
|
||||
}
|
||||
}
|
||||
|
||||
function assertOwnedFile(stat: Stats): void {
|
||||
const uid = currentUid();
|
||||
if (
|
||||
!stat.isFile() ||
|
||||
stat.nlink !== 1 ||
|
||||
(stat.mode & 0o777) !== 0o600 ||
|
||||
(uid !== undefined && stat.uid !== uid) ||
|
||||
!Number.isSafeInteger(stat.size) ||
|
||||
stat.size < 0 ||
|
||||
stat.size > MAXIMUM_ARTIFACT_BYTES
|
||||
) {
|
||||
throw new LocalRunAttemptLogRangeReadError('unsafe_path');
|
||||
}
|
||||
}
|
||||
|
||||
async function optionalPrivateDirectory(directory: string): Promise<boolean> {
|
||||
try {
|
||||
assertOwnedDirectory(await fs.lstat(directory));
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return false;
|
||||
if (error instanceof LocalRunAttemptLogRangeReadError) throw error;
|
||||
throw new LocalRunAttemptLogRangeReadError('unsafe_path', { cause: error });
|
||||
}
|
||||
}
|
||||
|
||||
async function openPrivateFile(
|
||||
filePath: string,
|
||||
): Promise<FileHandle | undefined> {
|
||||
try {
|
||||
return await fs.open(
|
||||
filePath,
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return undefined;
|
||||
throw new LocalRunAttemptLogRangeReadError('unsafe_path', { cause: error });
|
||||
}
|
||||
}
|
||||
|
||||
function exactFact(
|
||||
value: unknown,
|
||||
expected: Readonly<RunAttemptLogReadIdentity>,
|
||||
): Readonly<RunAttemptLogTruncationView> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
|
||||
}
|
||||
const fact = value as Record<string, unknown>;
|
||||
const keys = Object.keys(fact).sort();
|
||||
if (
|
||||
keys.join(',') !==
|
||||
'attemptId,logArtifactId,maximumBytes,observedAtMs,quotaReached,runId,schemaVersion' ||
|
||||
fact.schemaVersion !== 1 ||
|
||||
fact.runId !== expected.runId ||
|
||||
fact.attemptId !== expected.attemptId ||
|
||||
fact.logArtifactId !== expected.logArtifactId ||
|
||||
!Number.isSafeInteger(fact.maximumBytes) ||
|
||||
Number(fact.maximumBytes) < 64 * 1024 ||
|
||||
Number(fact.maximumBytes) > MAXIMUM_ARTIFACT_BYTES ||
|
||||
typeof fact.quotaReached !== 'boolean' ||
|
||||
!Number.isSafeInteger(fact.observedAtMs) ||
|
||||
Number(fact.observedAtMs) < 0
|
||||
) {
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
|
||||
}
|
||||
return Object.freeze({
|
||||
truncated: fact.quotaReached,
|
||||
maximumBytes: fact.maximumBytes as number,
|
||||
observedAtMs: fact.observedAtMs as number,
|
||||
});
|
||||
}
|
||||
|
||||
async function readTruncationFact(
|
||||
directory: string,
|
||||
expected: Readonly<RunAttemptLogReadIdentity>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Readonly<RunAttemptLogTruncationView>> {
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
const factPath = path.join(
|
||||
directory,
|
||||
`.${expected.logArtifactId}.log.truncated.json`,
|
||||
);
|
||||
const handle = await openPrivateFile(factPath);
|
||||
if (!handle) return Object.freeze({ truncated: 'unknown' as const });
|
||||
try {
|
||||
const before = await handle.stat();
|
||||
assertOwnedFile(before);
|
||||
if (before.size < 2 || before.size > MAXIMUM_FACT_BYTES) {
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
|
||||
}
|
||||
const content = Buffer.allocUnsafe(before.size);
|
||||
let read = 0;
|
||||
while (read < content.byteLength) {
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
const result = await handle.read(
|
||||
content,
|
||||
read,
|
||||
content.byteLength - read,
|
||||
read,
|
||||
);
|
||||
if (result.bytesRead < 1) {
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
|
||||
}
|
||||
read += result.bytesRead;
|
||||
}
|
||||
const after = await handle.stat();
|
||||
assertOwnedFile(after);
|
||||
if (
|
||||
after.dev !== before.dev ||
|
||||
after.ino !== before.ino ||
|
||||
after.size !== before.size
|
||||
) {
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
|
||||
}
|
||||
try {
|
||||
const text = new TextDecoder('utf-8', { fatal: true }).decode(content);
|
||||
return exactFact(JSON.parse(text), expected);
|
||||
} catch (error) {
|
||||
if (error instanceof LocalRunAttemptLogRangeReadError) throw error;
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch', {
|
||||
cause: error,
|
||||
});
|
||||
} finally {
|
||||
content.fill(0);
|
||||
}
|
||||
} finally {
|
||||
await handle.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalRunAttemptLogRangeReader implements RunAttemptLogRangeReader {
|
||||
private readonly root: string;
|
||||
|
||||
constructor(artifactRoot: string) {
|
||||
this.root = root(artifactRoot);
|
||||
}
|
||||
|
||||
async read(
|
||||
rawIdentity: Readonly<RunAttemptLogReadIdentity>,
|
||||
rawRange: Readonly<RunAttemptLogReadRange>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RunAttemptLogRangeReadResult> {
|
||||
const expected = identity(rawIdentity);
|
||||
const range = normalizeRunAttemptLogReadRange(rawRange);
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
if (!(await optionalPrivateDirectory(this.root))) {
|
||||
return Object.freeze({ status: 'missing' as const });
|
||||
}
|
||||
const directory = path.join(
|
||||
this.root,
|
||||
expected.logArtifactId.slice('local-'.length, 'local-'.length + 2),
|
||||
);
|
||||
if (!(await optionalPrivateDirectory(directory))) {
|
||||
return Object.freeze({ status: 'missing' as const });
|
||||
}
|
||||
const target = path.join(directory, `${expected.logArtifactId}.log`);
|
||||
const handle = await openPrivateFile(target);
|
||||
if (!handle) return Object.freeze({ status: 'missing' as const });
|
||||
try {
|
||||
const before = await handle.stat();
|
||||
assertOwnedFile(before);
|
||||
const start = Math.min(range.offset, before.size);
|
||||
const expectedBytes = Math.min(range.length, before.size - start);
|
||||
const content = Buffer.allocUnsafe(expectedBytes);
|
||||
let read = 0;
|
||||
while (read < expectedBytes) {
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
const result = await handle.read(
|
||||
content,
|
||||
read,
|
||||
expectedBytes - read,
|
||||
start + read,
|
||||
);
|
||||
if (result.bytesRead < 1) {
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
|
||||
}
|
||||
read += result.bytesRead;
|
||||
}
|
||||
const after = await handle.stat();
|
||||
assertOwnedFile(after);
|
||||
if (
|
||||
after.dev !== before.dev ||
|
||||
after.ino !== before.ino ||
|
||||
after.size < before.size
|
||||
) {
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
|
||||
}
|
||||
const endExclusive = start + content.byteLength;
|
||||
const truncation = await readTruncationFact(directory, expected, signal);
|
||||
return Object.freeze({
|
||||
status: 'available' as const,
|
||||
content,
|
||||
start,
|
||||
endExclusive,
|
||||
totalBytes: before.size,
|
||||
...(endExclusive < before.size ? { nextOffset: endExclusive } : {}),
|
||||
truncation,
|
||||
});
|
||||
} finally {
|
||||
await handle.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
LocalRunAttemptLogRangeReadError,
|
||||
LocalRunAttemptLogRangeReader,
|
||||
} = require('../dist/artifact-read/localRunAttemptLogRangeReader.js');
|
||||
|
||||
const artifactId = `local-${'a'.repeat(30)}`;
|
||||
const identity = Object.freeze({
|
||||
projectId: 'prj_default',
|
||||
runId: 'run_123',
|
||||
attemptId: 'attempt_123',
|
||||
logArtifactId: artifactId,
|
||||
});
|
||||
|
||||
async function fixture(t, content = Buffer.from('0123456789')) {
|
||||
const parent = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-log-read-'));
|
||||
t.after(() => fs.rm(parent, { recursive: true, force: true }));
|
||||
const root = path.join(parent, 'artifacts');
|
||||
const shard = path.join(root, 'aa');
|
||||
await fs.mkdir(shard, { recursive: true, mode: 0o700 });
|
||||
await fs.chmod(root, 0o700);
|
||||
await fs.chmod(shard, 0o700);
|
||||
const log = path.join(shard, `${artifactId}.log`);
|
||||
await fs.writeFile(log, content, { mode: 0o600 });
|
||||
await fs.chmod(log, 0o600);
|
||||
return { parent, root, shard, log };
|
||||
}
|
||||
|
||||
async function fact(shard, overrides = {}) {
|
||||
const file = path.join(shard, `.${artifactId}.log.truncated.json`);
|
||||
await fs.writeFile(
|
||||
file,
|
||||
JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
runId: identity.runId,
|
||||
attemptId: identity.attemptId,
|
||||
logArtifactId: identity.logArtifactId,
|
||||
maximumBytes: 64 * 1024,
|
||||
quotaReached: false,
|
||||
observedAtMs: 9,
|
||||
...overrides,
|
||||
}),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
await fs.chmod(file, 0o600);
|
||||
return file;
|
||||
}
|
||||
|
||||
test('reads one bounded private-file snapshot and canonical truncation fact', async (t) => {
|
||||
const value = await fixture(t);
|
||||
await fact(value.shard, { quotaReached: true });
|
||||
const result = await new LocalRunAttemptLogRangeReader(value.root).read(
|
||||
identity,
|
||||
{ offset: 2, length: 4 },
|
||||
);
|
||||
assert.equal(result.status, 'available');
|
||||
assert.equal(Buffer.from(result.content).toString(), '2345');
|
||||
assert.deepEqual(
|
||||
{
|
||||
start: result.start,
|
||||
endExclusive: result.endExclusive,
|
||||
totalBytes: result.totalBytes,
|
||||
nextOffset: result.nextOffset,
|
||||
truncation: result.truncation,
|
||||
},
|
||||
{
|
||||
start: 2,
|
||||
endExclusive: 6,
|
||||
totalBytes: 10,
|
||||
nextOffset: 6,
|
||||
truncation: {
|
||||
truncated: true,
|
||||
maximumBytes: 64 * 1024,
|
||||
observedAtMs: 9,
|
||||
},
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('returns unknown truncation and a stable empty range beyond the snapshot', async (t) => {
|
||||
const value = await fixture(t);
|
||||
const result = await new LocalRunAttemptLogRangeReader(value.root).read(
|
||||
identity,
|
||||
{ offset: 99, length: 4 },
|
||||
);
|
||||
assert.equal(result.status, 'available');
|
||||
assert.equal(result.content.byteLength, 0);
|
||||
assert.equal(result.start, 10);
|
||||
assert.equal(result.endExclusive, 10);
|
||||
assert.equal(result.totalBytes, 10);
|
||||
assert.equal(result.nextOffset, undefined);
|
||||
assert.deepEqual(result.truncation, { truncated: 'unknown' });
|
||||
});
|
||||
|
||||
test('treats absent root, shard and log as missing', async (t) => {
|
||||
const parent = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-log-missing-'));
|
||||
t.after(() => fs.rm(parent, { recursive: true, force: true }));
|
||||
const root = path.join(parent, 'artifacts');
|
||||
const reader = new LocalRunAttemptLogRangeReader(root);
|
||||
assert.deepEqual(await reader.read(identity, { offset: 0, length: 1 }), {
|
||||
status: 'missing',
|
||||
});
|
||||
await fs.mkdir(root, { mode: 0o700 });
|
||||
assert.deepEqual(await reader.read(identity, { offset: 0, length: 1 }), {
|
||||
status: 'missing',
|
||||
});
|
||||
const shard = path.join(root, 'aa');
|
||||
await fs.mkdir(shard, { mode: 0o700 });
|
||||
assert.deepEqual(await reader.read(identity, { offset: 0, length: 1 }), {
|
||||
status: 'missing',
|
||||
});
|
||||
});
|
||||
|
||||
test('fails closed for symlink targets and widened file permissions', async (t) => {
|
||||
const symlink = await fixture(t);
|
||||
await fs.rm(symlink.log);
|
||||
await fs.symlink(path.join(symlink.parent, 'outside'), symlink.log);
|
||||
await assert.rejects(
|
||||
new LocalRunAttemptLogRangeReader(symlink.root).read(identity, {
|
||||
offset: 0,
|
||||
length: 1,
|
||||
}),
|
||||
LocalRunAttemptLogRangeReadError,
|
||||
);
|
||||
|
||||
const widened = await fixture(t);
|
||||
await fs.chmod(widened.log, 0o644);
|
||||
await assert.rejects(
|
||||
new LocalRunAttemptLogRangeReader(widened.root).read(identity, {
|
||||
offset: 0,
|
||||
length: 1,
|
||||
}),
|
||||
LocalRunAttemptLogRangeReadError,
|
||||
);
|
||||
});
|
||||
|
||||
test('fails closed for truncation identity drift and an aborted request', async (t) => {
|
||||
const value = await fixture(t);
|
||||
await fact(value.shard, { attemptId: 'attempt_other' });
|
||||
const reader = new LocalRunAttemptLogRangeReader(value.root);
|
||||
await assert.rejects(
|
||||
reader.read(identity, { offset: 0, length: 1 }),
|
||||
(error) =>
|
||||
error instanceof LocalRunAttemptLogRangeReadError &&
|
||||
error.reason === 'integrity_mismatch',
|
||||
);
|
||||
const abort = new AbortController();
|
||||
abort.abort(new Error('cancelled'));
|
||||
await assert.rejects(
|
||||
reader.read(identity, { offset: 0, length: 1 }, abort.signal),
|
||||
/cancelled/,
|
||||
);
|
||||
});
|
||||
@@ -232,6 +232,9 @@
|
||||
],
|
||||
"bounded-run-step-list-projection": [
|
||||
"dist/run/projection/boundedRunStepListProjection.d.ts"
|
||||
],
|
||||
"run-attempt-log-read": [
|
||||
"dist/run/log-read/runAttemptLogRead.d.ts"
|
||||
]
|
||||
}
|
||||
},
|
||||
@@ -281,6 +284,11 @@
|
||||
"require": "./dist/run/projection/boundedRunStepListProjection.js",
|
||||
"default": "./dist/run/projection/boundedRunStepListProjection.js"
|
||||
},
|
||||
"./run-attempt-log-read": {
|
||||
"types": "./dist/run/log-read/runAttemptLogRead.d.ts",
|
||||
"require": "./dist/run/log-read/runAttemptLogRead.js",
|
||||
"default": "./dist/run/log-read/runAttemptLogRead.js"
|
||||
},
|
||||
"./task-definition": {
|
||||
"types": "./dist/task-definition/taskDefinition.d.ts",
|
||||
"require": "./dist/task-definition/taskDefinition.js",
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
import { RUN_ATTEMPT_STATUSES, type RunAttemptStatus } from '../run';
|
||||
import type { RunRepositoryReader } from '../runRepository';
|
||||
|
||||
export const MAX_RUN_ATTEMPT_LOG_READ_BYTES = 256 * 1024;
|
||||
|
||||
const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const TERMINAL_ATTEMPT_STATUSES = new Set<RunAttemptStatus>([
|
||||
'succeeded',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'timed_out',
|
||||
'lost',
|
||||
]);
|
||||
|
||||
export interface RunAttemptLogReadRange {
|
||||
readonly offset: number;
|
||||
readonly length: number;
|
||||
}
|
||||
|
||||
export interface RunAttemptLogReadIdentity {
|
||||
readonly projectId: string;
|
||||
readonly runId: string;
|
||||
readonly attemptId: string;
|
||||
readonly logArtifactId: string;
|
||||
}
|
||||
|
||||
export interface RunAttemptLogTruncationView {
|
||||
readonly truncated: boolean | 'unknown';
|
||||
readonly maximumBytes?: number;
|
||||
readonly observedAtMs?: number;
|
||||
}
|
||||
|
||||
export type RunAttemptLogRangeReadResult =
|
||||
| Readonly<{ readonly status: 'missing' }>
|
||||
| Readonly<{
|
||||
readonly status: 'available';
|
||||
readonly content: Uint8Array;
|
||||
readonly start: number;
|
||||
readonly endExclusive: number;
|
||||
readonly totalBytes: number;
|
||||
readonly nextOffset?: number;
|
||||
readonly truncation: Readonly<RunAttemptLogTruncationView>;
|
||||
}>;
|
||||
|
||||
export interface RunAttemptLogRangeReader {
|
||||
read(
|
||||
identity: Readonly<RunAttemptLogReadIdentity>,
|
||||
range: Readonly<RunAttemptLogReadRange>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RunAttemptLogRangeReadResult>;
|
||||
}
|
||||
|
||||
export interface RunAttemptLogReadRequest {
|
||||
readonly projectId: string;
|
||||
readonly runId: string;
|
||||
readonly attemptId: string;
|
||||
readonly range: Readonly<RunAttemptLogReadRange>;
|
||||
readonly signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export type RunAttemptLogReadResult =
|
||||
| Readonly<{ readonly status: 'not_found' }>
|
||||
| Readonly<{
|
||||
readonly status: 'pending';
|
||||
readonly projectId: string;
|
||||
readonly runId: string;
|
||||
readonly attemptId: string;
|
||||
readonly logArtifactId?: string;
|
||||
}>
|
||||
| (Readonly<RunAttemptLogReadIdentity> &
|
||||
Readonly<{ readonly status: 'missing' }>)
|
||||
| (Readonly<RunAttemptLogReadIdentity> &
|
||||
Extract<RunAttemptLogRangeReadResult, { readonly status: 'available' }>);
|
||||
|
||||
export interface RunAttemptLogReadServiceOptions {
|
||||
readonly executorType: 'local_process' | 'remote_worker';
|
||||
readonly artifactIdPattern: RegExp;
|
||||
readonly maximumReadBytes: number;
|
||||
readonly activeMissingIsPending?: boolean;
|
||||
}
|
||||
|
||||
export class InvalidRunAttemptLogReadError extends TypeError {
|
||||
constructor(message: string) {
|
||||
super(`Run Attempt log read is invalid: ${message}`);
|
||||
this.name = 'InvalidRunAttemptLogReadError';
|
||||
}
|
||||
}
|
||||
|
||||
export class RunAttemptLogReadUnavailableError extends Error {
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('Run Attempt log read is unavailable', options);
|
||||
this.name = 'RunAttemptLogReadUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function canonicalId(name: string, value: unknown): string {
|
||||
if (typeof value !== 'string' || !ID_PATTERN.test(value)) {
|
||||
throw new InvalidRunAttemptLogReadError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function exactKeys(
|
||||
value: object,
|
||||
required: readonly string[],
|
||||
optional: readonly string[],
|
||||
name: string,
|
||||
): void {
|
||||
const keys = Object.keys(value);
|
||||
const allowed = new Set([...required, ...optional]);
|
||||
if (
|
||||
required.some((key) => !Object.hasOwn(value, key)) ||
|
||||
keys.some((key) => !allowed.has(key))
|
||||
) {
|
||||
throw new InvalidRunAttemptLogReadError(`${name} shape is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeRunAttemptLogReadRange(
|
||||
value: Readonly<RunAttemptLogReadRange>,
|
||||
maximumReadBytes = MAX_RUN_ATTEMPT_LOG_READ_BYTES,
|
||||
): Readonly<RunAttemptLogReadRange> {
|
||||
if (
|
||||
!Number.isSafeInteger(maximumReadBytes) ||
|
||||
maximumReadBytes < 1 ||
|
||||
maximumReadBytes > MAX_RUN_ATTEMPT_LOG_READ_BYTES
|
||||
) {
|
||||
throw new InvalidRunAttemptLogReadError('maximum read bytes is invalid');
|
||||
}
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new InvalidRunAttemptLogReadError('range is invalid');
|
||||
}
|
||||
exactKeys(value, ['length', 'offset'], [], 'range');
|
||||
if (!Number.isSafeInteger(value.offset) || value.offset < 0) {
|
||||
throw new InvalidRunAttemptLogReadError('offset is invalid');
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(value.length) ||
|
||||
value.length < 1 ||
|
||||
value.length > maximumReadBytes
|
||||
) {
|
||||
throw new InvalidRunAttemptLogReadError('length is invalid');
|
||||
}
|
||||
return Object.freeze({ offset: value.offset, length: value.length });
|
||||
}
|
||||
|
||||
function prepareOptions(
|
||||
options: RunAttemptLogReadServiceOptions,
|
||||
): Readonly<RunAttemptLogReadServiceOptions> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
(options.executorType !== 'local_process' &&
|
||||
options.executorType !== 'remote_worker') ||
|
||||
!(options.artifactIdPattern instanceof RegExp) ||
|
||||
options.artifactIdPattern.global ||
|
||||
options.artifactIdPattern.sticky ||
|
||||
(options.activeMissingIsPending !== undefined &&
|
||||
typeof options.activeMissingIsPending !== 'boolean')
|
||||
) {
|
||||
throw new InvalidRunAttemptLogReadError('service options are invalid');
|
||||
}
|
||||
exactKeys(
|
||||
options,
|
||||
['artifactIdPattern', 'executorType', 'maximumReadBytes'],
|
||||
['activeMissingIsPending'],
|
||||
'service options',
|
||||
);
|
||||
normalizeRunAttemptLogReadRange(
|
||||
{ offset: 0, length: options.maximumReadBytes },
|
||||
options.maximumReadBytes,
|
||||
);
|
||||
return Object.freeze({ ...options });
|
||||
}
|
||||
|
||||
function truncation(
|
||||
value: Readonly<RunAttemptLogTruncationView>,
|
||||
): Readonly<RunAttemptLogTruncationView> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
(value.truncated !== true &&
|
||||
value.truncated !== false &&
|
||||
value.truncated !== 'unknown') ||
|
||||
(value.maximumBytes !== undefined &&
|
||||
(!Number.isSafeInteger(value.maximumBytes) || value.maximumBytes < 1)) ||
|
||||
(value.observedAtMs !== undefined &&
|
||||
(!Number.isSafeInteger(value.observedAtMs) || value.observedAtMs < 0)) ||
|
||||
(value.truncated === 'unknown' &&
|
||||
(value.maximumBytes !== undefined || value.observedAtMs !== undefined))
|
||||
) {
|
||||
throw new RunAttemptLogReadUnavailableError();
|
||||
}
|
||||
return Object.freeze({ ...value });
|
||||
}
|
||||
|
||||
function available(
|
||||
identity: Readonly<RunAttemptLogReadIdentity>,
|
||||
range: Readonly<RunAttemptLogReadRange>,
|
||||
result: Extract<
|
||||
RunAttemptLogRangeReadResult,
|
||||
{ readonly status: 'available' }
|
||||
>,
|
||||
): RunAttemptLogReadResult {
|
||||
if (
|
||||
!(result.content instanceof Uint8Array) ||
|
||||
!Number.isSafeInteger(result.start) ||
|
||||
!Number.isSafeInteger(result.endExclusive) ||
|
||||
!Number.isSafeInteger(result.totalBytes) ||
|
||||
result.start !== Math.min(range.offset, result.totalBytes) ||
|
||||
result.endExclusive !== result.start + result.content.byteLength ||
|
||||
result.endExclusive > result.totalBytes ||
|
||||
result.content.byteLength > range.length ||
|
||||
(result.nextOffset === undefined) !==
|
||||
(result.endExclusive === result.totalBytes) ||
|
||||
(result.nextOffset !== undefined &&
|
||||
result.nextOffset !== result.endExclusive)
|
||||
) {
|
||||
throw new RunAttemptLogReadUnavailableError();
|
||||
}
|
||||
return Object.freeze({
|
||||
status: 'available' as const,
|
||||
...identity,
|
||||
content: result.content,
|
||||
start: result.start,
|
||||
endExclusive: result.endExclusive,
|
||||
totalBytes: result.totalBytes,
|
||||
...(result.nextOffset === undefined
|
||||
? {}
|
||||
: { nextOffset: result.nextOffset }),
|
||||
truncation: truncation(result.truncation),
|
||||
});
|
||||
}
|
||||
|
||||
export class RunAttemptLogReadService {
|
||||
private readonly options: Readonly<RunAttemptLogReadServiceOptions>;
|
||||
|
||||
constructor(
|
||||
private readonly runs: Pick<
|
||||
RunRepositoryReader,
|
||||
'findRunById' | 'findAttemptById'
|
||||
>,
|
||||
private readonly reader: RunAttemptLogRangeReader,
|
||||
options: RunAttemptLogReadServiceOptions,
|
||||
) {
|
||||
if (
|
||||
!runs ||
|
||||
typeof runs.findRunById !== 'function' ||
|
||||
typeof runs.findAttemptById !== 'function' ||
|
||||
!reader ||
|
||||
typeof reader.read !== 'function'
|
||||
) {
|
||||
throw new InvalidRunAttemptLogReadError('dependencies are invalid');
|
||||
}
|
||||
this.options = prepareOptions(options);
|
||||
}
|
||||
|
||||
async read(
|
||||
request: Readonly<RunAttemptLogReadRequest>,
|
||||
): Promise<RunAttemptLogReadResult> {
|
||||
if (!request || typeof request !== 'object' || Array.isArray(request)) {
|
||||
throw new InvalidRunAttemptLogReadError('request is invalid');
|
||||
}
|
||||
exactKeys(
|
||||
request,
|
||||
['attemptId', 'projectId', 'range', 'runId'],
|
||||
['signal'],
|
||||
'request',
|
||||
);
|
||||
const projectId = canonicalId('projectId', request.projectId);
|
||||
const runId = canonicalId('runId', request.runId);
|
||||
const attemptId = canonicalId('attemptId', request.attemptId);
|
||||
const range = normalizeRunAttemptLogReadRange(
|
||||
request.range,
|
||||
this.options.maximumReadBytes,
|
||||
);
|
||||
if (request.signal?.aborted) {
|
||||
throw new RunAttemptLogReadUnavailableError({
|
||||
cause: request.signal.reason,
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const run = await this.runs.findRunById(runId);
|
||||
if (
|
||||
!run ||
|
||||
run.id !== runId ||
|
||||
run.projectId !== projectId ||
|
||||
run.executionOwner !== 'runtime'
|
||||
) {
|
||||
return Object.freeze({ status: 'not_found' as const });
|
||||
}
|
||||
const attempt = await this.runs.findAttemptById(attemptId);
|
||||
if (
|
||||
!attempt ||
|
||||
attempt.id !== attemptId ||
|
||||
attempt.runId !== runId ||
|
||||
attempt.executorType !== this.options.executorType ||
|
||||
!RUN_ATTEMPT_STATUSES.includes(attempt.status)
|
||||
) {
|
||||
return Object.freeze({ status: 'not_found' as const });
|
||||
}
|
||||
if (attempt.logArtifactId === undefined) {
|
||||
return Object.freeze({
|
||||
status: 'pending' as const,
|
||||
projectId,
|
||||
runId,
|
||||
attemptId,
|
||||
});
|
||||
}
|
||||
if (!this.options.artifactIdPattern.test(attempt.logArtifactId)) {
|
||||
return Object.freeze({ status: 'not_found' as const });
|
||||
}
|
||||
const identity = Object.freeze({
|
||||
projectId,
|
||||
runId,
|
||||
attemptId,
|
||||
logArtifactId: attempt.logArtifactId,
|
||||
});
|
||||
const result = await this.reader.read(identity, range, request.signal);
|
||||
if (result.status === 'missing') {
|
||||
if (
|
||||
this.options.activeMissingIsPending === true &&
|
||||
!TERMINAL_ATTEMPT_STATUSES.has(attempt.status)
|
||||
) {
|
||||
return Object.freeze({ status: 'pending' as const, ...identity });
|
||||
}
|
||||
return Object.freeze({ status: 'missing' as const, ...identity });
|
||||
}
|
||||
return available(identity, range, result);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof InvalidRunAttemptLogReadError ||
|
||||
error instanceof RunAttemptLogReadUnavailableError
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
throw new RunAttemptLogReadUnavailableError({ cause: error });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
InvalidRunAttemptLogReadError,
|
||||
MAX_RUN_ATTEMPT_LOG_READ_BYTES,
|
||||
RunAttemptLogReadService,
|
||||
RunAttemptLogReadUnavailableError,
|
||||
normalizeRunAttemptLogReadRange,
|
||||
} = require('../dist/run/log-read/runAttemptLogRead.js');
|
||||
|
||||
function run(overrides = {}) {
|
||||
return {
|
||||
id: 'run_123',
|
||||
projectId: 'prj_default',
|
||||
taskId: 'task_1',
|
||||
taskRevision: 'revision_1',
|
||||
triggerType: 'task_start',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
status: 'running',
|
||||
version: 2,
|
||||
eventSequence: 2,
|
||||
priority: 0,
|
||||
createdAtMs: 1,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function attempt(overrides = {}) {
|
||||
return {
|
||||
id: 'attempt_123',
|
||||
runId: 'run_123',
|
||||
attempt: 1,
|
||||
status: 'running',
|
||||
executorType: 'local_process',
|
||||
logArtifactId: `local-${'a'.repeat(30)}`,
|
||||
callbackSequence: 0,
|
||||
createdAtMs: 1,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function service(overrides = {}) {
|
||||
const calls = [];
|
||||
const runs = overrides.runs ?? {
|
||||
async findRunById() {
|
||||
return run();
|
||||
},
|
||||
async findAttemptById() {
|
||||
return attempt();
|
||||
},
|
||||
};
|
||||
const reader = overrides.reader ?? {
|
||||
async read(identity, range, signal) {
|
||||
calls.push({ identity, range, signal });
|
||||
return {
|
||||
status: 'available',
|
||||
content: Buffer.from('log'),
|
||||
start: range.offset,
|
||||
endExclusive: range.offset + 3,
|
||||
totalBytes: range.offset + 5,
|
||||
nextOffset: range.offset + 3,
|
||||
truncation: { truncated: false, maximumBytes: 1024, observedAtMs: 9 },
|
||||
};
|
||||
},
|
||||
};
|
||||
return {
|
||||
calls,
|
||||
value: new RunAttemptLogReadService(runs, reader, {
|
||||
executorType: overrides.executorType ?? 'local_process',
|
||||
artifactIdPattern: overrides.artifactIdPattern ?? /^local-[a-f0-9]{30}$/,
|
||||
maximumReadBytes: overrides.maximumReadBytes ?? 32 * 1024,
|
||||
...(overrides.activeMissingIsPending === undefined
|
||||
? {}
|
||||
: { activeMissingIsPending: overrides.activeMissingIsPending }),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function request(overrides = {}) {
|
||||
return {
|
||||
projectId: 'prj_default',
|
||||
runId: 'run_123',
|
||||
attemptId: 'attempt_123',
|
||||
range: { offset: 4, length: 16 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('normalizes only bounded safe ranges', () => {
|
||||
assert.deepEqual(normalizeRunAttemptLogReadRange({ offset: 0, length: 1 }), {
|
||||
offset: 0,
|
||||
length: 1,
|
||||
});
|
||||
assert.deepEqual(
|
||||
normalizeRunAttemptLogReadRange({
|
||||
offset: Number.MAX_SAFE_INTEGER,
|
||||
length: MAX_RUN_ATTEMPT_LOG_READ_BYTES,
|
||||
}),
|
||||
{ offset: Number.MAX_SAFE_INTEGER, length: MAX_RUN_ATTEMPT_LOG_READ_BYTES },
|
||||
);
|
||||
for (const range of [
|
||||
{ offset: -1, length: 1 },
|
||||
{ offset: 0.5, length: 1 },
|
||||
{ offset: 0, length: 0 },
|
||||
{ offset: 0, length: MAX_RUN_ATTEMPT_LOG_READ_BYTES + 1 },
|
||||
]) {
|
||||
assert.throws(
|
||||
() => normalizeRunAttemptLogReadRange(range),
|
||||
InvalidRunAttemptLogReadError,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('validates Project, Run, Attempt, owner and executor before storage access', async () => {
|
||||
const cases = [
|
||||
{ run: null },
|
||||
{ run: run({ projectId: 'prj_other' }) },
|
||||
{ run: run({ executionOwner: 'legacy' }) },
|
||||
{ attempt: null },
|
||||
{ attempt: attempt({ runId: 'run_other' }) },
|
||||
{ attempt: attempt({ executorType: 'remote_worker' }) },
|
||||
{ attempt: attempt({ logArtifactId: `wlog-${'a'.repeat(30)}` }) },
|
||||
];
|
||||
for (const values of cases) {
|
||||
let reads = 0;
|
||||
const { value } = service({
|
||||
runs: {
|
||||
async findRunById() {
|
||||
return values.run === undefined ? run() : values.run;
|
||||
},
|
||||
async findAttemptById() {
|
||||
return values.attempt === undefined ? attempt() : values.attempt;
|
||||
},
|
||||
},
|
||||
reader: {
|
||||
async read() {
|
||||
reads += 1;
|
||||
return { status: 'missing' };
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.deepEqual(await value.read(request()), { status: 'not_found' });
|
||||
assert.equal(reads, 0);
|
||||
}
|
||||
});
|
||||
|
||||
test('returns pending before Artifact binding and for active remote publication lag', async () => {
|
||||
const unbound = service({
|
||||
runs: {
|
||||
async findRunById() {
|
||||
return run();
|
||||
},
|
||||
async findAttemptById() {
|
||||
return attempt({ logArtifactId: undefined });
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.deepEqual(await unbound.value.read(request()), {
|
||||
status: 'pending',
|
||||
projectId: 'prj_default',
|
||||
runId: 'run_123',
|
||||
attemptId: 'attempt_123',
|
||||
});
|
||||
assert.equal(unbound.calls.length, 0);
|
||||
|
||||
const remote = service({
|
||||
executorType: 'remote_worker',
|
||||
artifactIdPattern: /^wlog-[a-f0-9]{30}$/,
|
||||
activeMissingIsPending: true,
|
||||
runs: {
|
||||
async findRunById() {
|
||||
return run();
|
||||
},
|
||||
async findAttemptById() {
|
||||
return attempt({
|
||||
executorType: 'remote_worker',
|
||||
logArtifactId: `wlog-${'b'.repeat(30)}`,
|
||||
});
|
||||
},
|
||||
},
|
||||
reader: {
|
||||
async read() {
|
||||
return { status: 'missing' };
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.equal((await remote.value.read(request())).status, 'pending');
|
||||
});
|
||||
|
||||
test('returns a validated bounded snapshot without copying storage bytes', async () => {
|
||||
const content = Buffer.from('log');
|
||||
const abort = new AbortController();
|
||||
const { value, calls } = service({
|
||||
reader: {
|
||||
async read(identity, range, signal) {
|
||||
assert.equal(signal, abort.signal);
|
||||
return {
|
||||
status: 'available',
|
||||
content,
|
||||
start: 4,
|
||||
endExclusive: 7,
|
||||
totalBytes: 9,
|
||||
nextOffset: 7,
|
||||
truncation: {
|
||||
truncated: true,
|
||||
maximumBytes: 64 * 1024,
|
||||
observedAtMs: 10,
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
});
|
||||
const result = await value.read(request({ signal: abort.signal }));
|
||||
assert.equal(result.status, 'available');
|
||||
assert.equal(result.content, content);
|
||||
assert.equal(result.nextOffset, 7);
|
||||
assert.deepEqual(result.truncation, {
|
||||
truncated: true,
|
||||
maximumBytes: 64 * 1024,
|
||||
observedAtMs: 10,
|
||||
});
|
||||
assert.equal(calls.length, 0);
|
||||
});
|
||||
|
||||
test('fails closed on malformed storage results and dependency failures', async () => {
|
||||
const malformed = service({
|
||||
reader: {
|
||||
async read() {
|
||||
return {
|
||||
status: 'available',
|
||||
content: Buffer.from('too-long'),
|
||||
start: 4,
|
||||
endExclusive: 12,
|
||||
totalBytes: 9,
|
||||
truncation: { truncated: 'unknown' },
|
||||
};
|
||||
},
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
malformed.value.read(request()),
|
||||
RunAttemptLogReadUnavailableError,
|
||||
);
|
||||
|
||||
const failed = service({
|
||||
runs: {
|
||||
async findRunById() {
|
||||
throw new Error('database detail');
|
||||
},
|
||||
async findAttemptById() {
|
||||
return null;
|
||||
},
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
failed.value.read(request()),
|
||||
RunAttemptLogReadUnavailableError,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user