mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): add profile-aware run log reads
This commit is contained in:
@@ -24,6 +24,7 @@ import { createClusterControlRunReadRoute } from '../run/runReadRoute';
|
||||
import { createClusterControlRunListRoute } from '../run/runListRoute';
|
||||
import { createClusterControlRunEventListRoute } from '../run/runEventListRoute';
|
||||
import { createClusterControlRunStepListRoute } from '../run/runStepListRoute';
|
||||
import { createClusterControlRunAttemptLogReadRoute } from '../run/runAttemptLogReadRoute';
|
||||
import { createClusterControlTaskListRoute } from '../task/taskListRoute';
|
||||
import { createClusterControlTaskReadRoute } from '../task/taskReadRoute';
|
||||
import { createClusterControlTaskStartRoute } from '../task/taskStartRoute';
|
||||
@@ -69,6 +70,7 @@ export const PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS = Object.freeze([
|
||||
'run.list',
|
||||
'run.events.list',
|
||||
'run.steps.list',
|
||||
'run.log.read',
|
||||
'run.cancel',
|
||||
'workflow.read',
|
||||
'workflow.run.read',
|
||||
@@ -195,6 +197,10 @@ export function createProductionClusterControlApplicationStack(
|
||||
input.runs,
|
||||
input.trustedToolStorage.stepRuns,
|
||||
),
|
||||
createClusterControlRunAttemptLogReadRoute(
|
||||
input.runs,
|
||||
input.workerRuntime?.runAttemptLogRead,
|
||||
),
|
||||
createClusterControlRunCancellationRoute(
|
||||
input.runCancellation,
|
||||
createEventId,
|
||||
|
||||
@@ -6,12 +6,19 @@ import {
|
||||
ChecksumMode,
|
||||
CopyObjectCommand,
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
MetadataDirective,
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
ServerSideEncryption,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import {
|
||||
normalizeRunAttemptLogReadRange,
|
||||
type RunAttemptLogRangeReadResult,
|
||||
type RunAttemptLogReadIdentity,
|
||||
type RunAttemptLogReadRange,
|
||||
} from '@qinglong/runtime-core/run-attempt-log-read';
|
||||
import {
|
||||
MAX_REMOTE_WORKER_ARTIFACT_BYTES,
|
||||
REMOTE_WORKER_ARTIFACT_CONTENT_TYPE,
|
||||
@@ -31,7 +38,8 @@ const TEMPORARY_METADATA_SCHEMA =
|
||||
const SHA256_PATTERN = /^[a-f0-9]{64}$/;
|
||||
const BUCKET_PATTERN = /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/;
|
||||
const PREFIX_PATTERN = /^[A-Za-z0-9][A-Za-z0-9/_=-]{0,254}$/;
|
||||
const TEMPORARY_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
const TEMPORARY_ID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
|
||||
type S3SendClient = Pick<S3Client, 'send'>;
|
||||
|
||||
@@ -70,8 +78,7 @@ export function createS3ClusterRemoteWorkerArtifactClient(
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
!/^[a-z0-9][a-z0-9-]{0,62}$/.test(options.region) ||
|
||||
(options.endpoint !== undefined &&
|
||||
typeof options.endpoint !== 'string') ||
|
||||
(options.endpoint !== undefined && typeof options.endpoint !== 'string') ||
|
||||
(options.forcePathStyle !== undefined &&
|
||||
typeof options.forcePathStyle !== 'boolean')
|
||||
) {
|
||||
@@ -79,9 +86,7 @@ export function createS3ClusterRemoteWorkerArtifactClient(
|
||||
}
|
||||
return new S3Client({
|
||||
region: options.region,
|
||||
...(options.endpoint === undefined
|
||||
? {}
|
||||
: { endpoint: options.endpoint }),
|
||||
...(options.endpoint === undefined ? {} : { endpoint: options.endpoint }),
|
||||
forcePathStyle: options.forcePathStyle ?? false,
|
||||
});
|
||||
}
|
||||
@@ -116,6 +121,11 @@ type ArtifactAuthority = Readonly<{
|
||||
logArtifactId: string;
|
||||
}>;
|
||||
|
||||
type StoredArtifactHead = Readonly<{
|
||||
receipt: Readonly<RemoteWorkerArtifactReceipt>;
|
||||
eTag?: string;
|
||||
}>;
|
||||
|
||||
type NormalizedStorageCommand = ArtifactAuthority &
|
||||
Readonly<{
|
||||
byteLength: number;
|
||||
@@ -127,7 +137,9 @@ const DIAGNOSTIC_CONTEXT = Object.freeze({
|
||||
});
|
||||
|
||||
function configurationError(message: string): TypeError {
|
||||
return new TypeError(`S3 Remote Worker Artifact store is invalid: ${message}`);
|
||||
return new TypeError(
|
||||
`S3 Remote Worker Artifact store is invalid: ${message}`,
|
||||
);
|
||||
}
|
||||
|
||||
function prepareOptions(
|
||||
@@ -175,14 +187,15 @@ function prepareOptions(
|
||||
throw configurationError('expected bucket owner is invalid');
|
||||
}
|
||||
const encryption = options.encryption;
|
||||
if (!encryption || typeof encryption !== 'object' || Array.isArray(encryption)) {
|
||||
if (
|
||||
!encryption ||
|
||||
typeof encryption !== 'object' ||
|
||||
Array.isArray(encryption)
|
||||
) {
|
||||
throw configurationError('encryption is required');
|
||||
}
|
||||
let preparedEncryption: PreparedOptions['encryption'];
|
||||
if (
|
||||
encryption.mode === 's3' &&
|
||||
Object.keys(encryption).length === 1
|
||||
) {
|
||||
if (encryption.mode === 's3' && Object.keys(encryption).length === 1) {
|
||||
preparedEncryption = Object.freeze({
|
||||
ServerSideEncryption: ServerSideEncryption.AES256,
|
||||
});
|
||||
@@ -383,9 +396,8 @@ function finalMetadata(
|
||||
),
|
||||
'ql3-byte-length': String(command.byteLength),
|
||||
'ql3-content-sha256': sha256,
|
||||
'ql3-truncated': command.truncated === undefined
|
||||
? 'omitted'
|
||||
: String(command.truncated),
|
||||
'ql3-truncated':
|
||||
command.truncated === undefined ? 'omitted' : String(command.truncated),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -460,6 +472,99 @@ function parseStoredReceipt(
|
||||
);
|
||||
}
|
||||
|
||||
function canonicalETag(value: unknown): string {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 3 ||
|
||||
value.length > 256 ||
|
||||
!/^"[^"\u0000-\u001f\u007f]+"$/.test(value)
|
||||
) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function assertRangeMetadata(
|
||||
authority: ArtifactAuthority,
|
||||
receipt: Readonly<RemoteWorkerArtifactReceipt>,
|
||||
output: Readonly<{
|
||||
ContentLength?: number | undefined;
|
||||
ContentRange?: string | undefined;
|
||||
ContentType?: string | undefined;
|
||||
ETag?: string | undefined;
|
||||
Metadata?: Readonly<Record<string, string | undefined>> | undefined;
|
||||
}>,
|
||||
eTag: string,
|
||||
start: number,
|
||||
endExclusive: number,
|
||||
): void {
|
||||
const metadata = output.Metadata;
|
||||
const truncated =
|
||||
receipt.truncated === undefined ? 'omitted' : String(receipt.truncated);
|
||||
if (
|
||||
output.ContentLength !== endExclusive - start ||
|
||||
output.ContentRange !==
|
||||
`bytes ${start}-${endExclusive - 1}/${receipt.byteLength}` ||
|
||||
output.ContentType !== REMOTE_WORKER_ARTIFACT_CONTENT_TYPE ||
|
||||
canonicalETag(output.ETag) !== eTag ||
|
||||
metadataValue(metadata, 'ql3-schema') !== METADATA_SCHEMA ||
|
||||
metadataValue(metadata, 'ql3-project-sha256') !==
|
||||
fieldDigest('project', authority.projectId) ||
|
||||
metadataValue(metadata, 'ql3-run-sha256') !==
|
||||
fieldDigest('run', authority.runId) ||
|
||||
metadataValue(metadata, 'ql3-attempt-sha256') !==
|
||||
fieldDigest('attempt', authority.attemptId) ||
|
||||
metadataValue(metadata, 'ql3-log-artifact-sha256') !==
|
||||
fieldDigest('log-artifact', authority.logArtifactId) ||
|
||||
metadataValue(metadata, 'ql3-byte-length') !== String(receipt.byteLength) ||
|
||||
metadataValue(metadata, 'ql3-content-sha256') !== receipt.sha256 ||
|
||||
metadataValue(metadata, 'ql3-truncated') !== truncated
|
||||
) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
}
|
||||
|
||||
async function readBoundedRangeBody(
|
||||
body: unknown,
|
||||
expectedBytes: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Buffer> {
|
||||
if (
|
||||
!body ||
|
||||
typeof body !== 'object' ||
|
||||
!(Symbol.asyncIterator in body) ||
|
||||
typeof (body as AsyncIterable<Uint8Array>)[Symbol.asyncIterator] !==
|
||||
'function'
|
||||
) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
const content = Buffer.allocUnsafe(expectedBytes);
|
||||
let received = 0;
|
||||
try {
|
||||
for await (const chunk of body as AsyncIterable<Uint8Array>) {
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
if (!(chunk instanceof Uint8Array) || chunk.byteLength === 0) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
if (received + chunk.byteLength > expectedBytes) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength).copy(
|
||||
content,
|
||||
received,
|
||||
);
|
||||
received += chunk.byteLength;
|
||||
}
|
||||
if (received !== expectedBytes) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
return content;
|
||||
} catch (error) {
|
||||
content.fill(0);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function isNotFound(error: unknown): boolean {
|
||||
if (!error || typeof error !== 'object') return false;
|
||||
const value = error as {
|
||||
@@ -467,13 +572,17 @@ function isNotFound(error: unknown): boolean {
|
||||
Code?: unknown;
|
||||
$metadata?: { httpStatusCode?: unknown };
|
||||
};
|
||||
return value.name === 'NotFound' ||
|
||||
return (
|
||||
value.name === 'NotFound' ||
|
||||
value.name === 'NoSuchKey' ||
|
||||
value.Code === 'NoSuchKey' ||
|
||||
value.$metadata?.httpStatusCode === 404;
|
||||
value.$metadata?.httpStatusCode === 404
|
||||
);
|
||||
}
|
||||
|
||||
function requestOptions(signal?: AbortSignal): { abortSignal: AbortSignal } | undefined {
|
||||
function requestOptions(
|
||||
signal?: AbortSignal,
|
||||
): { abortSignal: AbortSignal } | undefined {
|
||||
return signal === undefined ? undefined : { abortSignal: signal };
|
||||
}
|
||||
|
||||
@@ -549,7 +658,8 @@ class ArtifactContentDigest {
|
||||
* objects are never overwritten or deleted by this adapter.
|
||||
*/
|
||||
export class S3ClusterRemoteWorkerArtifactStore
|
||||
implements ClusterRemoteWorkerArtifactStore {
|
||||
implements ClusterRemoteWorkerArtifactStore
|
||||
{
|
||||
private readonly options: PreparedOptions;
|
||||
|
||||
constructor(options: S3ClusterRemoteWorkerArtifactStoreOptions) {
|
||||
@@ -561,6 +671,90 @@ export class S3ClusterRemoteWorkerArtifactStore
|
||||
signal?: AbortSignal,
|
||||
): Promise<Readonly<RemoteWorkerArtifactReceipt> | undefined> {
|
||||
const authority = normalizeLookup(lookup);
|
||||
return (await this.head(authority, signal))?.receipt;
|
||||
}
|
||||
|
||||
async readLogRange(
|
||||
rawIdentity: Readonly<RunAttemptLogReadIdentity>,
|
||||
rawRange: Readonly<RunAttemptLogReadRange>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RunAttemptLogRangeReadResult> {
|
||||
const authority = normalizeLookup(rawIdentity);
|
||||
const range = normalizeRunAttemptLogReadRange(rawRange);
|
||||
const stored = await this.head(authority, signal);
|
||||
if (!stored) return Object.freeze({ status: 'missing' as const });
|
||||
const start = Math.min(range.offset, stored.receipt.byteLength);
|
||||
const endExclusive = Math.min(
|
||||
start + range.length,
|
||||
stored.receipt.byteLength,
|
||||
);
|
||||
const truncation = Object.freeze({
|
||||
truncated: stored.receipt.truncated ?? ('unknown' as const),
|
||||
});
|
||||
if (start === endExclusive) {
|
||||
return Object.freeze({
|
||||
status: 'available' as const,
|
||||
content: Buffer.alloc(0),
|
||||
start,
|
||||
endExclusive,
|
||||
totalBytes: stored.receipt.byteLength,
|
||||
truncation,
|
||||
});
|
||||
}
|
||||
const eTag = canonicalETag(stored.eTag);
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
try {
|
||||
const output = await this.options.client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: finalObjectKey(this.options.prefix, authority),
|
||||
IfMatch: eTag,
|
||||
Range: `bytes=${start}-${endExclusive - 1}`,
|
||||
...(this.options.expectedBucketOwner === undefined
|
||||
? {}
|
||||
: { ExpectedBucketOwner: this.options.expectedBucketOwner }),
|
||||
}),
|
||||
requestOptions(signal),
|
||||
);
|
||||
assertRangeMetadata(
|
||||
authority,
|
||||
stored.receipt,
|
||||
output,
|
||||
eTag,
|
||||
start,
|
||||
endExclusive,
|
||||
);
|
||||
const bytes = await readBoundedRangeBody(
|
||||
output.Body,
|
||||
endExclusive - start,
|
||||
signal,
|
||||
);
|
||||
return Object.freeze({
|
||||
status: 'available' as const,
|
||||
content: bytes,
|
||||
start,
|
||||
endExclusive,
|
||||
totalBytes: stored.receipt.byteLength,
|
||||
...(endExclusive < stored.receipt.byteLength
|
||||
? { nextOffset: endExclusive }
|
||||
: {}),
|
||||
truncation,
|
||||
});
|
||||
} catch (error) {
|
||||
if (isNotFound(error)) {
|
||||
return Object.freeze({ status: 'missing' as const });
|
||||
}
|
||||
if (error instanceof S3ClusterRemoteWorkerArtifactStoreError) throw error;
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('unavailable', {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async head(
|
||||
authority: ArtifactAuthority,
|
||||
signal?: AbortSignal,
|
||||
): Promise<StoredArtifactHead | undefined> {
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
try {
|
||||
const output = await this.options.client.send(
|
||||
@@ -574,7 +768,10 @@ export class S3ClusterRemoteWorkerArtifactStore
|
||||
}),
|
||||
requestOptions(signal),
|
||||
);
|
||||
return parseStoredReceipt(authority, output);
|
||||
return Object.freeze({
|
||||
receipt: parseStoredReceipt(authority, output),
|
||||
...(output.ETag === undefined ? {} : { eTag: output.ETag }),
|
||||
});
|
||||
} catch (error) {
|
||||
if (isNotFound(error)) return undefined;
|
||||
if (error instanceof S3ClusterRemoteWorkerArtifactStoreError) throw error;
|
||||
@@ -604,9 +801,7 @@ export class S3ClusterRemoteWorkerArtifactStore
|
||||
existing.truncated !== command.truncated ||
|
||||
existing.sha256 !== incomingSha256
|
||||
) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError(
|
||||
'integrity_mismatch',
|
||||
);
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
return existing;
|
||||
}
|
||||
@@ -693,13 +888,11 @@ export class S3ClusterRemoteWorkerArtifactStore
|
||||
stored.truncated !== command.truncated ||
|
||||
stored.sha256 !== sha256
|
||||
) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError(
|
||||
'integrity_mismatch',
|
||||
);
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
result = Object.freeze({
|
||||
...stored,
|
||||
status: copied ? 'stored' as const : 'already_stored' as const,
|
||||
status: copied ? ('stored' as const) : ('already_stored' as const),
|
||||
});
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
@@ -770,9 +963,7 @@ export class S3ClusterRemoteWorkerArtifactStore
|
||||
output.Metadata?.['ql3-owner-sha256'] !== ownerSha256 ||
|
||||
canonicalChecksum(output.ChecksumSHA256) !== sha256
|
||||
) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError(
|
||||
'integrity_mismatch',
|
||||
);
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,11 @@ import {
|
||||
type RemoteWorkerCompletionRepository,
|
||||
type RemoteWorkerCompletionResult,
|
||||
} from '@qinglong/runtime-core/remote-worker-completion';
|
||||
import type {
|
||||
RunAttemptLogRangeReadResult,
|
||||
RunAttemptLogReadIdentity,
|
||||
RunAttemptLogReadRange,
|
||||
} from '@qinglong/runtime-core/run-attempt-log-read';
|
||||
|
||||
export interface ClusterRemoteWorkerArtifactStorageCommand {
|
||||
readonly projectId: string;
|
||||
@@ -47,6 +52,12 @@ export interface ClusterRemoteWorkerArtifactStore {
|
||||
lookup: Readonly<ClusterRemoteWorkerArtifactLookup>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Readonly<RemoteWorkerArtifactReceipt> | undefined>;
|
||||
/** Optional during Alpha so upload-only test and alternate stores remain compatible. */
|
||||
readLogRange?(
|
||||
identity: Readonly<RunAttemptLogReadIdentity>,
|
||||
range: Readonly<RunAttemptLogReadRange>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RunAttemptLogRangeReadResult>;
|
||||
}
|
||||
|
||||
export interface ClusterRemoteWorkerArtifactUploadInput {
|
||||
@@ -353,11 +364,13 @@ export class ClusterRemoteWorkerCompletionService {
|
||||
}
|
||||
try {
|
||||
const result = normalizeRemoteWorkerCompletionResult(
|
||||
await this.repository.complete(Object.freeze({
|
||||
...command,
|
||||
attemptEventId: eventId(this.createEventId),
|
||||
runEventId: eventId(this.createEventId),
|
||||
})),
|
||||
await this.repository.complete(
|
||||
Object.freeze({
|
||||
...command,
|
||||
attemptEventId: eventId(this.createEventId),
|
||||
runEventId: eventId(this.createEventId),
|
||||
}),
|
||||
),
|
||||
);
|
||||
if (
|
||||
result.runId !== command.runId ||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// Remote execution owns the least-privilege assembly of Worker-facing runtime capabilities.
|
||||
import type { PostgresPool } from '@qinglong/runtime-core';
|
||||
import type { RemoteWorkerSecretValueProvider } from '@qinglong/runtime-core/remote-secret-delivery';
|
||||
import type { RunAttemptLogRangeReader } from '@qinglong/runtime-core/run-attempt-log-read';
|
||||
import {
|
||||
PostgresClusterDispatchSource,
|
||||
PostgresRemoteRunActivationRepository,
|
||||
@@ -11,23 +12,15 @@ import {
|
||||
PostgresTaskExecutionRevisionSource,
|
||||
PostgresWorkerSessionRepository,
|
||||
} from '@qinglong/cluster-postgres/runtime';
|
||||
import {
|
||||
ClusterRemoteWorkerOfferClaimService,
|
||||
} from './remoteWorkerDispatcher';
|
||||
import {
|
||||
ClusterRemoteRunActivationService,
|
||||
} from './remoteRunActivationService';
|
||||
import {
|
||||
ClusterRemoteWorkerSecretDeliveryService,
|
||||
} from './remoteWorkerSecretDeliveryService';
|
||||
import { ClusterRemoteWorkerOfferClaimService } from './remoteWorkerDispatcher';
|
||||
import { ClusterRemoteRunActivationService } from './remoteRunActivationService';
|
||||
import { ClusterRemoteWorkerSecretDeliveryService } from './remoteWorkerSecretDeliveryService';
|
||||
import {
|
||||
ClusterRemoteWorkerArtifactService,
|
||||
ClusterRemoteWorkerCompletionService,
|
||||
type ClusterRemoteWorkerArtifactStore,
|
||||
} from './remoteWorkerCompletionService';
|
||||
import {
|
||||
ClusterRemoteWorkerLeaseControlService,
|
||||
} from './remoteWorkerLeaseControlService';
|
||||
import { ClusterRemoteWorkerLeaseControlService } from './remoteWorkerLeaseControlService';
|
||||
import type { WorkerIngressPipelineOptions } from '../worker-ingress/workerIngressPipeline';
|
||||
|
||||
export interface ClusterWorkerRuntimeDependencies {
|
||||
@@ -49,6 +42,7 @@ export interface ClusterWorkerRuntimePort {
|
||||
readonly leaseControl: NonNullable<
|
||||
WorkerIngressPipelineOptions['leaseControl']
|
||||
>;
|
||||
readonly runAttemptLogRead?: RunAttemptLogRangeReader;
|
||||
}
|
||||
|
||||
export function createClusterWorkerRuntimePort(
|
||||
@@ -67,9 +61,11 @@ export function createClusterWorkerRuntimePort(
|
||||
}
|
||||
|
||||
const workerSessions = new PostgresWorkerSessionRepository(pool);
|
||||
const completionRepository =
|
||||
new PostgresRemoteWorkerCompletionRepository(pool);
|
||||
const completionRepository = new PostgresRemoteWorkerCompletionRepository(
|
||||
pool,
|
||||
);
|
||||
const secretProvider = dependencies.secretProvider;
|
||||
const readLogRange = dependencies.artifactStore.readLogRange;
|
||||
return Object.freeze({
|
||||
offers: new ClusterRemoteWorkerOfferClaimService(
|
||||
new PostgresClusterDispatchSource(pool),
|
||||
@@ -99,5 +95,12 @@ export function createClusterWorkerRuntimePort(
|
||||
leaseControl: new ClusterRemoteWorkerLeaseControlService(
|
||||
new PostgresRemoteWorkerLeaseControlRepository(pool),
|
||||
),
|
||||
...(readLogRange === undefined
|
||||
? {}
|
||||
: {
|
||||
runAttemptLogRead: Object.freeze({
|
||||
read: readLogRange.bind(dependencies.artifactStore),
|
||||
}),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
import {
|
||||
InvalidRunAttemptLogReadError,
|
||||
RunAttemptLogReadService,
|
||||
RunAttemptLogReadUnavailableError,
|
||||
type RunAttemptLogRangeReader,
|
||||
type RunAttemptLogReadResult,
|
||||
} from '@qinglong/runtime-core/run-attempt-log-read';
|
||||
import type { RunRepositoryReader } from '@qinglong/runtime-core/run-repository';
|
||||
|
||||
import type { ClusterControlAdmissionResponse } from '../transport/httpSurface';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
ClusterControlRouteParameters,
|
||||
} from '../transport/routeRegistry';
|
||||
|
||||
export const CLUSTER_CONTROL_RUN_ATTEMPT_LOG_READ_ROUTE = Object.freeze({
|
||||
method: 'GET' as const,
|
||||
path: '/api/v3/projects/{projectId}/runs/{runId}/attempts/{attemptId}/log',
|
||||
operationId: 'run.log.read',
|
||||
permission: 'artifact.read',
|
||||
projectParameter: 'projectId',
|
||||
allowedQuery: Object.freeze(['length', 'offset']),
|
||||
});
|
||||
|
||||
const DEFAULT_READ_BYTES = 64 * 1024;
|
||||
const MAXIMUM_READ_BYTES = 256 * 1024;
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
function parseQuery(
|
||||
query: Readonly<Record<string, readonly string[]>>,
|
||||
): Readonly<{ offset: number; length: number }> {
|
||||
const offsetValues = query.offset;
|
||||
const lengthValues = query.length;
|
||||
if (
|
||||
(offsetValues !== undefined && offsetValues.length !== 1) ||
|
||||
(lengthValues !== undefined && lengthValues.length !== 1)
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
const rawOffset = offsetValues?.[0];
|
||||
const offset = rawOffset === undefined ? 0 : Number(rawOffset);
|
||||
const rawLength = lengthValues?.[0];
|
||||
const length =
|
||||
rawLength === undefined ? DEFAULT_READ_BYTES : Number(rawLength);
|
||||
if (
|
||||
!Number.isSafeInteger(offset) ||
|
||||
offset < 0 ||
|
||||
(rawOffset !== undefined && String(offset) !== rawOffset) ||
|
||||
!Number.isSafeInteger(length) ||
|
||||
length < 1 ||
|
||||
length > MAXIMUM_READ_BYTES ||
|
||||
(rawLength !== undefined && String(length) !== rawLength)
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
return Object.freeze({ offset, length });
|
||||
}
|
||||
|
||||
function validateQuery(
|
||||
query: Readonly<Record<string, readonly string[]>>,
|
||||
): void {
|
||||
parseQuery(query);
|
||||
}
|
||||
|
||||
function projection(
|
||||
result: Extract<RunAttemptLogReadResult, { readonly status: 'available' }>,
|
||||
): Readonly<Record<string, unknown>> {
|
||||
return Object.freeze({
|
||||
schema: 'qinglong/run-attempt-log-read-result@v1',
|
||||
status: 'available',
|
||||
projectId: result.projectId,
|
||||
runId: result.runId,
|
||||
attemptId: result.attemptId,
|
||||
range: Object.freeze({
|
||||
start: result.start,
|
||||
endExclusive: result.endExclusive,
|
||||
totalBytes: result.totalBytes,
|
||||
...(result.nextOffset === undefined
|
||||
? {}
|
||||
: { nextOffset: result.nextOffset }),
|
||||
}),
|
||||
encoding: 'base64',
|
||||
content: Buffer.from(
|
||||
result.content.buffer,
|
||||
result.content.byteOffset,
|
||||
result.content.byteLength,
|
||||
).toString('base64'),
|
||||
truncation: result.truncation,
|
||||
});
|
||||
}
|
||||
|
||||
export function createClusterControlRunAttemptLogReadRoute(
|
||||
runs: Pick<RunRepositoryReader, 'findRunById' | 'findAttemptById'>,
|
||||
reader?: RunAttemptLogRangeReader,
|
||||
): Readonly<ClusterControlRouteDefinition> {
|
||||
if (
|
||||
!runs ||
|
||||
typeof runs.findRunById !== 'function' ||
|
||||
typeof runs.findAttemptById !== 'function' ||
|
||||
(reader !== undefined && typeof reader.read !== 'function')
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Cluster-control Run Attempt log read dependencies are invalid',
|
||||
);
|
||||
}
|
||||
const service =
|
||||
reader === undefined
|
||||
? undefined
|
||||
: new RunAttemptLogReadService(runs, reader, {
|
||||
executorType: 'remote_worker',
|
||||
artifactIdPattern: /^wlog-[a-f0-9]{30}$/,
|
||||
maximumReadBytes: MAXIMUM_READ_BYTES,
|
||||
activeMissingIsPending: true,
|
||||
});
|
||||
return Object.freeze({
|
||||
...CLUSTER_CONTROL_RUN_ATTEMPT_LOG_READ_ROUTE,
|
||||
validateQuery,
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
if (authorized.request.body !== null) {
|
||||
return response(400, { code: 'invalid_request_body' });
|
||||
}
|
||||
if (authorized.projectId === null || service === undefined) {
|
||||
return response(503, { code: 'artifact_unavailable' });
|
||||
}
|
||||
let range;
|
||||
try {
|
||||
range = parseQuery(authorized.request.query);
|
||||
} catch {
|
||||
return response(400, { code: 'invalid_run_log_read_query' });
|
||||
}
|
||||
try {
|
||||
const result = await service.read({
|
||||
projectId: authorized.projectId,
|
||||
runId: parameters.runId!,
|
||||
attemptId: parameters.attemptId!,
|
||||
range,
|
||||
signal: authorized.request.signal,
|
||||
});
|
||||
if (result.status === 'not_found') {
|
||||
return response(404, { code: 'artifact_not_found' });
|
||||
}
|
||||
if (result.status === 'pending') {
|
||||
return response(202, {
|
||||
schema: 'qinglong/run-attempt-log-read-result@v1',
|
||||
status: 'pending',
|
||||
projectId: result.projectId,
|
||||
runId: result.runId,
|
||||
attemptId: result.attemptId,
|
||||
});
|
||||
}
|
||||
if (result.status === 'missing') {
|
||||
return response(503, { code: 'artifact_unavailable' });
|
||||
}
|
||||
return response(200, projection(result));
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidRunAttemptLogReadError) {
|
||||
return response(400, { code: 'invalid_run_log_read_request' });
|
||||
}
|
||||
if (error instanceof RunAttemptLogReadUnavailableError) {
|
||||
return response(503, { code: 'artifact_unavailable' });
|
||||
}
|
||||
return response(503, { code: 'artifact_unavailable' });
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -308,6 +308,13 @@ export function createClusterControlAdmissionPipeline(
|
||||
now,
|
||||
);
|
||||
if (decision.effect === 'deny') {
|
||||
if (route.permission === 'artifact.read') {
|
||||
throw securityError(
|
||||
404,
|
||||
'artifact_not_found',
|
||||
'Cluster-control Artifact is not available',
|
||||
);
|
||||
}
|
||||
throw securityError(
|
||||
403,
|
||||
'forbidden',
|
||||
@@ -315,6 +322,13 @@ export function createClusterControlAdmissionPipeline(
|
||||
);
|
||||
}
|
||||
if (decision.effect === 'require_approval') {
|
||||
if (route.permission === 'artifact.read') {
|
||||
throw securityError(
|
||||
404,
|
||||
'artifact_not_found',
|
||||
'Cluster-control Artifact is not available',
|
||||
);
|
||||
}
|
||||
throw securityError(
|
||||
403,
|
||||
'approval_required',
|
||||
|
||||
@@ -573,6 +573,9 @@ test('injects reviewed Worker operations without exposing the runtime Pool', asy
|
||||
async inspect() {
|
||||
throw new Error('not invoked during assembly');
|
||||
},
|
||||
async readLogRange() {
|
||||
throw new Error('not invoked during assembly');
|
||||
},
|
||||
};
|
||||
const result = await bootstrapClusterControlRuntime(
|
||||
bootstrapOptions(events, {
|
||||
@@ -596,6 +599,10 @@ test('injects reviewed Worker operations without exposing the runtime Pool', asy
|
||||
typeof input.workerRuntime.leaseControl.control,
|
||||
'function',
|
||||
);
|
||||
assert.equal(
|
||||
typeof input.workerRuntime.runAttemptLogRead.read,
|
||||
'function',
|
||||
);
|
||||
return activationStack(events);
|
||||
},
|
||||
}),
|
||||
|
||||
@@ -143,6 +143,9 @@ function fixture(overrides = {}) {
|
||||
},
|
||||
];
|
||||
},
|
||||
async findAttemptById() {
|
||||
return null;
|
||||
},
|
||||
},
|
||||
trustedToolStorage: {
|
||||
stepRuns: {
|
||||
@@ -190,7 +193,8 @@ function fixture(overrides = {}) {
|
||||
taskDefinitions: {
|
||||
async findCurrentTaskDefinition(projectId, taskId) {
|
||||
events.push(`task-get:${projectId}:${taskId}`);
|
||||
return projectId === currentTask.projectId && taskId === currentTask.taskId
|
||||
return projectId === currentTask.projectId &&
|
||||
taskId === currentTask.taskId
|
||||
? currentTask
|
||||
: null;
|
||||
},
|
||||
@@ -328,6 +332,7 @@ test('production composition exposes the reviewed Run and Workflow routes', asyn
|
||||
'run.list',
|
||||
'run.events.list',
|
||||
'run.steps.list',
|
||||
'run.log.read',
|
||||
'run.cancel',
|
||||
'workflow.read',
|
||||
'workflow.run.read',
|
||||
@@ -401,6 +406,15 @@ test('production composition exposes the reviewed Run and Workflow routes', asyn
|
||||
body: { steps: [], hasMore: false, next: null },
|
||||
});
|
||||
|
||||
const log = await invoke(
|
||||
stack,
|
||||
metadata('/api/v3/projects/project-1/runs/run-1/attempts/attempt-1/log'),
|
||||
);
|
||||
assert.deepEqual(log, {
|
||||
statusCode: 503,
|
||||
body: { code: 'artifact_unavailable' },
|
||||
});
|
||||
|
||||
const cancellation = await invoke(
|
||||
stack,
|
||||
metadata('/api/v3/projects/project-1/runs/run-1/cancellation', 'POST', {
|
||||
@@ -414,6 +428,7 @@ test('production composition exposes the reviewed Run and Workflow routes', asyn
|
||||
assert.equal(events.includes('audit:run.get:allowed'), true);
|
||||
assert.equal(events.includes('audit:run.events.list:allowed'), true);
|
||||
assert.equal(events.includes('audit:run.steps.list:allowed'), true);
|
||||
assert.equal(events.includes('audit:run.log.read:allowed'), true);
|
||||
assert.equal(events.includes('audit:run.cancel:allowed'), true);
|
||||
|
||||
const workflows = await invoke(
|
||||
@@ -539,6 +554,69 @@ test('production composition fails closed for an unreviewed route', async () =>
|
||||
);
|
||||
});
|
||||
|
||||
test('wires the production Worker object reader into the Project-scoped log route', async () => {
|
||||
const { input } = fixture();
|
||||
const run = await input.runs.findRunById('run-1');
|
||||
const logArtifactId = `wlog-${'a'.repeat(30)}`;
|
||||
const stack = createProductionClusterControlApplicationStack({
|
||||
...input,
|
||||
runs: {
|
||||
...input.runs,
|
||||
async findRunById() {
|
||||
return { ...run, status: 'running' };
|
||||
},
|
||||
async findAttemptById() {
|
||||
return {
|
||||
id: 'attempt-1',
|
||||
runId: 'run-1',
|
||||
attempt: 1,
|
||||
status: 'running',
|
||||
executorType: 'remote_worker',
|
||||
logArtifactId,
|
||||
callbackSequence: 0,
|
||||
createdAtMs: 1,
|
||||
};
|
||||
},
|
||||
},
|
||||
workerRuntime: {
|
||||
offers: { claimNext() {} },
|
||||
activation: {
|
||||
acknowledgeStarting() {},
|
||||
acknowledgeRunning() {},
|
||||
failStart() {},
|
||||
},
|
||||
artifacts: { upload() {} },
|
||||
completion: { complete() {} },
|
||||
leaseControl: { control() {} },
|
||||
runAttemptLogRead: {
|
||||
async read(identity, range) {
|
||||
assert.equal(identity.logArtifactId, logArtifactId);
|
||||
assert.deepEqual(range, { offset: 1, length: 4 });
|
||||
return {
|
||||
status: 'available',
|
||||
content: Buffer.from('prod'),
|
||||
start: 1,
|
||||
endExclusive: 5,
|
||||
totalBytes: 5,
|
||||
truncation: { truncated: false },
|
||||
};
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const result = await invoke(
|
||||
stack,
|
||||
metadata(
|
||||
'/api/v3/projects/project-1/runs/run-1/attempts/attempt-1/log',
|
||||
'GET',
|
||||
null,
|
||||
{ offset: ['1'], length: ['4'] },
|
||||
),
|
||||
);
|
||||
assert.equal(result.statusCode, 200);
|
||||
assert.equal(Buffer.from(result.body.content, 'base64').toString(), 'prod');
|
||||
});
|
||||
|
||||
test('optionally exposes Prompt execution behind shared admission and policy', async () => {
|
||||
const { events, input } = fixture();
|
||||
let command;
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
createClusterControlAdmissionPipeline,
|
||||
} = require('@qinglong/cluster-control/admission');
|
||||
const {
|
||||
createClusterControlRouteRegistry,
|
||||
} = require('@qinglong/cluster-control/routes');
|
||||
const {
|
||||
CLUSTER_CONTROL_RUN_ATTEMPT_LOG_READ_ROUTE,
|
||||
createClusterControlRunAttemptLogReadRoute,
|
||||
} = require('../dist/run/runAttemptLogReadRoute.js');
|
||||
|
||||
const PRINCIPAL = Object.freeze({
|
||||
subject: Object.freeze({ type: 'user', id: 'usr_viewer' }),
|
||||
authenticationId: 'session:viewer',
|
||||
authenticatedAtMs: 9_000,
|
||||
expiresAtMs: 11_000,
|
||||
assurance: 'single_factor',
|
||||
});
|
||||
|
||||
function run(overrides = {}) {
|
||||
return {
|
||||
id: 'run_123',
|
||||
projectId: 'prj_default',
|
||||
taskId: 'task_1',
|
||||
taskRevision: 'revision_1',
|
||||
triggerType: 'task_start',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
status: 'running',
|
||||
version: 2,
|
||||
eventSequence: 2,
|
||||
priority: 0,
|
||||
createdAtMs: 1,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function attempt(overrides = {}) {
|
||||
return {
|
||||
id: 'attempt_123',
|
||||
runId: 'run_123',
|
||||
attempt: 1,
|
||||
status: 'running',
|
||||
executorType: 'remote_worker',
|
||||
logArtifactId: `wlog-${'a'.repeat(30)}`,
|
||||
callbackSequence: 0,
|
||||
createdAtMs: 1,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function metadata(query = {}) {
|
||||
return Object.freeze({
|
||||
requestId: 'request-log-read',
|
||||
method: 'GET',
|
||||
path: '/api/v3/projects/prj_default/runs/run_123/attempts/attempt_123/log',
|
||||
query: Object.freeze(query),
|
||||
headers: Object.freeze({ authorization: 'Bearer opaque' }),
|
||||
signal: new AbortController().signal,
|
||||
});
|
||||
}
|
||||
|
||||
function pipeline(options = {}) {
|
||||
const events = options.events ?? [];
|
||||
const repository = options.repository ?? {
|
||||
async findRunById() {
|
||||
events.push('run');
|
||||
return run();
|
||||
},
|
||||
async findAttemptById() {
|
||||
events.push('attempt');
|
||||
return attempt();
|
||||
},
|
||||
};
|
||||
const reader = options.reader ?? {
|
||||
async read(identity, range) {
|
||||
events.push(`storage:${range.offset}:${range.length}`);
|
||||
return {
|
||||
status: 'available',
|
||||
content: Buffer.from('cluster-log'),
|
||||
start: range.offset,
|
||||
endExclusive: range.offset + 11,
|
||||
totalBytes: range.offset + 20,
|
||||
nextOffset: range.offset + 11,
|
||||
truncation: { truncated: false },
|
||||
};
|
||||
},
|
||||
};
|
||||
return createClusterControlAdmissionPipeline({
|
||||
routes: createClusterControlRouteRegistry([
|
||||
createClusterControlRunAttemptLogReadRoute(repository, reader),
|
||||
]),
|
||||
authenticator: {
|
||||
authenticate() {
|
||||
events.push('authenticate');
|
||||
return PRINCIPAL;
|
||||
},
|
||||
},
|
||||
policy: {
|
||||
authorize(request) {
|
||||
events.push(`authorize:${request.permission}`);
|
||||
return options.effect
|
||||
? { effect: options.effect, reasons: ['masked'], fence: null }
|
||||
: {
|
||||
effect: 'allow',
|
||||
reasons: ['role_grant'],
|
||||
fence: { projectVersion: 2, bindingVersion: 3 },
|
||||
};
|
||||
},
|
||||
},
|
||||
audit: {
|
||||
record(record) {
|
||||
events.push(`audit:${record.outcome}`);
|
||||
},
|
||||
},
|
||||
now: () => 10_000,
|
||||
});
|
||||
}
|
||||
|
||||
async function invoke(value, query = {}) {
|
||||
const prepared = await value.prepare(metadata(query));
|
||||
return prepared.handle(null);
|
||||
}
|
||||
|
||||
test('publishes the immutable Artifact-scoped route contract', () => {
|
||||
assert.deepEqual(CLUSTER_CONTROL_RUN_ATTEMPT_LOG_READ_ROUTE, {
|
||||
method: 'GET',
|
||||
path: '/api/v3/projects/{projectId}/runs/{runId}/attempts/{attemptId}/log',
|
||||
operationId: 'run.log.read',
|
||||
permission: 'artifact.read',
|
||||
projectParameter: 'projectId',
|
||||
allowedQuery: ['length', 'offset'],
|
||||
});
|
||||
});
|
||||
|
||||
test('authorizes and audits before metadata and one bounded range read', async () => {
|
||||
const events = [];
|
||||
const result = await invoke(pipeline({ events }), {
|
||||
offset: ['4'],
|
||||
length: ['16'],
|
||||
});
|
||||
assert.equal(result.statusCode, 200);
|
||||
assert.equal(result.body.schema, 'qinglong/run-attempt-log-read-result@v1');
|
||||
assert.equal(
|
||||
Buffer.from(result.body.content, 'base64').toString(),
|
||||
'cluster-log',
|
||||
);
|
||||
assert.deepEqual(result.body.range, {
|
||||
start: 4,
|
||||
endExclusive: 15,
|
||||
totalBytes: 24,
|
||||
nextOffset: 15,
|
||||
});
|
||||
assert.deepEqual(events, [
|
||||
'authenticate',
|
||||
'authorize:artifact.read',
|
||||
'audit:allowed',
|
||||
'run',
|
||||
'attempt',
|
||||
'storage:4:16',
|
||||
]);
|
||||
});
|
||||
|
||||
test('uses the Cluster default window and rejects unbounded query values', async () => {
|
||||
const events = [];
|
||||
assert.equal((await invoke(pipeline({ events }))).statusCode, 200);
|
||||
assert.equal(events.at(-1), `storage:0:${64 * 1024}`);
|
||||
for (const query of [
|
||||
{ offset: ['-1'] },
|
||||
{ offset: ['04'] },
|
||||
{ length: ['0'] },
|
||||
{ length: [String(256 * 1024 + 1)] },
|
||||
{ length: ['1', '2'] },
|
||||
]) {
|
||||
await assert.rejects(
|
||||
pipeline().prepare(metadata(query)),
|
||||
(error) =>
|
||||
error.statusCode === 400 && error.code === 'invalid_route_query',
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('masks deny and approval without reading Run or object storage', async () => {
|
||||
for (const effect of ['deny', 'require_approval']) {
|
||||
let touched = false;
|
||||
await assert.rejects(
|
||||
pipeline({
|
||||
effect,
|
||||
repository: {
|
||||
async findRunById() {
|
||||
touched = true;
|
||||
return run();
|
||||
},
|
||||
async findAttemptById() {
|
||||
touched = true;
|
||||
return attempt();
|
||||
},
|
||||
},
|
||||
reader: {
|
||||
async read() {
|
||||
touched = true;
|
||||
return { status: 'missing' };
|
||||
},
|
||||
},
|
||||
}).prepare(metadata()),
|
||||
(error) =>
|
||||
error.statusCode === 404 && error.code === 'artifact_not_found',
|
||||
);
|
||||
assert.equal(touched, false);
|
||||
}
|
||||
});
|
||||
|
||||
test('returns pending during upload and fails closed without an object reader', async () => {
|
||||
const pending = pipeline({
|
||||
reader: {
|
||||
async read() {
|
||||
return { status: 'missing' };
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.equal((await invoke(pending)).statusCode, 202);
|
||||
|
||||
const unavailableRoute = createClusterControlRunAttemptLogReadRoute({
|
||||
async findRunById() {
|
||||
return run();
|
||||
},
|
||||
async findAttemptById() {
|
||||
return attempt();
|
||||
},
|
||||
});
|
||||
const prepared = await createClusterControlAdmissionPipeline({
|
||||
routes: createClusterControlRouteRegistry([unavailableRoute]),
|
||||
authenticator: { authenticate: () => PRINCIPAL },
|
||||
policy: {
|
||||
authorize: () => ({
|
||||
effect: 'allow',
|
||||
reasons: ['role_grant'],
|
||||
fence: { projectVersion: 1, bindingVersion: 1 },
|
||||
}),
|
||||
},
|
||||
audit: { record() {} },
|
||||
now: () => 10_000,
|
||||
}).prepare(metadata());
|
||||
assert.deepEqual(await prepared.handle(null), {
|
||||
statusCode: 503,
|
||||
body: { code: 'artifact_unavailable' },
|
||||
});
|
||||
});
|
||||
@@ -19,80 +19,111 @@ const endpoint = process.env.QL3_TEST_S3_ENDPOINT;
|
||||
const accessKeyId = process.env.QL3_TEST_S3_ACCESS_KEY_ID;
|
||||
const secretAccessKey = process.env.QL3_TEST_S3_SECRET_ACCESS_KEY;
|
||||
|
||||
test('real S3-compatible service preserves immutable Artifact evidence', {
|
||||
skip: endpoint && accessKeyId && secretAccessKey
|
||||
? false
|
||||
: 'requires QL3_TEST_S3_ENDPOINT and credentials',
|
||||
}, async () => {
|
||||
const client = new S3Client({
|
||||
endpoint,
|
||||
region: 'us-east-1',
|
||||
forcePathStyle: true,
|
||||
credentials: { accessKeyId, secretAccessKey },
|
||||
});
|
||||
const bucket = `ql3-artifact-${process.pid}-${Date.now()}`.slice(0, 63);
|
||||
const command = Object.freeze({
|
||||
projectId: 'project-s3-integration',
|
||||
runId: 'run-s3-integration',
|
||||
attemptId: 'attempt-s3-integration',
|
||||
logArtifactId: `wlog-${'c'.repeat(30)}`,
|
||||
byteLength: 17,
|
||||
truncated: true,
|
||||
});
|
||||
const content = Buffer.from('real object bytes');
|
||||
const body = (value) => Object.freeze({
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield value.subarray(0, 4);
|
||||
yield value.subarray(4);
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
await client.send(new CreateBucketCommand({ Bucket: bucket }));
|
||||
const store = new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket,
|
||||
prefix: 'qinglong/integration',
|
||||
encryption: { mode: 's3' },
|
||||
test(
|
||||
'real S3-compatible service preserves immutable Artifact evidence',
|
||||
{
|
||||
skip:
|
||||
endpoint && accessKeyId && secretAccessKey
|
||||
? false
|
||||
: 'requires QL3_TEST_S3_ENDPOINT and credentials',
|
||||
},
|
||||
async () => {
|
||||
const client = new S3Client({
|
||||
endpoint,
|
||||
region: 'us-east-1',
|
||||
forcePathStyle: true,
|
||||
credentials: { accessKeyId, secretAccessKey },
|
||||
});
|
||||
const stored = await store.put(command, body(content));
|
||||
assert.equal(stored.status, 'stored');
|
||||
assert.equal(
|
||||
stored.sha256,
|
||||
createHash('sha256').update(content).digest('hex'),
|
||||
);
|
||||
const replay = await store.put(command, body(content));
|
||||
assert.equal(replay.status, 'already_stored');
|
||||
await assert.rejects(
|
||||
store.put(command, body(Buffer.from('REAL OBJECT BYTES'))),
|
||||
(error) =>
|
||||
error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
|
||||
error.reason === 'integrity_mismatch',
|
||||
);
|
||||
const objects = await client.send(new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
Prefix: 'qinglong/integration/',
|
||||
}));
|
||||
assert.equal(objects.KeyCount, 1);
|
||||
assert.match(objects.Contents[0].Key, /\/objects\//);
|
||||
} finally {
|
||||
const bucket = `ql3-artifact-${process.pid}-${Date.now()}`.slice(0, 63);
|
||||
const command = Object.freeze({
|
||||
projectId: 'project-s3-integration',
|
||||
runId: 'run-s3-integration',
|
||||
attemptId: 'attempt-s3-integration',
|
||||
logArtifactId: `wlog-${'c'.repeat(30)}`,
|
||||
byteLength: 17,
|
||||
truncated: true,
|
||||
});
|
||||
const content = Buffer.from('real object bytes');
|
||||
const body = (value) =>
|
||||
Object.freeze({
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield value.subarray(0, 4);
|
||||
yield value.subarray(4);
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const objects = await client.send(new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
}));
|
||||
if (objects.Contents?.length) {
|
||||
await client.send(new DeleteObjectsCommand({
|
||||
await client.send(new CreateBucketCommand({ Bucket: bucket }));
|
||||
const store = new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket,
|
||||
prefix: 'qinglong/integration',
|
||||
encryption: { mode: 's3' },
|
||||
});
|
||||
const stored = await store.put(command, body(content));
|
||||
assert.equal(stored.status, 'stored');
|
||||
assert.equal(
|
||||
stored.sha256,
|
||||
createHash('sha256').update(content).digest('hex'),
|
||||
);
|
||||
const replay = await store.put(command, body(content));
|
||||
assert.equal(replay.status, 'already_stored');
|
||||
const range = await store.readLogRange(
|
||||
{
|
||||
projectId: command.projectId,
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
logArtifactId: command.logArtifactId,
|
||||
},
|
||||
{
|
||||
offset: 5,
|
||||
length: 6,
|
||||
},
|
||||
);
|
||||
assert.equal(range.status, 'available');
|
||||
assert.equal(Buffer.from(range.content).toString(), 'object');
|
||||
assert.equal(range.start, 5);
|
||||
assert.equal(range.endExclusive, 11);
|
||||
assert.equal(range.totalBytes, content.byteLength);
|
||||
assert.equal(range.nextOffset, 11);
|
||||
assert.deepEqual(range.truncation, { truncated: true });
|
||||
await assert.rejects(
|
||||
store.put(command, body(Buffer.from('REAL OBJECT BYTES'))),
|
||||
(error) =>
|
||||
error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
|
||||
error.reason === 'integrity_mismatch',
|
||||
);
|
||||
const objects = await client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
Delete: {
|
||||
Objects: objects.Contents.map(({ Key }) => ({ Key })),
|
||||
Quiet: true,
|
||||
},
|
||||
}));
|
||||
Prefix: 'qinglong/integration/',
|
||||
}),
|
||||
);
|
||||
assert.equal(objects.KeyCount, 1);
|
||||
assert.match(objects.Contents[0].Key, /\/objects\//);
|
||||
} finally {
|
||||
try {
|
||||
const objects = await client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
}),
|
||||
);
|
||||
if (objects.Contents?.length) {
|
||||
await client.send(
|
||||
new DeleteObjectsCommand({
|
||||
Bucket: bucket,
|
||||
Delete: {
|
||||
Objects: objects.Contents.map(({ Key }) => ({ Key })),
|
||||
Quiet: true,
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
await client.send(new DeleteBucketCommand({ Bucket: bucket }));
|
||||
} catch {
|
||||
// Preserve the integration assertion; the ephemeral container is removed.
|
||||
}
|
||||
await client.send(new DeleteBucketCommand({ Bucket: bucket }));
|
||||
} catch {
|
||||
// Preserve the integration assertion; the ephemeral container is removed.
|
||||
client.destroy();
|
||||
}
|
||||
client.destroy();
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
@@ -6,6 +6,7 @@ const { test } = require('node:test');
|
||||
const {
|
||||
CopyObjectCommand,
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
PutObjectCommand,
|
||||
} = require('@aws-sdk/client-s3');
|
||||
@@ -57,19 +58,58 @@ class MemoryS3Client {
|
||||
const object = this.objects.get(input.Key);
|
||||
if (!object) throw notFound();
|
||||
const metadata = { ...object.metadata };
|
||||
if (this.options.corruptFinalMetadata && input.Key.includes('/objects/')) {
|
||||
if (
|
||||
this.options.corruptFinalMetadata &&
|
||||
input.Key.includes('/objects/')
|
||||
) {
|
||||
metadata['ql3-content-sha256'] = '0'.repeat(64);
|
||||
}
|
||||
return {
|
||||
ContentLength: object.content.byteLength,
|
||||
ContentType: object.contentType,
|
||||
ChecksumSHA256: this.options.corruptFinalChecksum &&
|
||||
input.Key.includes('/objects/')
|
||||
? Buffer.alloc(32, 9).toString('base64')
|
||||
: checksum(object.content),
|
||||
ETag: `"${checksum(object.content).slice(0, 32)}"`,
|
||||
ChecksumSHA256:
|
||||
this.options.corruptFinalChecksum && input.Key.includes('/objects/')
|
||||
? Buffer.alloc(32, 9).toString('base64')
|
||||
: checksum(object.content),
|
||||
Metadata: metadata,
|
||||
};
|
||||
}
|
||||
if (command instanceof GetObjectCommand) {
|
||||
const object = this.objects.get(input.Key);
|
||||
if (!object || this.options.rangeNotFound) throw notFound();
|
||||
const eTag = `"${checksum(object.content).slice(0, 32)}"`;
|
||||
assert.equal(input.IfMatch, eTag);
|
||||
const match = /^bytes=(\d+)-(\d+)$/.exec(input.Range);
|
||||
assert.ok(match);
|
||||
const start = Number(match[1]);
|
||||
const end = Number(match[2]);
|
||||
let content = object.content.subarray(start, end + 1);
|
||||
if (this.options.shortRangeBody) content = content.subarray(0, -1);
|
||||
if (this.options.oversizedRangeBody) {
|
||||
content = Buffer.concat([content, Buffer.from('x')]);
|
||||
}
|
||||
const metadata = { ...object.metadata };
|
||||
if (this.options.corruptRangeMetadata) {
|
||||
metadata['ql3-run-sha256'] = '0'.repeat(64);
|
||||
}
|
||||
return {
|
||||
ContentLength: end - start + 1,
|
||||
ContentRange: this.options.corruptContentRange
|
||||
? `bytes ${start}-${end}/${object.content.byteLength + 1}`
|
||||
: `bytes ${start}-${end}/${object.content.byteLength}`,
|
||||
ContentType: object.contentType,
|
||||
ETag: this.options.corruptRangeETag ? '"other"' : eTag,
|
||||
Metadata: metadata,
|
||||
Body: {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
const split = Math.min(2, content.byteLength);
|
||||
if (split > 0) yield content.subarray(0, split);
|
||||
if (split < content.byteLength) yield content.subarray(split);
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (command instanceof PutObjectCommand) {
|
||||
assert.equal(input.IfNoneMatch, '*');
|
||||
assert.equal(input.ChecksumAlgorithm, 'SHA256');
|
||||
@@ -183,14 +223,25 @@ test('streams to a checksummed temporary object then conditionally promotes it',
|
||||
],
|
||||
);
|
||||
const key = permanentKey(client);
|
||||
assert.match(key, /^tenant-a\/worker-artifacts\/objects\/[a-f0-9]{2}\/[a-f0-9]{64}$/);
|
||||
assert.match(
|
||||
key,
|
||||
/^tenant-a\/worker-artifacts\/objects\/[a-f0-9]{2}\/[a-f0-9]{64}$/,
|
||||
);
|
||||
assert.equal(key.includes(COMMAND.runId), false);
|
||||
assert.equal(client.objects.size, 1);
|
||||
|
||||
const copy = client.commands.find((command) => command instanceof CopyObjectCommand);
|
||||
const copy = client.commands.find(
|
||||
(command) => command instanceof CopyObjectCommand,
|
||||
);
|
||||
assert.equal(copy.input.Metadata['ql3-content-sha256'], CONTENT_SHA256);
|
||||
assert.equal(JSON.stringify(copy.input.Metadata).includes(COMMAND.projectId), false);
|
||||
assert.equal(JSON.stringify(copy.input.Metadata).includes(COMMAND.runId), false);
|
||||
assert.equal(
|
||||
JSON.stringify(copy.input.Metadata).includes(COMMAND.projectId),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
JSON.stringify(copy.input.Metadata).includes(COMMAND.runId),
|
||||
false,
|
||||
);
|
||||
|
||||
const inspected = await adapter.inspect(LOOKUP);
|
||||
assert.deepEqual(inspected, { ...receipt, status: 'already_stored' });
|
||||
@@ -219,6 +270,78 @@ test('exact replay consumes and hashes the whole body without another write', as
|
||||
);
|
||||
});
|
||||
|
||||
test('reads only one ETag-fenced immutable byte range and stable end snapshot', async () => {
|
||||
const client = new MemoryS3Client();
|
||||
const adapter = store(client);
|
||||
await adapter.put(COMMAND, chunks());
|
||||
client.commands.length = 0;
|
||||
|
||||
const result = await adapter.readLogRange(LOOKUP, { offset: 2, length: 4 });
|
||||
assert.equal(result.status, 'available');
|
||||
assert.equal(Buffer.from(result.content).toString(), 'llo ');
|
||||
assert.deepEqual(
|
||||
{
|
||||
start: result.start,
|
||||
endExclusive: result.endExclusive,
|
||||
totalBytes: result.totalBytes,
|
||||
nextOffset: result.nextOffset,
|
||||
truncation: result.truncation,
|
||||
},
|
||||
{
|
||||
start: 2,
|
||||
endExclusive: 6,
|
||||
totalBytes: 11,
|
||||
nextOffset: 6,
|
||||
truncation: { truncated: false },
|
||||
},
|
||||
);
|
||||
assert.deepEqual(
|
||||
client.commands.map((command) => command.constructor.name),
|
||||
['HeadObjectCommand', 'GetObjectCommand'],
|
||||
);
|
||||
assert.equal(client.commands[1].input.Range, 'bytes=2-5');
|
||||
|
||||
client.commands.length = 0;
|
||||
const ended = await adapter.readLogRange(LOOKUP, {
|
||||
offset: 999,
|
||||
length: 4,
|
||||
});
|
||||
assert.equal(ended.status, 'available');
|
||||
assert.equal(ended.content.byteLength, 0);
|
||||
assert.equal(ended.start, 11);
|
||||
assert.equal(ended.totalBytes, 11);
|
||||
assert.deepEqual(
|
||||
client.commands.map((command) => command.constructor.name),
|
||||
['HeadObjectCommand'],
|
||||
);
|
||||
});
|
||||
|
||||
test('maps absent objects and fails closed on range evidence drift', async () => {
|
||||
const absent = new MemoryS3Client();
|
||||
assert.deepEqual(
|
||||
await store(absent).readLogRange(LOOKUP, { offset: 0, length: 1 }),
|
||||
{ status: 'missing' },
|
||||
);
|
||||
for (const option of [
|
||||
'corruptContentRange',
|
||||
'corruptRangeETag',
|
||||
'corruptRangeMetadata',
|
||||
'shortRangeBody',
|
||||
'oversizedRangeBody',
|
||||
]) {
|
||||
const client = new MemoryS3Client();
|
||||
const adapter = store(client);
|
||||
await adapter.put(COMMAND, chunks());
|
||||
client.options[option] = true;
|
||||
await assert.rejects(
|
||||
adapter.readLogRange(LOOKUP, { offset: 0, length: 4 }),
|
||||
(error) =>
|
||||
error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
|
||||
error.reason === 'integrity_mismatch',
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('resolves a concurrent conditional-copy winner by immutable inspect', async () => {
|
||||
const client = new MemoryS3Client({ raceOnCopy: true });
|
||||
const receipt = await store(client).put(COMMAND, chunks());
|
||||
@@ -270,38 +393,40 @@ test('temporary cleanup failure is diagnostic and never reverses promotion', asy
|
||||
},
|
||||
}).put(COMMAND, chunks());
|
||||
assert.equal(receipt.status, 'stored');
|
||||
assert.deepEqual(diagnostics, [[
|
||||
'delete unavailable',
|
||||
'temporary_object_cleanup',
|
||||
]]);
|
||||
assert.deepEqual(diagnostics, [
|
||||
['delete unavailable', 'temporary_object_cleanup'],
|
||||
]);
|
||||
assert.equal(client.objects.size, 2);
|
||||
});
|
||||
|
||||
test('requires exact bucket, prefix, encryption and temporary ID configuration', async () => {
|
||||
const client = new MemoryS3Client();
|
||||
assert.throws(
|
||||
() => new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket: 'Invalid_Bucket',
|
||||
encryption: { mode: 's3' },
|
||||
}),
|
||||
() =>
|
||||
new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket: 'Invalid_Bucket',
|
||||
encryption: { mode: 's3' },
|
||||
}),
|
||||
/bucket is invalid/,
|
||||
);
|
||||
assert.throws(
|
||||
() => new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket: 'valid-bucket',
|
||||
prefix: '../escape',
|
||||
encryption: { mode: 's3' },
|
||||
}),
|
||||
() =>
|
||||
new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket: 'valid-bucket',
|
||||
prefix: '../escape',
|
||||
encryption: { mode: 's3' },
|
||||
}),
|
||||
/prefix is invalid/,
|
||||
);
|
||||
assert.throws(
|
||||
() => new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket: 'valid-bucket',
|
||||
encryption: { mode: 'kms' },
|
||||
}),
|
||||
() =>
|
||||
new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket: 'valid-bucket',
|
||||
encryption: { mode: 'kms' },
|
||||
}),
|
||||
/encryption is invalid/,
|
||||
);
|
||||
await assert.rejects(
|
||||
@@ -355,8 +480,7 @@ test('propagates KMS and expected-owner fences to both sides of promotion', asyn
|
||||
|
||||
test('never deletes a colliding temporary object it cannot prove it owns', async () => {
|
||||
const client = new MemoryS3Client();
|
||||
const temporaryKey =
|
||||
`tenant-a/worker-artifacts/temporary/${TEMPORARY_ID}`;
|
||||
const temporaryKey = `tenant-a/worker-artifacts/temporary/${TEMPORARY_ID}`;
|
||||
client.objects.set(temporaryKey, {
|
||||
content: Buffer.from('other operation'),
|
||||
contentType: 'application/octet-stream',
|
||||
|
||||
Reference in New Issue
Block a user