feat(ql3): add profile-aware run log reads

This commit is contained in:
whyour
2026-08-12 01:43:14 +08:00
parent c699c32461
commit 308aa75d89
33 changed files with 2880 additions and 306 deletions
@@ -573,6 +573,9 @@ test('injects reviewed Worker operations without exposing the runtime Pool', asy
async inspect() {
throw new Error('not invoked during assembly');
},
async readLogRange() {
throw new Error('not invoked during assembly');
},
};
const result = await bootstrapClusterControlRuntime(
bootstrapOptions(events, {
@@ -596,6 +599,10 @@ test('injects reviewed Worker operations without exposing the runtime Pool', asy
typeof input.workerRuntime.leaseControl.control,
'function',
);
assert.equal(
typeof input.workerRuntime.runAttemptLogRead.read,
'function',
);
return activationStack(events);
},
}),
@@ -143,6 +143,9 @@ function fixture(overrides = {}) {
},
];
},
async findAttemptById() {
return null;
},
},
trustedToolStorage: {
stepRuns: {
@@ -190,7 +193,8 @@ function fixture(overrides = {}) {
taskDefinitions: {
async findCurrentTaskDefinition(projectId, taskId) {
events.push(`task-get:${projectId}:${taskId}`);
return projectId === currentTask.projectId && taskId === currentTask.taskId
return projectId === currentTask.projectId &&
taskId === currentTask.taskId
? currentTask
: null;
},
@@ -328,6 +332,7 @@ test('production composition exposes the reviewed Run and Workflow routes', asyn
'run.list',
'run.events.list',
'run.steps.list',
'run.log.read',
'run.cancel',
'workflow.read',
'workflow.run.read',
@@ -401,6 +406,15 @@ test('production composition exposes the reviewed Run and Workflow routes', asyn
body: { steps: [], hasMore: false, next: null },
});
const log = await invoke(
stack,
metadata('/api/v3/projects/project-1/runs/run-1/attempts/attempt-1/log'),
);
assert.deepEqual(log, {
statusCode: 503,
body: { code: 'artifact_unavailable' },
});
const cancellation = await invoke(
stack,
metadata('/api/v3/projects/project-1/runs/run-1/cancellation', 'POST', {
@@ -414,6 +428,7 @@ test('production composition exposes the reviewed Run and Workflow routes', asyn
assert.equal(events.includes('audit:run.get:allowed'), true);
assert.equal(events.includes('audit:run.events.list:allowed'), true);
assert.equal(events.includes('audit:run.steps.list:allowed'), true);
assert.equal(events.includes('audit:run.log.read:allowed'), true);
assert.equal(events.includes('audit:run.cancel:allowed'), true);
const workflows = await invoke(
@@ -539,6 +554,69 @@ test('production composition fails closed for an unreviewed route', async () =>
);
});
test('wires the production Worker object reader into the Project-scoped log route', async () => {
const { input } = fixture();
const run = await input.runs.findRunById('run-1');
const logArtifactId = `wlog-${'a'.repeat(30)}`;
const stack = createProductionClusterControlApplicationStack({
...input,
runs: {
...input.runs,
async findRunById() {
return { ...run, status: 'running' };
},
async findAttemptById() {
return {
id: 'attempt-1',
runId: 'run-1',
attempt: 1,
status: 'running',
executorType: 'remote_worker',
logArtifactId,
callbackSequence: 0,
createdAtMs: 1,
};
},
},
workerRuntime: {
offers: { claimNext() {} },
activation: {
acknowledgeStarting() {},
acknowledgeRunning() {},
failStart() {},
},
artifacts: { upload() {} },
completion: { complete() {} },
leaseControl: { control() {} },
runAttemptLogRead: {
async read(identity, range) {
assert.equal(identity.logArtifactId, logArtifactId);
assert.deepEqual(range, { offset: 1, length: 4 });
return {
status: 'available',
content: Buffer.from('prod'),
start: 1,
endExclusive: 5,
totalBytes: 5,
truncation: { truncated: false },
};
},
},
},
});
const result = await invoke(
stack,
metadata(
'/api/v3/projects/project-1/runs/run-1/attempts/attempt-1/log',
'GET',
null,
{ offset: ['1'], length: ['4'] },
),
);
assert.equal(result.statusCode, 200);
assert.equal(Buffer.from(result.body.content, 'base64').toString(), 'prod');
});
test('optionally exposes Prompt execution behind shared admission and policy', async () => {
const { events, input } = fixture();
let command;
@@ -0,0 +1,251 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
createClusterControlAdmissionPipeline,
} = require('@qinglong/cluster-control/admission');
const {
createClusterControlRouteRegistry,
} = require('@qinglong/cluster-control/routes');
const {
CLUSTER_CONTROL_RUN_ATTEMPT_LOG_READ_ROUTE,
createClusterControlRunAttemptLogReadRoute,
} = require('../dist/run/runAttemptLogReadRoute.js');
const PRINCIPAL = Object.freeze({
subject: Object.freeze({ type: 'user', id: 'usr_viewer' }),
authenticationId: 'session:viewer',
authenticatedAtMs: 9_000,
expiresAtMs: 11_000,
assurance: 'single_factor',
});
function run(overrides = {}) {
return {
id: 'run_123',
projectId: 'prj_default',
taskId: 'task_1',
taskRevision: 'revision_1',
triggerType: 'task_start',
executionOrigin: 'manual',
executionOwner: 'runtime',
status: 'running',
version: 2,
eventSequence: 2,
priority: 0,
createdAtMs: 1,
...overrides,
};
}
function attempt(overrides = {}) {
return {
id: 'attempt_123',
runId: 'run_123',
attempt: 1,
status: 'running',
executorType: 'remote_worker',
logArtifactId: `wlog-${'a'.repeat(30)}`,
callbackSequence: 0,
createdAtMs: 1,
...overrides,
};
}
function metadata(query = {}) {
return Object.freeze({
requestId: 'request-log-read',
method: 'GET',
path: '/api/v3/projects/prj_default/runs/run_123/attempts/attempt_123/log',
query: Object.freeze(query),
headers: Object.freeze({ authorization: 'Bearer opaque' }),
signal: new AbortController().signal,
});
}
function pipeline(options = {}) {
const events = options.events ?? [];
const repository = options.repository ?? {
async findRunById() {
events.push('run');
return run();
},
async findAttemptById() {
events.push('attempt');
return attempt();
},
};
const reader = options.reader ?? {
async read(identity, range) {
events.push(`storage:${range.offset}:${range.length}`);
return {
status: 'available',
content: Buffer.from('cluster-log'),
start: range.offset,
endExclusive: range.offset + 11,
totalBytes: range.offset + 20,
nextOffset: range.offset + 11,
truncation: { truncated: false },
};
},
};
return createClusterControlAdmissionPipeline({
routes: createClusterControlRouteRegistry([
createClusterControlRunAttemptLogReadRoute(repository, reader),
]),
authenticator: {
authenticate() {
events.push('authenticate');
return PRINCIPAL;
},
},
policy: {
authorize(request) {
events.push(`authorize:${request.permission}`);
return options.effect
? { effect: options.effect, reasons: ['masked'], fence: null }
: {
effect: 'allow',
reasons: ['role_grant'],
fence: { projectVersion: 2, bindingVersion: 3 },
};
},
},
audit: {
record(record) {
events.push(`audit:${record.outcome}`);
},
},
now: () => 10_000,
});
}
async function invoke(value, query = {}) {
const prepared = await value.prepare(metadata(query));
return prepared.handle(null);
}
test('publishes the immutable Artifact-scoped route contract', () => {
assert.deepEqual(CLUSTER_CONTROL_RUN_ATTEMPT_LOG_READ_ROUTE, {
method: 'GET',
path: '/api/v3/projects/{projectId}/runs/{runId}/attempts/{attemptId}/log',
operationId: 'run.log.read',
permission: 'artifact.read',
projectParameter: 'projectId',
allowedQuery: ['length', 'offset'],
});
});
test('authorizes and audits before metadata and one bounded range read', async () => {
const events = [];
const result = await invoke(pipeline({ events }), {
offset: ['4'],
length: ['16'],
});
assert.equal(result.statusCode, 200);
assert.equal(result.body.schema, 'qinglong/run-attempt-log-read-result@v1');
assert.equal(
Buffer.from(result.body.content, 'base64').toString(),
'cluster-log',
);
assert.deepEqual(result.body.range, {
start: 4,
endExclusive: 15,
totalBytes: 24,
nextOffset: 15,
});
assert.deepEqual(events, [
'authenticate',
'authorize:artifact.read',
'audit:allowed',
'run',
'attempt',
'storage:4:16',
]);
});
test('uses the Cluster default window and rejects unbounded query values', async () => {
const events = [];
assert.equal((await invoke(pipeline({ events }))).statusCode, 200);
assert.equal(events.at(-1), `storage:0:${64 * 1024}`);
for (const query of [
{ offset: ['-1'] },
{ offset: ['04'] },
{ length: ['0'] },
{ length: [String(256 * 1024 + 1)] },
{ length: ['1', '2'] },
]) {
await assert.rejects(
pipeline().prepare(metadata(query)),
(error) =>
error.statusCode === 400 && error.code === 'invalid_route_query',
);
}
});
test('masks deny and approval without reading Run or object storage', async () => {
for (const effect of ['deny', 'require_approval']) {
let touched = false;
await assert.rejects(
pipeline({
effect,
repository: {
async findRunById() {
touched = true;
return run();
},
async findAttemptById() {
touched = true;
return attempt();
},
},
reader: {
async read() {
touched = true;
return { status: 'missing' };
},
},
}).prepare(metadata()),
(error) =>
error.statusCode === 404 && error.code === 'artifact_not_found',
);
assert.equal(touched, false);
}
});
test('returns pending during upload and fails closed without an object reader', async () => {
const pending = pipeline({
reader: {
async read() {
return { status: 'missing' };
},
},
});
assert.equal((await invoke(pending)).statusCode, 202);
const unavailableRoute = createClusterControlRunAttemptLogReadRoute({
async findRunById() {
return run();
},
async findAttemptById() {
return attempt();
},
});
const prepared = await createClusterControlAdmissionPipeline({
routes: createClusterControlRouteRegistry([unavailableRoute]),
authenticator: { authenticate: () => PRINCIPAL },
policy: {
authorize: () => ({
effect: 'allow',
reasons: ['role_grant'],
fence: { projectVersion: 1, bindingVersion: 1 },
}),
},
audit: { record() {} },
now: () => 10_000,
}).prepare(metadata());
assert.deepEqual(await prepared.handle(null), {
statusCode: 503,
body: { code: 'artifact_unavailable' },
});
});
@@ -19,80 +19,111 @@ const endpoint = process.env.QL3_TEST_S3_ENDPOINT;
const accessKeyId = process.env.QL3_TEST_S3_ACCESS_KEY_ID;
const secretAccessKey = process.env.QL3_TEST_S3_SECRET_ACCESS_KEY;
test('real S3-compatible service preserves immutable Artifact evidence', {
skip: endpoint && accessKeyId && secretAccessKey
? false
: 'requires QL3_TEST_S3_ENDPOINT and credentials',
}, async () => {
const client = new S3Client({
endpoint,
region: 'us-east-1',
forcePathStyle: true,
credentials: { accessKeyId, secretAccessKey },
});
const bucket = `ql3-artifact-${process.pid}-${Date.now()}`.slice(0, 63);
const command = Object.freeze({
projectId: 'project-s3-integration',
runId: 'run-s3-integration',
attemptId: 'attempt-s3-integration',
logArtifactId: `wlog-${'c'.repeat(30)}`,
byteLength: 17,
truncated: true,
});
const content = Buffer.from('real object bytes');
const body = (value) => Object.freeze({
async *[Symbol.asyncIterator]() {
yield value.subarray(0, 4);
yield value.subarray(4);
},
});
try {
await client.send(new CreateBucketCommand({ Bucket: bucket }));
const store = new S3ClusterRemoteWorkerArtifactStore({
client,
bucket,
prefix: 'qinglong/integration',
encryption: { mode: 's3' },
test(
'real S3-compatible service preserves immutable Artifact evidence',
{
skip:
endpoint && accessKeyId && secretAccessKey
? false
: 'requires QL3_TEST_S3_ENDPOINT and credentials',
},
async () => {
const client = new S3Client({
endpoint,
region: 'us-east-1',
forcePathStyle: true,
credentials: { accessKeyId, secretAccessKey },
});
const stored = await store.put(command, body(content));
assert.equal(stored.status, 'stored');
assert.equal(
stored.sha256,
createHash('sha256').update(content).digest('hex'),
);
const replay = await store.put(command, body(content));
assert.equal(replay.status, 'already_stored');
await assert.rejects(
store.put(command, body(Buffer.from('REAL OBJECT BYTES'))),
(error) =>
error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
error.reason === 'integrity_mismatch',
);
const objects = await client.send(new ListObjectsV2Command({
Bucket: bucket,
Prefix: 'qinglong/integration/',
}));
assert.equal(objects.KeyCount, 1);
assert.match(objects.Contents[0].Key, /\/objects\//);
} finally {
const bucket = `ql3-artifact-${process.pid}-${Date.now()}`.slice(0, 63);
const command = Object.freeze({
projectId: 'project-s3-integration',
runId: 'run-s3-integration',
attemptId: 'attempt-s3-integration',
logArtifactId: `wlog-${'c'.repeat(30)}`,
byteLength: 17,
truncated: true,
});
const content = Buffer.from('real object bytes');
const body = (value) =>
Object.freeze({
async *[Symbol.asyncIterator]() {
yield value.subarray(0, 4);
yield value.subarray(4);
},
});
try {
const objects = await client.send(new ListObjectsV2Command({
Bucket: bucket,
}));
if (objects.Contents?.length) {
await client.send(new DeleteObjectsCommand({
await client.send(new CreateBucketCommand({ Bucket: bucket }));
const store = new S3ClusterRemoteWorkerArtifactStore({
client,
bucket,
prefix: 'qinglong/integration',
encryption: { mode: 's3' },
});
const stored = await store.put(command, body(content));
assert.equal(stored.status, 'stored');
assert.equal(
stored.sha256,
createHash('sha256').update(content).digest('hex'),
);
const replay = await store.put(command, body(content));
assert.equal(replay.status, 'already_stored');
const range = await store.readLogRange(
{
projectId: command.projectId,
runId: command.runId,
attemptId: command.attemptId,
logArtifactId: command.logArtifactId,
},
{
offset: 5,
length: 6,
},
);
assert.equal(range.status, 'available');
assert.equal(Buffer.from(range.content).toString(), 'object');
assert.equal(range.start, 5);
assert.equal(range.endExclusive, 11);
assert.equal(range.totalBytes, content.byteLength);
assert.equal(range.nextOffset, 11);
assert.deepEqual(range.truncation, { truncated: true });
await assert.rejects(
store.put(command, body(Buffer.from('REAL OBJECT BYTES'))),
(error) =>
error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
error.reason === 'integrity_mismatch',
);
const objects = await client.send(
new ListObjectsV2Command({
Bucket: bucket,
Delete: {
Objects: objects.Contents.map(({ Key }) => ({ Key })),
Quiet: true,
},
}));
Prefix: 'qinglong/integration/',
}),
);
assert.equal(objects.KeyCount, 1);
assert.match(objects.Contents[0].Key, /\/objects\//);
} finally {
try {
const objects = await client.send(
new ListObjectsV2Command({
Bucket: bucket,
}),
);
if (objects.Contents?.length) {
await client.send(
new DeleteObjectsCommand({
Bucket: bucket,
Delete: {
Objects: objects.Contents.map(({ Key }) => ({ Key })),
Quiet: true,
},
}),
);
}
await client.send(new DeleteBucketCommand({ Bucket: bucket }));
} catch {
// Preserve the integration assertion; the ephemeral container is removed.
}
await client.send(new DeleteBucketCommand({ Bucket: bucket }));
} catch {
// Preserve the integration assertion; the ephemeral container is removed.
client.destroy();
}
client.destroy();
}
});
},
);
@@ -6,6 +6,7 @@ const { test } = require('node:test');
const {
CopyObjectCommand,
DeleteObjectCommand,
GetObjectCommand,
HeadObjectCommand,
PutObjectCommand,
} = require('@aws-sdk/client-s3');
@@ -57,19 +58,58 @@ class MemoryS3Client {
const object = this.objects.get(input.Key);
if (!object) throw notFound();
const metadata = { ...object.metadata };
if (this.options.corruptFinalMetadata && input.Key.includes('/objects/')) {
if (
this.options.corruptFinalMetadata &&
input.Key.includes('/objects/')
) {
metadata['ql3-content-sha256'] = '0'.repeat(64);
}
return {
ContentLength: object.content.byteLength,
ContentType: object.contentType,
ChecksumSHA256: this.options.corruptFinalChecksum &&
input.Key.includes('/objects/')
? Buffer.alloc(32, 9).toString('base64')
: checksum(object.content),
ETag: `"${checksum(object.content).slice(0, 32)}"`,
ChecksumSHA256:
this.options.corruptFinalChecksum && input.Key.includes('/objects/')
? Buffer.alloc(32, 9).toString('base64')
: checksum(object.content),
Metadata: metadata,
};
}
if (command instanceof GetObjectCommand) {
const object = this.objects.get(input.Key);
if (!object || this.options.rangeNotFound) throw notFound();
const eTag = `"${checksum(object.content).slice(0, 32)}"`;
assert.equal(input.IfMatch, eTag);
const match = /^bytes=(\d+)-(\d+)$/.exec(input.Range);
assert.ok(match);
const start = Number(match[1]);
const end = Number(match[2]);
let content = object.content.subarray(start, end + 1);
if (this.options.shortRangeBody) content = content.subarray(0, -1);
if (this.options.oversizedRangeBody) {
content = Buffer.concat([content, Buffer.from('x')]);
}
const metadata = { ...object.metadata };
if (this.options.corruptRangeMetadata) {
metadata['ql3-run-sha256'] = '0'.repeat(64);
}
return {
ContentLength: end - start + 1,
ContentRange: this.options.corruptContentRange
? `bytes ${start}-${end}/${object.content.byteLength + 1}`
: `bytes ${start}-${end}/${object.content.byteLength}`,
ContentType: object.contentType,
ETag: this.options.corruptRangeETag ? '"other"' : eTag,
Metadata: metadata,
Body: {
async *[Symbol.asyncIterator]() {
const split = Math.min(2, content.byteLength);
if (split > 0) yield content.subarray(0, split);
if (split < content.byteLength) yield content.subarray(split);
},
},
};
}
if (command instanceof PutObjectCommand) {
assert.equal(input.IfNoneMatch, '*');
assert.equal(input.ChecksumAlgorithm, 'SHA256');
@@ -183,14 +223,25 @@ test('streams to a checksummed temporary object then conditionally promotes it',
],
);
const key = permanentKey(client);
assert.match(key, /^tenant-a\/worker-artifacts\/objects\/[a-f0-9]{2}\/[a-f0-9]{64}$/);
assert.match(
key,
/^tenant-a\/worker-artifacts\/objects\/[a-f0-9]{2}\/[a-f0-9]{64}$/,
);
assert.equal(key.includes(COMMAND.runId), false);
assert.equal(client.objects.size, 1);
const copy = client.commands.find((command) => command instanceof CopyObjectCommand);
const copy = client.commands.find(
(command) => command instanceof CopyObjectCommand,
);
assert.equal(copy.input.Metadata['ql3-content-sha256'], CONTENT_SHA256);
assert.equal(JSON.stringify(copy.input.Metadata).includes(COMMAND.projectId), false);
assert.equal(JSON.stringify(copy.input.Metadata).includes(COMMAND.runId), false);
assert.equal(
JSON.stringify(copy.input.Metadata).includes(COMMAND.projectId),
false,
);
assert.equal(
JSON.stringify(copy.input.Metadata).includes(COMMAND.runId),
false,
);
const inspected = await adapter.inspect(LOOKUP);
assert.deepEqual(inspected, { ...receipt, status: 'already_stored' });
@@ -219,6 +270,78 @@ test('exact replay consumes and hashes the whole body without another write', as
);
});
test('reads only one ETag-fenced immutable byte range and stable end snapshot', async () => {
const client = new MemoryS3Client();
const adapter = store(client);
await adapter.put(COMMAND, chunks());
client.commands.length = 0;
const result = await adapter.readLogRange(LOOKUP, { offset: 2, length: 4 });
assert.equal(result.status, 'available');
assert.equal(Buffer.from(result.content).toString(), 'llo ');
assert.deepEqual(
{
start: result.start,
endExclusive: result.endExclusive,
totalBytes: result.totalBytes,
nextOffset: result.nextOffset,
truncation: result.truncation,
},
{
start: 2,
endExclusive: 6,
totalBytes: 11,
nextOffset: 6,
truncation: { truncated: false },
},
);
assert.deepEqual(
client.commands.map((command) => command.constructor.name),
['HeadObjectCommand', 'GetObjectCommand'],
);
assert.equal(client.commands[1].input.Range, 'bytes=2-5');
client.commands.length = 0;
const ended = await adapter.readLogRange(LOOKUP, {
offset: 999,
length: 4,
});
assert.equal(ended.status, 'available');
assert.equal(ended.content.byteLength, 0);
assert.equal(ended.start, 11);
assert.equal(ended.totalBytes, 11);
assert.deepEqual(
client.commands.map((command) => command.constructor.name),
['HeadObjectCommand'],
);
});
test('maps absent objects and fails closed on range evidence drift', async () => {
const absent = new MemoryS3Client();
assert.deepEqual(
await store(absent).readLogRange(LOOKUP, { offset: 0, length: 1 }),
{ status: 'missing' },
);
for (const option of [
'corruptContentRange',
'corruptRangeETag',
'corruptRangeMetadata',
'shortRangeBody',
'oversizedRangeBody',
]) {
const client = new MemoryS3Client();
const adapter = store(client);
await adapter.put(COMMAND, chunks());
client.options[option] = true;
await assert.rejects(
adapter.readLogRange(LOOKUP, { offset: 0, length: 4 }),
(error) =>
error instanceof S3ClusterRemoteWorkerArtifactStoreError &&
error.reason === 'integrity_mismatch',
);
}
});
test('resolves a concurrent conditional-copy winner by immutable inspect', async () => {
const client = new MemoryS3Client({ raceOnCopy: true });
const receipt = await store(client).put(COMMAND, chunks());
@@ -270,38 +393,40 @@ test('temporary cleanup failure is diagnostic and never reverses promotion', asy
},
}).put(COMMAND, chunks());
assert.equal(receipt.status, 'stored');
assert.deepEqual(diagnostics, [[
'delete unavailable',
'temporary_object_cleanup',
]]);
assert.deepEqual(diagnostics, [
['delete unavailable', 'temporary_object_cleanup'],
]);
assert.equal(client.objects.size, 2);
});
test('requires exact bucket, prefix, encryption and temporary ID configuration', async () => {
const client = new MemoryS3Client();
assert.throws(
() => new S3ClusterRemoteWorkerArtifactStore({
client,
bucket: 'Invalid_Bucket',
encryption: { mode: 's3' },
}),
() =>
new S3ClusterRemoteWorkerArtifactStore({
client,
bucket: 'Invalid_Bucket',
encryption: { mode: 's3' },
}),
/bucket is invalid/,
);
assert.throws(
() => new S3ClusterRemoteWorkerArtifactStore({
client,
bucket: 'valid-bucket',
prefix: '../escape',
encryption: { mode: 's3' },
}),
() =>
new S3ClusterRemoteWorkerArtifactStore({
client,
bucket: 'valid-bucket',
prefix: '../escape',
encryption: { mode: 's3' },
}),
/prefix is invalid/,
);
assert.throws(
() => new S3ClusterRemoteWorkerArtifactStore({
client,
bucket: 'valid-bucket',
encryption: { mode: 'kms' },
}),
() =>
new S3ClusterRemoteWorkerArtifactStore({
client,
bucket: 'valid-bucket',
encryption: { mode: 'kms' },
}),
/encryption is invalid/,
);
await assert.rejects(
@@ -355,8 +480,7 @@ test('propagates KMS and expected-owner fences to both sides of promotion', asyn
test('never deletes a colliding temporary object it cannot prove it owns', async () => {
const client = new MemoryS3Client();
const temporaryKey =
`tenant-a/worker-artifacts/temporary/${TEMPORARY_ID}`;
const temporaryKey = `tenant-a/worker-artifacts/temporary/${TEMPORARY_ID}`;
client.objects.set(temporaryKey, {
content: Buffer.from('other operation'),
contentType: 'application/octet-stream',