open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-22 10:32:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): add profile-aware run log reads

Browse Source
This commit is contained in:
whyour
2026-08-12 01:43:14 +08:00
parent c699c32461
commit 308aa75d89
33 changed files with 2880 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-local-api/test/admission.test.cjs
+53
View File
@@ -101,6 +101,14 @@ function fixture(overrides = {}) {
return { statusCode: 202, body: { status: 'accepted' } };
},
},
runAttemptLogReadRoute: {
async handle(value) {
events.push(
`log:${value.projectId}:${value.runId}:${value.attemptId}:${value.offset}:${value.length}`,
);
return { statusCode: 200, body: { status: 'available' } };
},
},
taskListRoute: {
async handle(value) {
events.push(`tasks:${value.projectId}:${value.input.limit ?? 32}`);
@@ -148,6 +156,51 @@ test('authenticates, authorizes, durably audits and re-confirms before reading',
]);
});
test('uses artifact.read and masks denied or approval-fenced log existence', async () => {
const operation = Object.freeze({
operationId: 'run.log.read',
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_123',
offset: 4,
length: 16,
});
const allowed = fixture();
assert.deepEqual(await execute(allowed.admission, request({ operation })), {
statusCode: 200,
body: { status: 'available' },
});
assert.deepEqual(allowed.events, [
'authenticate',
'authorize:artifact.read:prj_default',
'audit:allowed:run.log.read',
'confirm',
'log:prj_default:run_123:attempt_123:4:16',
]);
for (const effect of ['deny', 'require_approval']) {
let routed = false;
const denied = fixture({
policy: {
async authorize() {
return { effect, reasons: ['masked'], fence: null };
},
},
runAttemptLogReadRoute: {
async handle() {
routed = true;
throw new Error('must not route');
},
},
});
assert.deepEqual(await execute(denied.admission, request({ operation })), {
statusCode: 404,
body: { code: 'artifact_not_found' },
});
assert.equal(routed, false);
}
});
test('uses the same admission chain with a route-owned run.list audit identity', async () => {
const { admission, events } = fixture();
assert.deepEqual(
packages/ql3-local-api/test/httpSurface.test.cjs
+135 -93
View File
@@ -74,60 +74,71 @@ test('serves only the fixed canonical loopback Run route and drains idempotently
host: '127.0.0.1',
port,
admission: preparedAdmission(async (value, body) => {
observed.push(value);
if (
value.operation.operationId === 'run.cancel' ||
value.operation.operationId === 'task.start'
) {
return { statusCode: 202, body: { accepted: body } };
}
if (value.operation.operationId === 'run.get') {
return {
statusCode: 200,
body: { run: { id: value.operation.runId } },
};
}
if (value.operation.operationId === 'run.events.list') {
return {
statusCode: 200,
body: {
events: [],
hasMore: false,
nextAfterSequence: value.operation.input.afterSequence ?? 0,
},
};
}
if (value.operation.operationId === 'run.steps.list') {
return {
statusCode: 200,
body: {
steps: [],
hasMore: false,
next: value.operation.input.after ?? null,
},
};
}
if (value.operation.operationId === 'task.list') {
return {
statusCode: 200,
body: {
tasks: [],
hasMore: false,
input: value.operation.input,
},
};
}
if (value.operation.operationId === 'task.get') {
return {
statusCode: 200,
body: { task: { taskId: value.operation.taskId } },
};
}
observed.push(value);
if (
value.operation.operationId === 'run.cancel' ||
value.operation.operationId === 'task.start'
) {
return { statusCode: 202, body: { accepted: body } };
}
if (value.operation.operationId === 'run.get') {
return {
statusCode: 200,
body: { runs: [], hasMore: false, input: value.operation.input },
body: { run: { id: value.operation.runId } },
};
}),
}
if (value.operation.operationId === 'run.events.list') {
return {
statusCode: 200,
body: {
events: [],
hasMore: false,
nextAfterSequence: value.operation.input.afterSequence ?? 0,
},
};
}
if (value.operation.operationId === 'run.steps.list') {
return {
statusCode: 200,
body: {
steps: [],
hasMore: false,
next: value.operation.input.after ?? null,
},
};
}
if (value.operation.operationId === 'run.log.read') {
return {
statusCode: 200,
body: {
range: {
offset: value.operation.offset,
length: value.operation.length,
},
},
};
}
if (value.operation.operationId === 'task.list') {
return {
statusCode: 200,
body: {
tasks: [],
hasMore: false,
input: value.operation.input,
},
};
}
if (value.operation.operationId === 'task.get') {
return {
statusCode: 200,
body: { task: { taskId: value.operation.taskId } },
};
}
return {
statusCode: 200,
body: { runs: [], hasMore: false, input: value.operation.input },
};
}),
randomUuid: () => '019f70c0-0000-4000-8000-000000000003',
});
t.after(() => surface.stopAndDrain());
@@ -212,10 +223,7 @@ test('serves only the fixed canonical loopback Run route and drains idempotently
input: { after: { taskId: 'task_100' }, limit: 8 },
});
const task = await request(
port,
'/api/v3/projects/prj_default/tasks/task_1',
);
const task = await request(port, '/api/v3/projects/prj_default/tasks/task_1');
assert.deepEqual(task.body, { task: { taskId: 'task_1' } });
assert.deepEqual(observed[5].operation, {
operationId: 'task.get',
@@ -275,6 +283,28 @@ test('serves only the fixed canonical loopback Run route and drains idempotently
taskId: 'task_1',
});
const log = await request(
port,
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?offset=4&length=32',
);
assert.deepEqual(log.body, { range: { offset: 4, length: 32 } });
assert.deepEqual(observed[8].operation, {
operationId: 'run.log.read',
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_1',
offset: 4,
length: 32,
});
const defaultLog = await request(
port,
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log',
);
assert.deepEqual(defaultLog.body, {
range: { offset: 0, length: 16 * 1024 },
});
for (const invalidPath of [
'/api/v3/projects/prj_default/runs/run_123?expanded=true',
'/api/v3/projects/prj_default/runs/run%5f123',
@@ -299,6 +329,18 @@ test('serves only the fixed canonical loopback Run route and drains idempotently
assert.equal(invalid.statusCode, 400);
assert.deepEqual(invalid.body, { code: 'invalid_run_list_query' });
}
for (const invalidQuery of [
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?',
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?offset=-1',
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?offset=04',
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?length=0',
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?length=32769',
'/api/v3/projects/prj_default/runs/run_123/attempts/attempt_1/log?unknown=1',
]) {
const invalid = await request(port, invalidQuery);
assert.equal(invalid.statusCode, 400);
assert.deepEqual(invalid.body, { code: 'invalid_run_log_read_query' });
}
for (const invalidQuery of [
'/api/v3/projects/prj_default/tasks?',
'/api/v3/projects/prj_default/tasks?limit=08',
@@ -332,7 +374,7 @@ test('serves only the fixed canonical loopback Run route and drains idempotently
assert.equal(invalid.statusCode, 400);
assert.deepEqual(invalid.body, { code: 'invalid_run_step_list_query' });
}
assert.equal(observed.length, 8);
assert.equal(observed.length, 10);
assert.deepEqual(
await Promise.all([surface.stopAndDrain(), surface.stopAndDrain()]),
['stopped', 'stopped'],
@@ -347,9 +389,9 @@ test('rejects GET bodies without invoking the prepared route handler', async (t)
host: '127.0.0.1',
port,
admission: preparedAdmission(async () => {
handlers += 1;
return { statusCode: 200, body: {} };
}),
handlers += 1;
return { statusCode: 200, body: {} };
}),
});
t.after(() => surface.stopAndDrain());
const response = await request(
@@ -477,15 +519,15 @@ test('serves the reviewed worst-case 64-item Run list inside the fixed response
host: '127.0.0.1',
port,
admission: preparedAdmission(async () => {
return {
statusCode: 200,
body: {
runs: Object.freeze(Array.from({ length: 64 }, () => item)),
hasMore: true,
next: { createdAtMs: Number.MAX_SAFE_INTEGER, runId: id128 },
},
};
}),
return {
statusCode: 200,
body: {
runs: Object.freeze(Array.from({ length: 64 }, () => item)),
hasMore: true,
next: { createdAtMs: Number.MAX_SAFE_INTEGER, runId: id128 },
},
};
}),
});
t.after(() => surface.stopAndDrain());
const response = await request(
@@ -545,15 +587,15 @@ test('serves the reviewed worst-case 64-item RunEvent list inside the fixed resp
host: '127.0.0.1',
port,
admission: preparedAdmission(async () => {
return {
statusCode: 200,
body: {
events: Object.freeze(Array.from({ length: 64 }, () => event)),
hasMore: true,
nextAfterSequence: event.sequence,
},
};
}),
return {
statusCode: 200,
body: {
events: Object.freeze(Array.from({ length: 64 }, () => event)),
hasMore: true,
nextAfterSequence: event.sequence,
},
};
}),
});
t.after(() => surface.stopAndDrain());
const response = await request(
@@ -589,15 +631,15 @@ test('serves the reviewed worst-case 64-item Run Step list inside the fixed resp
host: '127.0.0.1',
port,
admission: preparedAdmission(async () => {
return {
statusCode: 200,
body: {
steps: Object.freeze(Array.from({ length: 64 }, () => item)),
hasMore: true,
next: { stepKey: id128, stepRunId: id128 },
},
};
}),
return {
statusCode: 200,
body: {
steps: Object.freeze(Array.from({ length: 64 }, () => item)),
hasMore: true,
next: { stepKey: id128, stepRunId: id128 },
},
};
}),
});
t.after(() => surface.stopAndDrain());
const response = await request(
@@ -621,13 +663,13 @@ test('bounds Edge admission concurrency and drains accepted work', async (t) =>
host: '127.0.0.1',
port,
admission: preparedAdmission(async (value) => {
admissions += 1;
await barrier;
return {
statusCode: 200,
body: { run: { id: value.operation.runId } },
};
}),
admissions += 1;
await barrier;
return {
statusCode: 200,
body: { run: { id: value.operation.runId } },
};
}),
});
t.after(() => surface.stopAndDrain());
const accepted = Array.from({ length: 4 }, () =>
packages/ql3-local-api/test/runAttemptLogReadRoute.test.cjs
+109
View File
@@ -0,0 +1,109 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
RunAttemptLogReadUnavailableError,
} = require('@qinglong/runtime-core/run-attempt-log-read');
const {
createLocalApiRunAttemptLogReadRoute,
} = require('../dist/run/runAttemptLogReadRoute.js');
function request(overrides = {}) {
return {
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_123',
offset: 2,
length: 16,
...overrides,
};
}
test('projects an available byte range as bounded base64 JSON', async () => {
const route = createLocalApiRunAttemptLogReadRoute({
async read(value) {
assert.deepEqual(value.range, { offset: 2, length: 16 });
return {
status: 'available',
projectId: value.projectId,
runId: value.runId,
attemptId: value.attemptId,
logArtifactId: `local-${'a'.repeat(30)}`,
content: Buffer.from('hello'),
start: 2,
endExclusive: 7,
totalBytes: 9,
nextOffset: 7,
truncation: { truncated: 'unknown' },
};
},
});
assert.deepEqual(await route.handle(request()), {
statusCode: 200,
body: {
schema: 'qinglong/run-attempt-log-read-result@v1',
status: 'available',
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_123',
range: { start: 2, endExclusive: 7, totalBytes: 9, nextOffset: 7 },
encoding: 'base64',
content: Buffer.from('hello').toString('base64'),
truncation: { truncated: 'unknown' },
},
});
});
test('maps pending, masked absence, missing storage and unavailable evidence', async () => {
const cases = [
[
{
status: 'pending',
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_123',
},
{
statusCode: 202,
body: {
schema: 'qinglong/run-attempt-log-read-result@v1',
status: 'pending',
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_123',
},
},
],
[
{ status: 'not_found' },
{ statusCode: 404, body: { code: 'artifact_not_found' } },
],
[
{
status: 'missing',
projectId: 'prj_default',
runId: 'run_123',
attemptId: 'attempt_123',
logArtifactId: `local-${'a'.repeat(30)}`,
},
{ statusCode: 503, body: { code: 'artifact_unavailable' } },
],
];
for (const [result, expected] of cases) {
const route = createLocalApiRunAttemptLogReadRoute({
async read() {
return result;
},
});
assert.deepEqual(await route.handle(request()), expected);
}
const unavailable = createLocalApiRunAttemptLogReadRoute({
async read() {
throw new RunAttemptLogReadUnavailableError();
},
});
assert.deepEqual(await unavailable.handle(request()), {
statusCode: 503,
body: { code: 'artifact_unavailable' },
});
});
packages/ql3-local-api/test/sqliteHttp.integration.test.cjs
+68 -12
View File
@@ -24,6 +24,9 @@ const {
const {
compileLocalCommandTaskDefinition,
} = require('@qinglong/runtime-core/task-definition-execution-compiler');
const {
RunAttemptLogReadService,
} = require('@qinglong/runtime-core/run-attempt-log-read');
const {
createBuiltInTaskSpecSemanticRegistry,
} = require('@qinglong/runtime-core/task-spec-semantic');
@@ -34,11 +37,16 @@ const {
const {
createLocalApiProductSurface,
} = require('../dist/application-runtime/localApiProductSurface.js');
const {
LocalRunAttemptLogRangeReader,
} = require('../../ql3-local-execution/dist/artifact-read/localRunAttemptLogRangeReader.js');
const NOW = 1_800_000_000_000;
const PEPPER_KEY_ID = 'local-api-pepper-v1';
const CREDENTIAL_ID = 'local-api-owner';
const RUN_ID = 'run_local_api_1';
const ATTEMPT_ID = 'attempt_local_api_1';
const LOG_ARTIFACT_ID = `local-${'a'.repeat(30)}`;
const SECRET = Buffer.alloc(32, 81).toString('base64url');
const PEPPER = Buffer.alloc(32, 82).toString('base64url');
const TOKEN = formatApiCredentialToken(CREDENTIAL_ID, SECRET);
@@ -206,15 +214,18 @@ function seed(databasePath, materialDigest) {
enabled: true,
occurredAtMs: NOW - 200,
};
const taskDefinition = createTaskDefinitionRecord({
...taskCommand,
spec: taskSemantics.normalize({
projectId: taskCommand.projectId,
taskId: taskCommand.taskId,
kind: taskCommand.kind,
spec: taskCommand.spec,
}),
}, NOW - 200);
const taskDefinition = createTaskDefinitionRecord(
{
...taskCommand,
spec: taskSemantics.normalize({
projectId: taskCommand.projectId,
taskId: taskCommand.taskId,
kind: taskCommand.kind,
spec: taskCommand.spec,
}),
},
NOW - 200,
);
const taskExecution = compileLocalCommandTaskDefinition(
taskDefinition,
taskSemantics,
@@ -298,6 +309,15 @@ function seed(databasePath, materialDigest) {
'manual', 'runtime', 'running', 1, 1, 0, ?)`,
)
.run(RUN_ID, NOW - 100);
client
.prepare(
`INSERT INTO "RunAttempts" (
"id", "run_id", "attempt", "status", "executor_type",
"log_artifact_id", "callback_sequence", "created_at_ms",
"started_at_ms"
) VALUES (?, ?, 1, 'running', 'local_process', ?, 0, ?, ?)`,
)
.run(ATTEMPT_ID, RUN_ID, LOG_ARTIFACT_ID, NOW - 90, NOW - 80);
client
.prepare(
`INSERT INTO "StepRuns" (
@@ -357,6 +377,15 @@ test('serves an authenticated Run through one real SQLite authority and durable
fs.chmodSync(root, 0o700);
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
const databasePath = path.join(root, 'qinglong3.sqlite');
const artifactRoot = path.join(root, 'artifacts');
const artifactShard = path.join(artifactRoot, 'aa');
fs.mkdirSync(artifactShard, { recursive: true, mode: 0o700 });
fs.chmodSync(artifactRoot, 0o700);
fs.chmodSync(artifactShard, 0o700);
const logContent = Buffer.from('local-api-log-line\n', 'utf8');
const logPath = path.join(artifactShard, `${LOG_ARTIFACT_ID}.log`);
fs.writeFileSync(logPath, logContent, { mode: 0o600 });
fs.chmodSync(logPath, 0o600);
const keyringDirectory = path.join(root, 'owner-pepper');
fs.mkdirSync(keyringDirectory, { mode: 0o700 });
const summary = provisionLocalOwnerPepperKey({
@@ -398,6 +427,15 @@ test('serves an authenticated Run through one real SQLite authority and durable
stepRuns: await runtime.stepRunReader(),
runCancellation: await runtime.runCancellationRepository(),
taskStart: await runtime.taskStartRepository(),
runAttemptLogRead: new RunAttemptLogReadService(
runtime.runRepository,
new LocalRunAttemptLogRangeReader(artifactRoot),
{
executorType: 'local_process',
artifactIdPattern: /^local-[a-f0-9]{30}$/,
maximumReadBytes: 32 * 1024,
},
),
taskDefinitions: runtime.taskDefinitions,
apiCredentials: runtime.apiCredentials,
ownerPepper: runtime.ownerPepper,
@@ -572,12 +610,29 @@ test('serves an authenticated Run through one real SQLite authority and durable
});
assert.equal(JSON.stringify(steps).includes('private'), false);
const log = await request(
port,
`Bearer ${TOKEN}`,
`/api/v3/projects/default/runs/${RUN_ID}/attempts/${ATTEMPT_ID}/log?offset=0&length=8`,
);
assert.equal(log.statusCode, 200);
assert.equal(log.body.schema, 'qinglong/run-attempt-log-read-result@v1');
assert.equal(log.body.status, 'available');
assert.equal(log.body.encoding, 'base64');
assert.equal(Buffer.from(log.body.content, 'base64').toString(), 'local-ap');
assert.deepEqual(log.body.range, {
start: 0,
endExclusive: 8,
totalBytes: logContent.byteLength,
nextOffset: 8,
});
assert.deepEqual(log.body.truncation, { truncated: 'unknown' });
const cancellationBody = JSON.stringify({
schema: 'qinglong/run-cancellation@v1',
mutationId: 'cancel-local-api-1',
});
const cancellationPath =
`/api/v3/projects/default/runs/${RUN_ID}/cancellation`;
const cancellationPath = `/api/v3/projects/default/runs/${RUN_ID}/cancellation`;
const cancellationOptions = {
method: 'POST',
headers: {
@@ -623,7 +678,7 @@ test('serves an authenticated Run through one real SQLite authority and durable
WHERE operation_id IN (
'run.get', 'run.list', 'run.events.list', 'run.steps.list',
'run.cancel', 'task.get', 'task.list'
, 'task.start'
, 'task.start', 'run.log.read'
)
ORDER BY operation_id, outcome`,
)
@@ -636,6 +691,7 @@ test('serves an authenticated Run through one real SQLite authority and durable
'run.get:allowed',
'run.get:authentication_rejected',
'run.list:allowed',
'run.log.read:allowed',
'run.steps.list:allowed',
'task.get:allowed',
'task.get:allowed',