mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): add profile-aware run log reads
This commit is contained in:
whyour
@@ -28,6 +28,11 @@
|
||||
"require": "./dist/dispatch/index.js",
|
||||
"default": "./dist/dispatch/index.js"
|
||||
},
|
||||
"./artifact-read": {
|
||||
"types": "./dist/artifact-read/index.d.ts",
|
||||
"require": "./dist/artifact-read/index.js",
|
||||
"default": "./dist/artifact-read/index.js"
|
||||
},
|
||||
"./scheduler": {
|
||||
"types": "./dist/scheduler/index.d.ts",
|
||||
"require": "./dist/scheduler/index.js",
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export * from './localRunAttemptLogRangeReader';
|
||||
@@ -0,0 +1,285 @@
|
||||
import { constants, type Stats } from 'node:fs';
|
||||
import fs, { type FileHandle } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
import {
|
||||
normalizeRunAttemptLogReadRange,
|
||||
type RunAttemptLogRangeReader,
|
||||
type RunAttemptLogRangeReadResult,
|
||||
type RunAttemptLogReadIdentity,
|
||||
type RunAttemptLogReadRange,
|
||||
type RunAttemptLogTruncationView,
|
||||
} from '@qinglong/runtime-core/run-attempt-log-read';
|
||||
|
||||
const LOCAL_ARTIFACT_ID = /^local-[a-f0-9]{30}$/;
|
||||
const MAXIMUM_ARTIFACT_BYTES = 1024 * 1024 * 1024;
|
||||
const MAXIMUM_FACT_BYTES = 1024;
|
||||
|
||||
export class LocalRunAttemptLogRangeReadError extends Error {
|
||||
constructor(
|
||||
readonly reason:
|
||||
| 'invalid_configuration'
|
||||
| 'unsafe_path'
|
||||
| 'integrity_mismatch',
|
||||
options?: ErrorOptions,
|
||||
) {
|
||||
super(`Local Run Attempt log range read failed: ${reason}`, options);
|
||||
this.name = 'LocalRunAttemptLogRangeReadError';
|
||||
}
|
||||
}
|
||||
|
||||
function isCode(error: unknown, code: string): boolean {
|
||||
return (
|
||||
!!error &&
|
||||
typeof error === 'object' &&
|
||||
'code' in error &&
|
||||
(error as { code?: unknown }).code === code
|
||||
);
|
||||
}
|
||||
|
||||
function currentUid(): number | undefined {
|
||||
return typeof process.getuid === 'function' ? process.getuid() : undefined;
|
||||
}
|
||||
|
||||
function root(value: string): string {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
!path.isAbsolute(value) ||
|
||||
path.parse(value).root === value ||
|
||||
value.includes('\0') ||
|
||||
Buffer.byteLength(value, 'utf8') > 4096
|
||||
) {
|
||||
throw new LocalRunAttemptLogRangeReadError('invalid_configuration');
|
||||
}
|
||||
return path.resolve(value);
|
||||
}
|
||||
|
||||
function identity(
|
||||
value: Readonly<RunAttemptLogReadIdentity>,
|
||||
): Readonly<RunAttemptLogReadIdentity> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
!LOCAL_ARTIFACT_ID.test(value.logArtifactId)
|
||||
) {
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function assertOwnedDirectory(stat: Stats): void {
|
||||
const uid = currentUid();
|
||||
if (
|
||||
!stat.isDirectory() ||
|
||||
stat.isSymbolicLink() ||
|
||||
(stat.mode & 0o777) !== 0o700 ||
|
||||
(uid !== undefined && stat.uid !== uid)
|
||||
) {
|
||||
throw new LocalRunAttemptLogRangeReadError('unsafe_path');
|
||||
}
|
||||
}
|
||||
|
||||
function assertOwnedFile(stat: Stats): void {
|
||||
const uid = currentUid();
|
||||
if (
|
||||
!stat.isFile() ||
|
||||
stat.nlink !== 1 ||
|
||||
(stat.mode & 0o777) !== 0o600 ||
|
||||
(uid !== undefined && stat.uid !== uid) ||
|
||||
!Number.isSafeInteger(stat.size) ||
|
||||
stat.size < 0 ||
|
||||
stat.size > MAXIMUM_ARTIFACT_BYTES
|
||||
) {
|
||||
throw new LocalRunAttemptLogRangeReadError('unsafe_path');
|
||||
}
|
||||
}
|
||||
|
||||
async function optionalPrivateDirectory(directory: string): Promise<boolean> {
|
||||
try {
|
||||
assertOwnedDirectory(await fs.lstat(directory));
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return false;
|
||||
if (error instanceof LocalRunAttemptLogRangeReadError) throw error;
|
||||
throw new LocalRunAttemptLogRangeReadError('unsafe_path', { cause: error });
|
||||
}
|
||||
}
|
||||
|
||||
async function openPrivateFile(
|
||||
filePath: string,
|
||||
): Promise<FileHandle | undefined> {
|
||||
try {
|
||||
return await fs.open(
|
||||
filePath,
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return undefined;
|
||||
throw new LocalRunAttemptLogRangeReadError('unsafe_path', { cause: error });
|
||||
}
|
||||
}
|
||||
|
||||
function exactFact(
|
||||
value: unknown,
|
||||
expected: Readonly<RunAttemptLogReadIdentity>,
|
||||
): Readonly<RunAttemptLogTruncationView> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
|
||||
}
|
||||
const fact = value as Record<string, unknown>;
|
||||
const keys = Object.keys(fact).sort();
|
||||
if (
|
||||
keys.join(',') !==
|
||||
'attemptId,logArtifactId,maximumBytes,observedAtMs,quotaReached,runId,schemaVersion' ||
|
||||
fact.schemaVersion !== 1 ||
|
||||
fact.runId !== expected.runId ||
|
||||
fact.attemptId !== expected.attemptId ||
|
||||
fact.logArtifactId !== expected.logArtifactId ||
|
||||
!Number.isSafeInteger(fact.maximumBytes) ||
|
||||
Number(fact.maximumBytes) < 64 * 1024 ||
|
||||
Number(fact.maximumBytes) > MAXIMUM_ARTIFACT_BYTES ||
|
||||
typeof fact.quotaReached !== 'boolean' ||
|
||||
!Number.isSafeInteger(fact.observedAtMs) ||
|
||||
Number(fact.observedAtMs) < 0
|
||||
) {
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
|
||||
}
|
||||
return Object.freeze({
|
||||
truncated: fact.quotaReached,
|
||||
maximumBytes: fact.maximumBytes as number,
|
||||
observedAtMs: fact.observedAtMs as number,
|
||||
});
|
||||
}
|
||||
|
||||
async function readTruncationFact(
|
||||
directory: string,
|
||||
expected: Readonly<RunAttemptLogReadIdentity>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Readonly<RunAttemptLogTruncationView>> {
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
const factPath = path.join(
|
||||
directory,
|
||||
`.${expected.logArtifactId}.log.truncated.json`,
|
||||
);
|
||||
const handle = await openPrivateFile(factPath);
|
||||
if (!handle) return Object.freeze({ truncated: 'unknown' as const });
|
||||
try {
|
||||
const before = await handle.stat();
|
||||
assertOwnedFile(before);
|
||||
if (before.size < 2 || before.size > MAXIMUM_FACT_BYTES) {
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
|
||||
}
|
||||
const content = Buffer.allocUnsafe(before.size);
|
||||
let read = 0;
|
||||
while (read < content.byteLength) {
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
const result = await handle.read(
|
||||
content,
|
||||
read,
|
||||
content.byteLength - read,
|
||||
read,
|
||||
);
|
||||
if (result.bytesRead < 1) {
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
|
||||
}
|
||||
read += result.bytesRead;
|
||||
}
|
||||
const after = await handle.stat();
|
||||
assertOwnedFile(after);
|
||||
if (
|
||||
after.dev !== before.dev ||
|
||||
after.ino !== before.ino ||
|
||||
after.size !== before.size
|
||||
) {
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
|
||||
}
|
||||
try {
|
||||
const text = new TextDecoder('utf-8', { fatal: true }).decode(content);
|
||||
return exactFact(JSON.parse(text), expected);
|
||||
} catch (error) {
|
||||
if (error instanceof LocalRunAttemptLogRangeReadError) throw error;
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch', {
|
||||
cause: error,
|
||||
});
|
||||
} finally {
|
||||
content.fill(0);
|
||||
}
|
||||
} finally {
|
||||
await handle.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalRunAttemptLogRangeReader implements RunAttemptLogRangeReader {
|
||||
private readonly root: string;
|
||||
|
||||
constructor(artifactRoot: string) {
|
||||
this.root = root(artifactRoot);
|
||||
}
|
||||
|
||||
async read(
|
||||
rawIdentity: Readonly<RunAttemptLogReadIdentity>,
|
||||
rawRange: Readonly<RunAttemptLogReadRange>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RunAttemptLogRangeReadResult> {
|
||||
const expected = identity(rawIdentity);
|
||||
const range = normalizeRunAttemptLogReadRange(rawRange);
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
if (!(await optionalPrivateDirectory(this.root))) {
|
||||
return Object.freeze({ status: 'missing' as const });
|
||||
}
|
||||
const directory = path.join(
|
||||
this.root,
|
||||
expected.logArtifactId.slice('local-'.length, 'local-'.length + 2),
|
||||
);
|
||||
if (!(await optionalPrivateDirectory(directory))) {
|
||||
return Object.freeze({ status: 'missing' as const });
|
||||
}
|
||||
const target = path.join(directory, `${expected.logArtifactId}.log`);
|
||||
const handle = await openPrivateFile(target);
|
||||
if (!handle) return Object.freeze({ status: 'missing' as const });
|
||||
try {
|
||||
const before = await handle.stat();
|
||||
assertOwnedFile(before);
|
||||
const start = Math.min(range.offset, before.size);
|
||||
const expectedBytes = Math.min(range.length, before.size - start);
|
||||
const content = Buffer.allocUnsafe(expectedBytes);
|
||||
let read = 0;
|
||||
while (read < expectedBytes) {
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
const result = await handle.read(
|
||||
content,
|
||||
read,
|
||||
expectedBytes - read,
|
||||
start + read,
|
||||
);
|
||||
if (result.bytesRead < 1) {
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
|
||||
}
|
||||
read += result.bytesRead;
|
||||
}
|
||||
const after = await handle.stat();
|
||||
assertOwnedFile(after);
|
||||
if (
|
||||
after.dev !== before.dev ||
|
||||
after.ino !== before.ino ||
|
||||
after.size < before.size
|
||||
) {
|
||||
throw new LocalRunAttemptLogRangeReadError('integrity_mismatch');
|
||||
}
|
||||
const endExclusive = start + content.byteLength;
|
||||
const truncation = await readTruncationFact(directory, expected, signal);
|
||||
return Object.freeze({
|
||||
status: 'available' as const,
|
||||
content,
|
||||
start,
|
||||
endExclusive,
|
||||
totalBytes: before.size,
|
||||
...(endExclusive < before.size ? { nextOffset: endExclusive } : {}),
|
||||
truncation,
|
||||
});
|
||||
} finally {
|
||||
await handle.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
LocalRunAttemptLogRangeReadError,
|
||||
LocalRunAttemptLogRangeReader,
|
||||
} = require('../dist/artifact-read/localRunAttemptLogRangeReader.js');
|
||||
|
||||
const artifactId = `local-${'a'.repeat(30)}`;
|
||||
const identity = Object.freeze({
|
||||
projectId: 'prj_default',
|
||||
runId: 'run_123',
|
||||
attemptId: 'attempt_123',
|
||||
logArtifactId: artifactId,
|
||||
});
|
||||
|
||||
async function fixture(t, content = Buffer.from('0123456789')) {
|
||||
const parent = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-log-read-'));
|
||||
t.after(() => fs.rm(parent, { recursive: true, force: true }));
|
||||
const root = path.join(parent, 'artifacts');
|
||||
const shard = path.join(root, 'aa');
|
||||
await fs.mkdir(shard, { recursive: true, mode: 0o700 });
|
||||
await fs.chmod(root, 0o700);
|
||||
await fs.chmod(shard, 0o700);
|
||||
const log = path.join(shard, `${artifactId}.log`);
|
||||
await fs.writeFile(log, content, { mode: 0o600 });
|
||||
await fs.chmod(log, 0o600);
|
||||
return { parent, root, shard, log };
|
||||
}
|
||||
|
||||
async function fact(shard, overrides = {}) {
|
||||
const file = path.join(shard, `.${artifactId}.log.truncated.json`);
|
||||
await fs.writeFile(
|
||||
file,
|
||||
JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
runId: identity.runId,
|
||||
attemptId: identity.attemptId,
|
||||
logArtifactId: identity.logArtifactId,
|
||||
maximumBytes: 64 * 1024,
|
||||
quotaReached: false,
|
||||
observedAtMs: 9,
|
||||
...overrides,
|
||||
}),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
await fs.chmod(file, 0o600);
|
||||
return file;
|
||||
}
|
||||
|
||||
test('reads one bounded private-file snapshot and canonical truncation fact', async (t) => {
|
||||
const value = await fixture(t);
|
||||
await fact(value.shard, { quotaReached: true });
|
||||
const result = await new LocalRunAttemptLogRangeReader(value.root).read(
|
||||
identity,
|
||||
{ offset: 2, length: 4 },
|
||||
);
|
||||
assert.equal(result.status, 'available');
|
||||
assert.equal(Buffer.from(result.content).toString(), '2345');
|
||||
assert.deepEqual(
|
||||
{
|
||||
start: result.start,
|
||||
endExclusive: result.endExclusive,
|
||||
totalBytes: result.totalBytes,
|
||||
nextOffset: result.nextOffset,
|
||||
truncation: result.truncation,
|
||||
},
|
||||
{
|
||||
start: 2,
|
||||
endExclusive: 6,
|
||||
totalBytes: 10,
|
||||
nextOffset: 6,
|
||||
truncation: {
|
||||
truncated: true,
|
||||
maximumBytes: 64 * 1024,
|
||||
observedAtMs: 9,
|
||||
},
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('returns unknown truncation and a stable empty range beyond the snapshot', async (t) => {
|
||||
const value = await fixture(t);
|
||||
const result = await new LocalRunAttemptLogRangeReader(value.root).read(
|
||||
identity,
|
||||
{ offset: 99, length: 4 },
|
||||
);
|
||||
assert.equal(result.status, 'available');
|
||||
assert.equal(result.content.byteLength, 0);
|
||||
assert.equal(result.start, 10);
|
||||
assert.equal(result.endExclusive, 10);
|
||||
assert.equal(result.totalBytes, 10);
|
||||
assert.equal(result.nextOffset, undefined);
|
||||
assert.deepEqual(result.truncation, { truncated: 'unknown' });
|
||||
});
|
||||
|
||||
test('treats absent root, shard and log as missing', async (t) => {
|
||||
const parent = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-log-missing-'));
|
||||
t.after(() => fs.rm(parent, { recursive: true, force: true }));
|
||||
const root = path.join(parent, 'artifacts');
|
||||
const reader = new LocalRunAttemptLogRangeReader(root);
|
||||
assert.deepEqual(await reader.read(identity, { offset: 0, length: 1 }), {
|
||||
status: 'missing',
|
||||
});
|
||||
await fs.mkdir(root, { mode: 0o700 });
|
||||
assert.deepEqual(await reader.read(identity, { offset: 0, length: 1 }), {
|
||||
status: 'missing',
|
||||
});
|
||||
const shard = path.join(root, 'aa');
|
||||
await fs.mkdir(shard, { mode: 0o700 });
|
||||
assert.deepEqual(await reader.read(identity, { offset: 0, length: 1 }), {
|
||||
status: 'missing',
|
||||
});
|
||||
});
|
||||
|
||||
test('fails closed for symlink targets and widened file permissions', async (t) => {
|
||||
const symlink = await fixture(t);
|
||||
await fs.rm(symlink.log);
|
||||
await fs.symlink(path.join(symlink.parent, 'outside'), symlink.log);
|
||||
await assert.rejects(
|
||||
new LocalRunAttemptLogRangeReader(symlink.root).read(identity, {
|
||||
offset: 0,
|
||||
length: 1,
|
||||
}),
|
||||
LocalRunAttemptLogRangeReadError,
|
||||
);
|
||||
|
||||
const widened = await fixture(t);
|
||||
await fs.chmod(widened.log, 0o644);
|
||||
await assert.rejects(
|
||||
new LocalRunAttemptLogRangeReader(widened.root).read(identity, {
|
||||
offset: 0,
|
||||
length: 1,
|
||||
}),
|
||||
LocalRunAttemptLogRangeReadError,
|
||||
);
|
||||
});
|
||||
|
||||
test('fails closed for truncation identity drift and an aborted request', async (t) => {
|
||||
const value = await fixture(t);
|
||||
await fact(value.shard, { attemptId: 'attempt_other' });
|
||||
const reader = new LocalRunAttemptLogRangeReader(value.root);
|
||||
await assert.rejects(
|
||||
reader.read(identity, { offset: 0, length: 1 }),
|
||||
(error) =>
|
||||
error instanceof LocalRunAttemptLogRangeReadError &&
|
||||
error.reason === 'integrity_mismatch',
|
||||
);
|
||||
const abort = new AbortController();
|
||||
abort.abort(new Error('cancelled'));
|
||||
await assert.rejects(
|
||||
reader.read(identity, { offset: 0, length: 1 }, abort.signal),
|
||||
/cancelled/,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user