mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 18:08:20 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,185 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { constants } from 'node:fs';
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import type { LocalDispatchCandidate } from '@qinglong/runtime-core/local-dispatch';
|
||||
import { normalizeLocalDispatchCandidate } from '@qinglong/runtime-core/local-dispatch';
|
||||
|
||||
export const MIN_LOCAL_ARTIFACT_MAXIMUM_BYTES = 64 * 1024;
|
||||
export const MAX_LOCAL_ARTIFACT_MAXIMUM_BYTES = 1024 * 1024 * 1024;
|
||||
export const MAX_LOCAL_ARTIFACT_MINIMUM_FREE_BYTES = 1024 ** 4;
|
||||
|
||||
export interface LocalArtifactCapacityPolicy {
|
||||
readonly maximumAttemptBytes: number;
|
||||
readonly minimumFreeBytes: number;
|
||||
}
|
||||
|
||||
export interface LocalArtifactCapacityProbe {
|
||||
inspect(directory: string): Promise<bigint>;
|
||||
}
|
||||
|
||||
export interface PreparedLocalArtifact {
|
||||
readonly logArtifactId: string;
|
||||
readonly output: Readonly<{
|
||||
filePath: string;
|
||||
maximumBytes: number;
|
||||
logArtifactId: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface LocalArtifactAllocator {
|
||||
prepare(candidate: LocalDispatchCandidate): Promise<PreparedLocalArtifact>;
|
||||
}
|
||||
|
||||
export class LocalArtifactCapacityUnavailableError extends Error {
|
||||
readonly code = 'LOCAL_ARTIFACT_CAPACITY_UNAVAILABLE';
|
||||
|
||||
constructor() {
|
||||
super('Local Artifact capacity is unavailable');
|
||||
this.name = 'LocalArtifactCapacityUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalArtifactIdentityConflictError extends Error {
|
||||
readonly code = 'LOCAL_ARTIFACT_IDENTITY_CONFLICT';
|
||||
|
||||
constructor() {
|
||||
super('Local Artifact identity is already occupied');
|
||||
this.name = 'LocalArtifactIdentityConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
class StatFsLocalArtifactCapacityProbe implements LocalArtifactCapacityProbe {
|
||||
async inspect(directory: string): Promise<bigint> {
|
||||
const stat = await fs.statfs(directory, { bigint: true });
|
||||
return stat.bavail * stat.bsize;
|
||||
}
|
||||
}
|
||||
|
||||
function assertAbsoluteRoot(value: string): string {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
!path.isAbsolute(value) ||
|
||||
path.parse(value).root === value ||
|
||||
value.includes('\0') ||
|
||||
Buffer.byteLength(value, 'utf8') > 4096
|
||||
) {
|
||||
throw new TypeError('Local Artifact root is invalid');
|
||||
}
|
||||
return path.resolve(value);
|
||||
}
|
||||
|
||||
function normalizePolicy(
|
||||
policy: LocalArtifactCapacityPolicy,
|
||||
): Readonly<LocalArtifactCapacityPolicy> {
|
||||
if (!policy || typeof policy !== 'object' || Array.isArray(policy)) {
|
||||
throw new TypeError('Local Artifact capacity policy is invalid');
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(policy.maximumAttemptBytes) ||
|
||||
policy.maximumAttemptBytes < MIN_LOCAL_ARTIFACT_MAXIMUM_BYTES ||
|
||||
policy.maximumAttemptBytes > MAX_LOCAL_ARTIFACT_MAXIMUM_BYTES ||
|
||||
!Number.isSafeInteger(policy.minimumFreeBytes) ||
|
||||
policy.minimumFreeBytes < 0 ||
|
||||
policy.minimumFreeBytes > MAX_LOCAL_ARTIFACT_MINIMUM_FREE_BYTES
|
||||
) {
|
||||
throw new RangeError('Local Artifact capacity policy is out of range');
|
||||
}
|
||||
return Object.freeze({ ...policy });
|
||||
}
|
||||
|
||||
async function ensurePrivateDirectory(directory: string): Promise<void> {
|
||||
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
const stat = await fs.lstat(directory);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
||||
throw new TypeError('Local Artifact directory is unsafe');
|
||||
}
|
||||
await fs.chmod(directory, 0o700);
|
||||
}
|
||||
|
||||
export function localArtifactCapacityPolicyForProfile(
|
||||
profile: 'edge' | 'standalone',
|
||||
): Readonly<LocalArtifactCapacityPolicy> {
|
||||
if (profile === 'edge') {
|
||||
return Object.freeze({
|
||||
maximumAttemptBytes: 4 * 1024 * 1024,
|
||||
minimumFreeBytes: 32 * 1024 * 1024,
|
||||
});
|
||||
}
|
||||
if (profile === 'standalone') {
|
||||
return Object.freeze({
|
||||
maximumAttemptBytes: 64 * 1024 * 1024,
|
||||
minimumFreeBytes: 256 * 1024 * 1024,
|
||||
});
|
||||
}
|
||||
throw new TypeError('Local Artifact Profile is invalid');
|
||||
}
|
||||
|
||||
export function localArtifactId(candidate: LocalDispatchCandidate): string {
|
||||
const normalized = normalizeLocalDispatchCandidate(candidate);
|
||||
return `local-${createHash('sha256')
|
||||
.update(normalized.runId, 'utf8')
|
||||
.update('\0', 'utf8')
|
||||
.update(normalized.attemptId, 'utf8')
|
||||
.digest('hex')
|
||||
.slice(0, 30)}`;
|
||||
}
|
||||
|
||||
export class LocalFileArtifactAllocator implements LocalArtifactAllocator {
|
||||
private readonly root: string;
|
||||
private readonly policy: Readonly<LocalArtifactCapacityPolicy>;
|
||||
|
||||
constructor(
|
||||
root: string,
|
||||
policy: LocalArtifactCapacityPolicy,
|
||||
private readonly capacity: LocalArtifactCapacityProbe = new StatFsLocalArtifactCapacityProbe(),
|
||||
) {
|
||||
this.root = assertAbsoluteRoot(root);
|
||||
this.policy = normalizePolicy(policy);
|
||||
}
|
||||
|
||||
async prepare(
|
||||
candidate: LocalDispatchCandidate,
|
||||
): Promise<PreparedLocalArtifact> {
|
||||
const normalized = normalizeLocalDispatchCandidate(candidate);
|
||||
const logArtifactId = localArtifactId(normalized);
|
||||
const shard = logArtifactId.slice(6, 8);
|
||||
const directory = path.join(this.root, shard);
|
||||
await ensurePrivateDirectory(this.root);
|
||||
const availableBytes = await this.capacity.inspect(this.root);
|
||||
const requiredBytes =
|
||||
BigInt(this.policy.minimumFreeBytes) +
|
||||
BigInt(this.policy.maximumAttemptBytes);
|
||||
if (availableBytes < requiredBytes) {
|
||||
throw new LocalArtifactCapacityUnavailableError();
|
||||
}
|
||||
await ensurePrivateDirectory(directory);
|
||||
const filePath = path.join(directory, `${logArtifactId}.log`);
|
||||
let file: fs.FileHandle | undefined;
|
||||
try {
|
||||
file = await fs.open(
|
||||
filePath,
|
||||
constants.O_WRONLY |
|
||||
constants.O_CREAT |
|
||||
constants.O_APPEND |
|
||||
(constants.O_NOFOLLOW ?? 0),
|
||||
0o600,
|
||||
);
|
||||
const stat = await file.stat();
|
||||
if (!stat.isFile() || stat.size !== 0) {
|
||||
throw new LocalArtifactIdentityConflictError();
|
||||
}
|
||||
await file.chmod(0o600);
|
||||
} finally {
|
||||
await file?.close().catch(() => undefined);
|
||||
}
|
||||
return Object.freeze({
|
||||
logArtifactId,
|
||||
output: Object.freeze({
|
||||
filePath,
|
||||
maximumBytes: this.policy.maximumAttemptBytes,
|
||||
logArtifactId,
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
import {
|
||||
LOCAL_PROCESS_EXECUTOR_TYPE,
|
||||
assertLocalDispatchPageSize,
|
||||
normalizeLocalDispatchCandidate,
|
||||
type LocalDispatchCandidate,
|
||||
type LocalDispatchCandidateCursor,
|
||||
type LocalDispatchCandidateSource,
|
||||
} from '@qinglong/runtime-core/local-dispatch';
|
||||
import {
|
||||
LocalExecutionLaunchError,
|
||||
LocalExecutionRejectedError,
|
||||
type LocalExecutionStartCommand,
|
||||
type LocalExecutionStartResult,
|
||||
} from '../execution/coordinator';
|
||||
import type { LocalDispatchPlanSource } from './materializer';
|
||||
|
||||
export interface LocalDispatchActivator {
|
||||
start(
|
||||
command: LocalExecutionStartCommand,
|
||||
): Promise<LocalExecutionStartResult>;
|
||||
}
|
||||
|
||||
export interface LocalRunDispatcherOptions {
|
||||
readonly pageSize?: number;
|
||||
readonly maxPages?: number;
|
||||
readonly onCompletion?: (attemptId: string) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface LocalRunDispatcherStats {
|
||||
readonly pages: number;
|
||||
readonly candidatesScanned: number;
|
||||
readonly plansUnavailable: number;
|
||||
readonly activationRaces: number;
|
||||
}
|
||||
|
||||
export type LocalRunDispatcherResult =
|
||||
| Readonly<{
|
||||
status: 'activated';
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
stats: LocalRunDispatcherStats;
|
||||
truncated: boolean;
|
||||
}>
|
||||
| Readonly<{
|
||||
status: 'activation_failed';
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
stats: LocalRunDispatcherStats;
|
||||
truncated: boolean;
|
||||
}>
|
||||
| Readonly<{
|
||||
status: 'idle';
|
||||
reason:
|
||||
| 'no_candidates'
|
||||
| 'plans_unavailable'
|
||||
| 'activation_raced'
|
||||
| 'scan_budget_exhausted';
|
||||
stats: LocalRunDispatcherStats;
|
||||
truncated: boolean;
|
||||
}>;
|
||||
|
||||
function cursorOf(
|
||||
candidate: LocalDispatchCandidate,
|
||||
): LocalDispatchCandidateCursor {
|
||||
return Object.freeze({
|
||||
priority: candidate.priority,
|
||||
queuedAtMs: candidate.queuedAtMs,
|
||||
attemptCreatedAtMs: candidate.attemptCreatedAtMs,
|
||||
attemptId: candidate.attemptId,
|
||||
});
|
||||
}
|
||||
|
||||
function advances(
|
||||
previous: LocalDispatchCandidateCursor,
|
||||
next: LocalDispatchCandidateCursor,
|
||||
): boolean {
|
||||
return (
|
||||
next.priority < previous.priority ||
|
||||
(next.priority === previous.priority &&
|
||||
(next.queuedAtMs > previous.queuedAtMs ||
|
||||
(next.queuedAtMs === previous.queuedAtMs &&
|
||||
(next.attemptCreatedAtMs > previous.attemptCreatedAtMs ||
|
||||
(next.attemptCreatedAtMs === previous.attemptCreatedAtMs &&
|
||||
next.attemptId > previous.attemptId)))))
|
||||
);
|
||||
}
|
||||
|
||||
export class LocalRunDispatcher {
|
||||
private readonly pageSize: number;
|
||||
private readonly maxPages: number;
|
||||
private readonly onCompletion?: LocalRunDispatcherOptions['onCompletion'];
|
||||
|
||||
constructor(
|
||||
private readonly candidates: LocalDispatchCandidateSource,
|
||||
private readonly plans: LocalDispatchPlanSource,
|
||||
private readonly activator: LocalDispatchActivator,
|
||||
options: LocalRunDispatcherOptions = {},
|
||||
) {
|
||||
this.pageSize = options.pageSize ?? 8;
|
||||
this.maxPages = options.maxPages ?? 1;
|
||||
this.onCompletion = options.onCompletion;
|
||||
assertLocalDispatchPageSize(this.pageSize);
|
||||
if (
|
||||
!Number.isSafeInteger(this.maxPages) ||
|
||||
this.maxPages < 1 ||
|
||||
this.maxPages > 16
|
||||
) {
|
||||
throw new RangeError('Local dispatch maxPages must be between 1 and 16');
|
||||
}
|
||||
}
|
||||
|
||||
async dispatchOnce(): Promise<LocalRunDispatcherResult> {
|
||||
const stats = {
|
||||
pages: 0,
|
||||
candidatesScanned: 0,
|
||||
plansUnavailable: 0,
|
||||
activationRaces: 0,
|
||||
};
|
||||
const seen = new Set<string>();
|
||||
let after: LocalDispatchCandidateCursor | undefined;
|
||||
let truncated = false;
|
||||
for (let pageIndex = 0; pageIndex < this.maxPages; pageIndex += 1) {
|
||||
const page = await this.candidates.listLocalDispatchCandidates({
|
||||
limit: this.pageSize,
|
||||
...(after === undefined ? {} : { after }),
|
||||
});
|
||||
if (page.candidates.length > this.pageSize) {
|
||||
throw new RangeError('Local dispatch source exceeded its page size');
|
||||
}
|
||||
stats.pages += 1;
|
||||
truncated = page.truncated;
|
||||
let previous = after;
|
||||
for (const value of page.candidates) {
|
||||
const candidate = normalizeLocalDispatchCandidate(value);
|
||||
if (candidate.executorType !== LOCAL_PROCESS_EXECUTOR_TYPE) {
|
||||
throw new TypeError(
|
||||
'Local dispatch source returned another executor',
|
||||
);
|
||||
}
|
||||
const cursor = cursorOf(candidate);
|
||||
if (
|
||||
seen.has(candidate.attemptId) ||
|
||||
(previous !== undefined && !advances(previous, cursor))
|
||||
) {
|
||||
throw new TypeError('Local dispatch page is not strictly ordered');
|
||||
}
|
||||
seen.add(candidate.attemptId);
|
||||
previous = cursor;
|
||||
stats.candidatesScanned += 1;
|
||||
const plan = await this.plans.prepare(candidate);
|
||||
if (!plan) {
|
||||
stats.plansUnavailable += 1;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const active = await this.activator.start(plan.command);
|
||||
void active.handle.completion
|
||||
.then(
|
||||
() => this.onCompletion?.(active.attempt.id),
|
||||
() => this.onCompletion?.(active.attempt.id),
|
||||
)
|
||||
.catch(() => undefined);
|
||||
return Object.freeze({
|
||||
status: 'activated' as const,
|
||||
runId: active.run.id,
|
||||
attemptId: active.attempt.id,
|
||||
stats: Object.freeze({ ...stats }),
|
||||
truncated,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof LocalExecutionRejectedError) {
|
||||
if (
|
||||
error.reason === 'aggregate_mismatch' ||
|
||||
error.reason === 'executor_mismatch'
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
stats.activationRaces += 1;
|
||||
continue;
|
||||
}
|
||||
if (error instanceof LocalExecutionLaunchError) {
|
||||
return Object.freeze({
|
||||
status: 'activation_failed' as const,
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
stats: Object.freeze({ ...stats }),
|
||||
truncated,
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
if (!page.truncated) return this.idle(stats, false);
|
||||
const last = page.candidates.at(-1);
|
||||
if (!last) {
|
||||
throw new TypeError(
|
||||
'Local dispatch source reported an empty truncated page',
|
||||
);
|
||||
}
|
||||
after = cursorOf(last);
|
||||
}
|
||||
return Object.freeze({
|
||||
status: 'idle' as const,
|
||||
reason: 'scan_budget_exhausted' as const,
|
||||
stats: Object.freeze({ ...stats }),
|
||||
truncated: true,
|
||||
});
|
||||
}
|
||||
|
||||
private idle(
|
||||
stats: LocalRunDispatcherStats,
|
||||
truncated: boolean,
|
||||
): LocalRunDispatcherResult {
|
||||
const reason =
|
||||
stats.candidatesScanned === 0
|
||||
? 'no_candidates'
|
||||
: stats.plansUnavailable === stats.candidatesScanned
|
||||
? 'plans_unavailable'
|
||||
: 'activation_raced';
|
||||
return Object.freeze({
|
||||
status: 'idle' as const,
|
||||
reason,
|
||||
stats: Object.freeze({ ...stats }),
|
||||
truncated,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
export * from './artifact';
|
||||
export * from './materializer';
|
||||
export * from './dispatcher';
|
||||
export type {
|
||||
LocalDispatchDefinitionWriter,
|
||||
LocalDispatchStore,
|
||||
LocalSecretEnvironmentProvider,
|
||||
} from '@qinglong/runtime-core/local-dispatch';
|
||||
@@ -0,0 +1,133 @@
|
||||
import type {
|
||||
LocalDispatchCandidate,
|
||||
LocalDispatchStore,
|
||||
LocalSecretEnvironmentProvider,
|
||||
} from '@qinglong/runtime-core/local-dispatch';
|
||||
import {
|
||||
MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES,
|
||||
MAX_LOCAL_DISPATCH_SECRET_REFS,
|
||||
normalizeLocalDispatchCandidate,
|
||||
normalizeLocalExecutionContextRecipe,
|
||||
normalizeLocalTaskExecutionRevision,
|
||||
} from '@qinglong/runtime-core/local-dispatch';
|
||||
import type { LocalExecutionStartCommand } from '../execution/coordinator';
|
||||
import type { LocalArtifactAllocator } from './artifact';
|
||||
|
||||
export interface LocalDispatchPlan {
|
||||
readonly command: LocalExecutionStartCommand;
|
||||
}
|
||||
|
||||
export interface LocalDispatchPlanSource {
|
||||
prepare(candidate: LocalDispatchCandidate): Promise<LocalDispatchPlan | null>;
|
||||
}
|
||||
|
||||
function assertEnvironmentValue(value: unknown): asserts value is string {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.includes('\0') ||
|
||||
Buffer.byteLength(value, 'utf8') > 16 * 1024
|
||||
) {
|
||||
throw new TypeError('Local dispatch environment value is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalDispatchPlanMaterializer implements LocalDispatchPlanSource {
|
||||
constructor(
|
||||
private readonly definitions: Pick<
|
||||
LocalDispatchStore,
|
||||
'resolveLocalTaskExecutionRevision' | 'resolveLocalExecutionContextRecipe'
|
||||
>,
|
||||
private readonly artifacts: LocalArtifactAllocator,
|
||||
private readonly secrets?: LocalSecretEnvironmentProvider,
|
||||
) {}
|
||||
|
||||
async prepare(
|
||||
candidate: LocalDispatchCandidate,
|
||||
): Promise<LocalDispatchPlan | null> {
|
||||
const normalizedCandidate = normalizeLocalDispatchCandidate(candidate);
|
||||
const revision = await this.definitions.resolveLocalTaskExecutionRevision({
|
||||
projectId: normalizedCandidate.projectId,
|
||||
taskId: normalizedCandidate.taskId,
|
||||
taskRevision: normalizedCandidate.taskRevision,
|
||||
});
|
||||
if (!revision) return null;
|
||||
const normalizedRevision = normalizeLocalTaskExecutionRevision(revision);
|
||||
if (
|
||||
normalizedRevision.projectId !== normalizedCandidate.projectId ||
|
||||
normalizedRevision.taskId !== normalizedCandidate.taskId ||
|
||||
normalizedRevision.taskRevision !== normalizedCandidate.taskRevision ||
|
||||
normalizedRevision.executorType !== normalizedCandidate.executorType
|
||||
) {
|
||||
throw new TypeError('Local Task revision does not match its candidate');
|
||||
}
|
||||
const recipe = await this.definitions.resolveLocalExecutionContextRecipe(
|
||||
normalizedRevision.contextRef,
|
||||
);
|
||||
if (!recipe) return null;
|
||||
const normalizedRecipe = normalizeLocalExecutionContextRecipe(recipe);
|
||||
if (normalizedRecipe.contextRef !== normalizedRevision.contextRef) {
|
||||
throw new TypeError('Local context recipe does not match its revision');
|
||||
}
|
||||
const secretRefs = Object.freeze([
|
||||
...new Set(
|
||||
normalizedRecipe.environment.flatMap((binding) =>
|
||||
binding.kind === 'secret' ? [binding.secretRef] : [],
|
||||
),
|
||||
),
|
||||
]);
|
||||
if (secretRefs.length > MAX_LOCAL_DISPATCH_SECRET_REFS) {
|
||||
throw new RangeError('Local dispatch Secret reference budget exceeded');
|
||||
}
|
||||
let secretValues: readonly string[] = [];
|
||||
if (secretRefs.length > 0) {
|
||||
if (!this.secrets) return null;
|
||||
const resolved = await this.secrets.resolveLocalSecretEnvironment({
|
||||
candidate: normalizedCandidate,
|
||||
secretRefs,
|
||||
});
|
||||
if (!resolved) return null;
|
||||
if (resolved.length !== secretRefs.length) {
|
||||
throw new TypeError('Local Secret provider returned an invalid result');
|
||||
}
|
||||
secretValues = resolved;
|
||||
}
|
||||
const secretByRef = new Map(
|
||||
secretRefs.map((secretRef, index) => [secretRef, secretValues[index]]),
|
||||
);
|
||||
const environment: Record<string, string> = Object.create(null);
|
||||
let environmentBytes = 0;
|
||||
for (const binding of normalizedRecipe.environment) {
|
||||
const value =
|
||||
binding.kind === 'public'
|
||||
? binding.value
|
||||
: secretByRef.get(binding.secretRef);
|
||||
assertEnvironmentValue(value);
|
||||
environmentBytes +=
|
||||
Buffer.byteLength(binding.name, 'utf8') +
|
||||
Buffer.byteLength(value, 'utf8');
|
||||
if (environmentBytes > MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES) {
|
||||
throw new RangeError('Local dispatch environment byte budget exceeded');
|
||||
}
|
||||
environment[binding.name] = value;
|
||||
}
|
||||
const artifact = await this.artifacts.prepare(normalizedCandidate);
|
||||
return Object.freeze({
|
||||
command: Object.freeze({
|
||||
runId: normalizedCandidate.runId,
|
||||
attemptId: normalizedCandidate.attemptId,
|
||||
...(normalizedCandidate.stepRunId === undefined
|
||||
? {}
|
||||
: { stepRunId: normalizedCandidate.stepRunId }),
|
||||
command: normalizedRevision.command,
|
||||
environment: Object.freeze(environment),
|
||||
...(normalizedRevision.workingDirectory === undefined
|
||||
? {}
|
||||
: { workingDirectory: normalizedRevision.workingDirectory }),
|
||||
...(normalizedRevision.timeoutMs === undefined
|
||||
? {}
|
||||
: { timeoutMs: normalizedRevision.timeoutMs }),
|
||||
output: artifact.output,
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user