open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-22 10:32:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): add opaque cluster environment bundle delivery

Browse Source
This commit is contained in:
whyour
2026-08-24 19:31:03 +08:00
parent cf2c0ec7b3
commit 4abf125ce9
36 changed files with 1682 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-runtime-core/src/task-definition/clusterExecutionRevision.ts
+27 -10
View File
@@ -32,6 +32,7 @@ export interface ClusterTaskExecutionRevisionContent {
readonly planSchema: typeof CLUSTER_EXECUTION_PLAN_SCHEMA;
readonly command: LocalDispatchCommand;
readonly environment: readonly LocalExecutionEnvironmentBinding[];
readonly environmentBundleRef?: string;
readonly workingDirectory?: string;
readonly timeoutMs?: number;
readonly placement?: RemoteWorkerPlacementSpec;
@@ -79,9 +80,7 @@ function revision(value: unknown): number {
(value as number) < 1 ||
(value as number) > 2_147_483_647
) {
throw new InvalidClusterExecutionRevisionError(
'sourceRevision is invalid',
);
throw new InvalidClusterExecutionRevisionError('sourceRevision is invalid');
}
return value as number;
}
@@ -103,6 +102,7 @@ function normalizeContent(
'command',
'createdAtMs',
'environment',
'environmentBundleRef',
'executorType',
'planSchema',
'placement',
@@ -155,6 +155,7 @@ function normalizeContent(
}
let command: LocalDispatchCommand;
let environment: readonly LocalExecutionEnvironmentBinding[];
let environmentBundleRef: string | undefined;
try {
command = normalizeLocalDispatchCommand(value.command);
environment = createLocalExecutionContextRecipe({
@@ -169,6 +170,16 @@ function normalizeContent(
throw new Error('cross-project Secret reference');
}
}
if (value.environmentBundleRef !== undefined) {
const reference = parseSecretRef(value.environmentBundleRef);
if (
reference.projectId !== projectId ||
reference.version === undefined
) {
throw new Error('invalid environment bundle Secret reference');
}
environmentBundleRef = value.environmentBundleRef;
}
} catch {
throw new InvalidClusterExecutionRevisionError(
'command or environment is invalid',
@@ -200,9 +211,10 @@ function normalizeContent(
timeoutMs = value.timeoutMs;
}
const createdAtMs = timestamp(value.createdAtMs);
const placement = value.placement === undefined
? undefined
: effectiveRemoteWorkerPlacement(value.placement);
const placement =
value.placement === undefined
? undefined
: effectiveRemoteWorkerPlacement(value.placement);
const normalized = Object.freeze({
projectId,
taskId,
@@ -213,20 +225,22 @@ function normalizeContent(
planSchema: CLUSTER_EXECUTION_PLAN_SCHEMA,
command,
environment,
...(environmentBundleRef === undefined ? {} : { environmentBundleRef }),
...(workingDirectory === undefined ? {} : { workingDirectory }),
...(timeoutMs === undefined ? {} : { timeoutMs }),
...(placement === undefined ? {} : { placement }),
createdAtMs,
});
if (Buffer.byteLength(JSON.stringify(normalized), 'utf8') > MAX_CLUSTER_EXECUTION_PLAN_BYTES) {
if (
Buffer.byteLength(JSON.stringify(normalized), 'utf8') >
MAX_CLUSTER_EXECUTION_PLAN_BYTES
) {
throw new InvalidClusterExecutionRevisionError('plan byte budget exceeded');
}
return normalized;
}
function digest(
content: ClusterTaskExecutionRevisionContent,
): string {
function digest(content: ClusterTaskExecutionRevisionContent): string {
const { createdAtMs: _createdAtMs, ...immutable } = content;
return createHash('sha256')
.update('qinglong.cluster-task-execution-revision.v1\0', 'utf8')
@@ -273,6 +287,9 @@ export function compileClusterCommandTaskDefinition(
planSchema: CLUSTER_EXECUTION_PLAN_SCHEMA,
command: plan.command,
environment: plan.environment,
...(plan.environmentBundleRef === undefined
? {}
: { environmentBundleRef: plan.environmentBundleRef }),
...(plan.workingDirectory === undefined
? {}
: { workingDirectory: plan.workingDirectory }),
packages/ql3-runtime-core/src/task-definition/taskDefinitionExecutionCompiler.ts
+12 -4
View File
@@ -36,6 +36,7 @@ export interface CommandTaskExecutionPlan {
readonly sourceContentDigest: string;
readonly command: LocalDispatchCommand;
readonly environment: readonly LocalExecutionEnvironmentBinding[];
readonly environmentBundleRef?: string;
readonly workingDirectory?: string;
readonly timeoutMs?: number;
readonly placement?: RemoteWorkerPlacementSpec;
@@ -112,13 +113,13 @@ export function parseTaskDefinitionRevisionRef(
return Object.freeze({ revision, contentDigest });
}
function canonicalRecord(definition: TaskDefinitionRecord): TaskDefinitionRecord {
function canonicalRecord(
definition: TaskDefinitionRecord,
): TaskDefinitionRecord {
try {
return normalizeTaskDefinitionRecord(definition);
} catch {
throw new InvalidTaskDefinitionCompilationError(
'source record is invalid',
);
throw new InvalidTaskDefinitionCompilationError('source record is invalid');
}
}
@@ -172,6 +173,7 @@ export function compileCommandTaskDefinition(
const config = semanticSpec.config as unknown as Readonly<{
command: LocalDispatchCommand;
environment: readonly LocalExecutionEnvironmentBinding[];
environmentBundleRef?: string;
workingDirectory?: string;
timeoutMs?: number;
placement?: RemoteWorkerPlacementSpec;
@@ -188,6 +190,9 @@ export function compileCommandTaskDefinition(
sourceContentDigest: source.contentDigest,
command: config.command,
environment: config.environment,
...(config.environmentBundleRef === undefined
? {}
: { environmentBundleRef: config.environmentBundleRef }),
...(config.workingDirectory === undefined
? {}
: { workingDirectory: config.workingDirectory }),
@@ -202,6 +207,9 @@ export function compileLocalCommandTaskDefinition(
semanticRegistry: TaskSpecSemanticRegistry,
): LocalCommandTaskExecutionPlan {
const source = compileCommandTaskDefinition(definition, semanticRegistry);
if (source.environmentBundleRef !== undefined) {
throw new UnsupportedTaskDefinitionCompilationError();
}
const contextRecipe = createLocalExecutionContextRecipe({
environment: source.environment,
createdAtMs: source.createdAtMs,
packages/ql3-runtime-core/src/task-definition/taskSpecSemantic.ts
+41 -16
View File
@@ -196,9 +196,7 @@ function normalizeEnvironment(
);
});
if (bytes > MAX_COMMAND_TASK_ENVIRONMENT_BYTES) {
throw new InvalidTaskSpecSemanticError(
'environment byte budget exceeded',
);
throw new InvalidTaskSpecSemanticError('environment byte budget exceeded');
}
environment.sort((left, right) =>
(left as { name: string }).name.localeCompare(
@@ -215,14 +213,46 @@ function normalizeCommandConfig(
exactKeys(
config,
['command'],
['environment', 'placement', 'timeoutMs', 'workingDirectory'],
[
'environment',
'environmentBundleRef',
'placement',
'timeoutMs',
'workingDirectory',
],
'command config',
);
const command = normalizeCommand(config.command);
const environment = normalizeEnvironment(config.environment ?? [], context.projectId);
const placement = config.placement === undefined
? undefined
: normalizeRemoteWorkerPlacement(config.placement);
const environment = normalizeEnvironment(
config.environment ?? [],
context.projectId,
);
let environmentBundleRef: string | undefined;
if (config.environmentBundleRef !== undefined) {
environmentBundleRef = boundedText(
config.environmentBundleRef,
'environmentBundleRef',
512,
);
let reference;
try {
reference = parseSecretRef(environmentBundleRef);
} catch {
throw new InvalidTaskSpecSemanticError('environmentBundleRef is invalid');
}
if (
reference.projectId !== context.projectId ||
reference.version === undefined
) {
throw new InvalidTaskSpecSemanticError(
'environmentBundleRef must pin a version in the same Project',
);
}
}
const placement =
config.placement === undefined
? undefined
: normalizeRemoteWorkerPlacement(config.placement);
let workingDirectory: string | undefined;
if (config.workingDirectory !== undefined) {
workingDirectory = boundedText(
@@ -250,6 +280,7 @@ function normalizeCommandConfig(
return Object.freeze({
command,
environment,
...(environmentBundleRef === undefined ? {} : { environmentBundleRef }),
...(placement === undefined
? {}
: { placement: placement as unknown as TaskDefinitionJson }),
@@ -268,10 +299,7 @@ const BUILT_IN_DESCRIPTORS: readonly TaskSpecSemanticDescriptor[] =
]);
export class TaskSpecSemanticRegistry {
readonly #descriptors: ReadonlyMap<
string,
TaskSpecSemanticDescriptor
>;
readonly #descriptors: ReadonlyMap<string, TaskSpecSemanticDescriptor>;
readonly #metadata: readonly TaskSpecSemanticMetadata[];
constructor(descriptors: readonly TaskSpecSemanticDescriptor[]) {
@@ -393,8 +421,5 @@ export function createTaskSpecSemanticRegistry(
'extension descriptor uses the reserved qinglong namespace',
);
}
return new TaskSpecSemanticRegistry([
...BUILT_IN_DESCRIPTORS,
...extensions,
]);
return new TaskSpecSemanticRegistry([...BUILT_IN_DESCRIPTORS, ...extensions]);
}