open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-22 10:32:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): add opaque cluster environment bundle delivery

Browse Source
This commit is contained in:
whyour
2026-08-24 19:31:03 +08:00
parent cf2c0ec7b3
commit 4abf125ce9
36 changed files with 1682 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-worker-runtime/src/remote-execution/executionContextMaterializer.ts
+135 -48
View File
@@ -1,8 +1,10 @@
// Remote Execution owns bounded Secret and Artifact context materialization.
import {
MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES,
MAX_LOCAL_DISPATCH_ENVIRONMENT_ENTRIES,
MAX_LOCAL_DISPATCH_SECRET_REFS,
} from '@qinglong/runtime-core/local-dispatch';
import { parseEnvironmentBundle } from '@qinglong/runtime-core/environment-bundle';
import type { ClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import { createClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import { assertRunDispatchId } from '@qinglong/runtime-core/run-dispatch-lease';
@@ -17,20 +19,27 @@ export interface WorkerRemoteSecretResolution {
secretRef: string;
value: string;
}>[];
readonly environmentBundles: readonly Readonly<{
secretRef: string;
value: string;
}>[];
readonly dispose?: () => Promise<void> | void;
}
export interface WorkerRemoteSecretEnvironmentProvider {
resolve(request: Readonly<{
projectId: string;
taskId: string;
taskRevision: string;
runId: string;
attemptId: string;
offerId: string;
executionDigest: string;
secretRefs: readonly string[];
}>): Promise<WorkerRemoteSecretResolution | undefined>;
resolve(
request: Readonly<{
projectId: string;
taskId: string;
taskRevision: string;
runId: string;
attemptId: string;
offerId: string;
executionDigest: string;
secretRefs: readonly string[];
environmentBundleRefs: readonly string[];
}>,
): Promise<WorkerRemoteSecretResolution | undefined>;
}
export interface WorkerRemoteLogArtifactPreparation {
@@ -42,12 +51,14 @@ export interface WorkerRemoteLogArtifactPreparation {
}
export interface WorkerRemoteLogArtifactAllocator {
prepare(request: Readonly<{
projectId: string;
runId: string;
attemptId: string;
offerId: string;
}>): Promise<WorkerRemoteLogArtifactPreparation | undefined>;
prepare(
request: Readonly<{
projectId: string;
runId: string;
attemptId: string;
offerId: string;
}>,
): Promise<WorkerRemoteLogArtifactPreparation | undefined>;
}
export interface BoundedWorkerRemoteExecutionContextMaterializerOptions {
@@ -86,11 +97,14 @@ function environmentValue(value: unknown): string {
async function disposeQuietly(
operation: (() => Promise<void> | void) | undefined,
): Promise<void> {
await Promise.resolve().then(() => operation?.()).catch(() => undefined);
await Promise.resolve()
.then(() => operation?.())
.catch(() => undefined);
}
export class BoundedWorkerRemoteExecutionContextMaterializer
implements WorkerRemoteExecutionContextMaterializer {
implements WorkerRemoteExecutionContextMaterializer
{
private readonly artifacts: WorkerRemoteLogArtifactAllocator;
private readonly secrets?: WorkerRemoteSecretEnvironmentProvider;
@@ -109,9 +123,11 @@ export class BoundedWorkerRemoteExecutionContextMaterializer
this.secrets = options.secrets;
}
async prepare(input: Readonly<{
offer: ClusterRemoteExecutionOffer;
}>): Promise<MaterializedWorkerRemoteExecutionContext> {
async prepare(
input: Readonly<{
offer: ClusterRemoteExecutionOffer;
}>,
): Promise<MaterializedWorkerRemoteExecutionContext> {
let offer: ClusterRemoteExecutionOffer;
try {
offer = createClusterRemoteExecutionOffer(input?.offer);
@@ -122,9 +138,17 @@ export class BoundedWorkerRemoteExecutionContextMaterializer
}
const bindings = offer.executionRevision.environment;
const secretRefs = Object.freeze([
...new Set(bindings.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [])),
...new Set(
bindings.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [],
),
),
]);
const environmentBundleRefs = Object.freeze(
offer.executionRevision.environmentBundleRef === undefined
? []
: [offer.executionRevision.environmentBundleRef],
);
if (secretRefs.length > MAX_LOCAL_DISPATCH_SECRET_REFS) {
throw new WorkerRemoteExecutionMaterializationError(
'environment_budget_exceeded',
@@ -132,23 +156,26 @@ export class BoundedWorkerRemoteExecutionContextMaterializer
}
let secretResolution: WorkerRemoteSecretResolution | undefined;
const secretByRef = new Map<string, string>();
if (secretRefs.length > 0) {
if (secretRefs.length > 0 || environmentBundleRefs.length > 0) {
if (!this.secrets) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_unavailable',
);
}
try {
secretResolution = await this.secrets.resolve(Object.freeze({
projectId: offer.candidate.projectId,
taskId: offer.candidate.taskId,
taskRevision: offer.candidate.taskRevision,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
executionDigest: offer.executionDigest,
secretRefs,
}));
secretResolution = await this.secrets.resolve(
Object.freeze({
projectId: offer.candidate.projectId,
taskId: offer.candidate.taskId,
taskRevision: offer.candidate.taskRevision,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
executionDigest: offer.executionDigest,
secretRefs,
environmentBundleRefs,
}),
);
} catch {
throw new WorkerRemoteExecutionMaterializationError(
'secret_unavailable',
@@ -160,10 +187,17 @@ export class BoundedWorkerRemoteExecutionContextMaterializer
);
}
if (
Object.keys(secretResolution).some((key) =>
key !== 'values' && key !== 'dispose') ||
Object.keys(secretResolution).some(
(key) =>
key !== 'values' &&
key !== 'environmentBundles' &&
key !== 'dispose',
) ||
!Array.isArray(secretResolution.values) ||
secretResolution.values.length !== secretRefs.length ||
!Array.isArray(secretResolution.environmentBundles) ||
secretResolution.environmentBundles.length !==
environmentBundleRefs.length ||
(secretResolution.dispose !== undefined &&
typeof secretResolution.dispose !== 'function')
) {
@@ -198,36 +232,89 @@ export class BoundedWorkerRemoteExecutionContextMaterializer
let environmentBytes = 0;
let environment: MaterializedWorkerRemoteExecutionContext['environment'];
try {
environment = Object.freeze(bindings.map((binding) => {
const value = binding.kind === 'public'
? binding.value
: secretByRef.get(binding.secretRef);
const names = new Set<string>();
const materialized = bindings.map((binding) => {
const value =
binding.kind === 'public'
? binding.value
: secretByRef.get(binding.secretRef);
if (value === undefined) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
environmentBytes += Buffer.byteLength(binding.name, 'utf8') +
environmentBytes +=
Buffer.byteLength(binding.name, 'utf8') +
Buffer.byteLength(value, 'utf8');
names.add(binding.name);
if (environmentBytes > MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES) {
throw new WorkerRemoteExecutionMaterializationError(
'environment_budget_exceeded',
);
}
return Object.freeze({ name: binding.name, value });
}));
});
for (const entry of secretResolution?.environmentBundles ?? []) {
if (
!entry ||
typeof entry !== 'object' ||
Object.keys(entry).length !== 2 ||
!Object.hasOwn(entry, 'secretRef') ||
!Object.hasOwn(entry, 'value') ||
typeof entry.secretRef !== 'string' ||
!environmentBundleRefs.includes(entry.secretRef) ||
typeof entry.value !== 'string'
) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
let bundle;
try {
bundle = parseEnvironmentBundle(entry.value);
} catch {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
for (const binding of bundle.entries) {
if (names.has(binding.name)) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
names.add(binding.name);
environmentBytes +=
Buffer.byteLength(binding.name, 'utf8') +
Buffer.byteLength(binding.value, 'utf8');
if (
materialized.length >= MAX_LOCAL_DISPATCH_ENVIRONMENT_ENTRIES ||
environmentBytes > MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES
) {
throw new WorkerRemoteExecutionMaterializationError(
'environment_budget_exceeded',
);
}
materialized.push(
Object.freeze({ name: binding.name, value: binding.value }),
);
}
}
environment = Object.freeze(materialized);
} catch (error) {
await disposeQuietly(secretResolution?.dispose);
throw error;
}
let artifact: WorkerRemoteLogArtifactPreparation | undefined;
try {
artifact = await this.artifacts.prepare(Object.freeze({
projectId: offer.candidate.projectId,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
}));
artifact = await this.artifacts.prepare(
Object.freeze({
projectId: offer.candidate.projectId,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
}),
);
} catch {
await disposeQuietly(secretResolution?.dispose);
throw new WorkerRemoteExecutionMaterializationError(
packages/ql3-worker-runtime/src/remote-execution/transport/remoteSecretHttpsProvider.ts
+37 -11
View File
@@ -36,7 +36,8 @@ export interface WorkerRemoteSecretHttpsProviderOptions {
}
export class WorkerRemoteSecretHttpsProvider
implements WorkerRemoteSecretEnvironmentProvider {
implements WorkerRemoteSecretEnvironmentProvider
{
private readonly client: Pick<WorkerIngressHttpsClient, 'postJson'>;
private readonly inbox: Pick<WorkerRemoteExecutionInbox, 'readOffer'>;
@@ -45,13 +46,15 @@ export class WorkerRemoteSecretHttpsProvider
!options ||
typeof options.client?.postJson !== 'function' ||
typeof options.inbox?.readOffer !== 'function'
) throw new WorkerRemoteSecretHttpsProviderError('invalid_configuration');
)
throw new WorkerRemoteSecretHttpsProviderError('invalid_configuration');
this.client = options.client;
this.inbox = options.inbox;
}
async resolve(request: Parameters<WorkerRemoteSecretEnvironmentProvider['resolve']>[0])
: Promise<WorkerRemoteSecretResolution | undefined> {
async resolve(
request: Parameters<WorkerRemoteSecretEnvironmentProvider['resolve']>[0],
): Promise<WorkerRemoteSecretResolution | undefined> {
let record;
try {
record = await this.inbox.readOffer(request.offerId);
@@ -68,9 +71,17 @@ export class WorkerRemoteSecretHttpsProvider
throw new WorkerRemoteSecretHttpsProviderError('authority_mismatch');
}
const expectedRefs = Object.freeze([
...new Set(offer.executionRevision.environment.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [])),
...new Set(
offer.executionRevision.environment.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [],
),
),
]);
const expectedEnvironmentBundleRefs = Object.freeze(
offer.executionRevision.environmentBundleRef === undefined
? []
: [offer.executionRevision.environmentBundleRef],
);
if (
offer.offerId !== request.offerId ||
offer.executionDigest !== request.executionDigest ||
@@ -79,10 +90,14 @@ export class WorkerRemoteSecretHttpsProvider
offer.candidate.taskRevision !== request.taskRevision ||
offer.candidate.runId !== request.runId ||
offer.candidate.attemptId !== request.attemptId ||
JSON.stringify(expectedRefs) !== JSON.stringify(request.secretRefs)
) throw new WorkerRemoteSecretHttpsProviderError('authority_mismatch');
JSON.stringify(expectedRefs) !== JSON.stringify(request.secretRefs) ||
JSON.stringify(expectedEnvironmentBundleRefs) !==
JSON.stringify(request.environmentBundleRefs)
)
throw new WorkerRemoteSecretHttpsProviderError('authority_mismatch');
const path = `/api/v3/worker-ingress/workers/${offer.worker.workerId}` +
const path =
`/api/v3/worker-ingress/workers/${offer.worker.workerId}` +
`/sessions/${offer.worker.sessionId}/secrets`;
const body = createRemoteWorkerSecretDeliveryRequestBody({
workerId: offer.worker.workerId,
@@ -99,6 +114,7 @@ export class WorkerRemoteSecretHttpsProvider
leaseToken: offer.leaseToken,
expectedLeaseVersion: offer.lease.version,
secretRefs: expectedRefs,
environmentBundleRefs: expectedEnvironmentBundleRefs,
});
let serialized: Uint8Array;
try {
@@ -119,11 +135,21 @@ export class WorkerRemoteSecretHttpsProvider
offerId: offer.offerId,
executionDigest: offer.executionDigest,
secretRefs: expectedRefs,
environmentBundleRefs: expectedEnvironmentBundleRefs,
});
const values = Object.freeze(delivered.values.map((entry) =>
Object.freeze({ secretRef: entry.secretRef, value: entry.value })));
const values = Object.freeze(
delivered.values.map((entry) =>
Object.freeze({ secretRef: entry.secretRef, value: entry.value }),
),
);
const environmentBundles = Object.freeze(
delivered.environmentBundles.map((entry) =>
Object.freeze({ secretRef: entry.secretRef, value: entry.value }),
),
);
return Object.freeze({
values,
environmentBundles,
dispose() {
// JavaScript strings cannot be zeroized. Drop all retained references;
// the transport bytes were already scrubbed by the parser.
packages/ql3-worker-runtime/src/remote-execution/transport/workerIngressHttpsClient.ts
+3 -1
View File
@@ -17,7 +17,9 @@ const MAX_TLS_MATERIAL_BYTES = 1024 * 1024;
const MAX_REQUEST_BYTES = 4096;
const HARD_MAX_REQUEST_BYTES = 64 * 1024;
const HARD_MAX_STREAM_REQUEST_BYTES = 64 * 1024 * 1024 + 4 * 1024 + 4;
const MAX_RESPONSE_BYTES = 128 * 1024;
// JSON routes retain their own smaller requested caps. The shared ceiling must
// also admit one bounded environment bundle response.
const MAX_RESPONSE_BYTES = 256 * 1024;
const CREDENTIAL_POOL_KEY = Symbol('qinglong.worker-ingress-credential-pool-key');
export const WORKER_INGRESS_ARTIFACT_CONTENT_TYPE =