open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-23 12:05:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): add opaque cluster environment bundle delivery

Browse Source
This commit is contained in:
whyour
2026-08-24 19:31:03 +08:00
parent cf2c0ec7b3
commit 4abf125ce9
36 changed files with 1682 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-cluster-control/src/remote-execution/mountedSecretProvider.ts
+24 -1
View File
@@ -1,6 +1,7 @@
// Remote Execution owns mounted Secret resolution for authenticated delivery.
import {
MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES,
MAX_REMOTE_ENVIRONMENT_BUNDLE_VALUE_BYTES,
MAX_REMOTE_SECRET_VALUE_BYTES,
normalizeRemoteWorkerSecretDeliveryAuthority,
type RemoteWorkerSecretDeliveryAuthority,
@@ -82,7 +83,7 @@ export class ClusterMountedSecretProvider
this.reader = new PrivateProjectedFileReader({
rootDirectory: options.rootDirectory,
minimumBytes: 0,
maximumBytes: MAX_REMOTE_SECRET_VALUE_BYTES,
maximumBytes: MAX_REMOTE_ENVIRONMENT_BUNDLE_VALUE_BYTES,
access: 'private_material',
});
} catch (error) {
@@ -116,6 +117,7 @@ export class ClusterMountedSecretProvider
const buffers: Buffer[] = [];
try {
const values = [];
const environmentBundles = [];
let totalBytes = 0;
for (const secretRef of normalized.secretRefs) {
const bytes = await this.reader
@@ -127,6 +129,9 @@ export class ClusterMountedSecretProvider
);
});
buffers.push(bytes);
if (bytes.byteLength > MAX_REMOTE_SECRET_VALUE_BYTES) {
throw new ClusterMountedSecretProviderError('material_unavailable');
}
totalBytes += bytes.byteLength;
if (totalBytes > MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES) {
throw new ClusterMountedSecretProviderError('material_unavailable');
@@ -138,9 +143,27 @@ export class ClusterMountedSecretProvider
}),
);
}
for (const secretRef of normalized.environmentBundleRefs) {
const bytes = await this.reader
.read(clusterMountedSecretFileName(secretRef))
.catch((error) => {
throw new ClusterMountedSecretProviderError(
'material_unavailable',
{ cause: error },
);
});
buffers.push(bytes);
environmentBundles.push(
Object.freeze({
secretRef,
value: secretValue(bytes),
}),
);
}
let disposed = false;
return Object.freeze({
values: Object.freeze(values),
environmentBundles: Object.freeze(environmentBundles),
dispose() {
if (disposed) return;
disposed = true;
packages/ql3-cluster-control/src/remote-execution/remoteWorkerSecretDeliveryService.ts
+11 -3
View File
@@ -66,7 +66,9 @@ export class ClusterRemoteWorkerSecretDeliveryService {
authorized.offerId !== command.offerId ||
authorized.leaseGeneration !== command.leaseGeneration ||
authorized.leaseVersion !== command.expectedLeaseVersion ||
JSON.stringify(authorized.secretRefs) !== JSON.stringify(command.secretRefs)
JSON.stringify(authorized.secretRefs) !== JSON.stringify(command.secretRefs) ||
JSON.stringify(authorized.environmentBundleRefs) !==
JSON.stringify(command.environmentBundleRefs)
) throw new InvalidRemoteWorkerSecretDeliveryError(
'repository authority does not match command',
);
@@ -88,7 +90,8 @@ export class ClusterRemoteWorkerSecretDeliveryService {
if (
typeof resolution !== 'object' ||
Array.isArray(resolution) ||
Object.keys(resolution).some((key) => key !== 'values' && key !== 'dispose') ||
Object.keys(resolution).some((key) =>
key !== 'values' && key !== 'environmentBundles' && key !== 'dispose') ||
(resolution.dispose !== undefined &&
typeof resolution.dispose !== 'function')
) throw new InvalidRemoteWorkerSecretDeliveryError(
@@ -100,13 +103,18 @@ export class ClusterRemoteWorkerSecretDeliveryService {
offerId: authorized.offerId,
executionDigest: authorized.executionDigest,
values: resolution.values,
}, authorized.secretRefs);
environmentBundles: resolution.environmentBundles,
}, {
secretRefs: authorized.secretRefs,
environmentBundleRefs: authorized.environmentBundleRefs,
});
return Object.freeze({
runId: body.runId,
attemptId: body.attemptId,
offerId: body.offerId,
executionDigest: body.executionDigest,
values: body.values,
environmentBundles: body.environmentBundles,
...(resolution.dispose === undefined
? {}
: { dispose: resolution.dispose }),