open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-23 12:05:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): add opaque cluster environment bundle delivery

Browse Source
This commit is contained in:
whyour
2026-08-24 19:31:03 +08:00
parent cf2c0ec7b3
commit 4abf125ce9
36 changed files with 1682 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-cluster-control/src/remote-execution/mountedSecretProvider.ts
+24 -1
View File
@@ -1,6 +1,7 @@
// Remote Execution owns mounted Secret resolution for authenticated delivery.
import {
MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES,
MAX_REMOTE_ENVIRONMENT_BUNDLE_VALUE_BYTES,
MAX_REMOTE_SECRET_VALUE_BYTES,
normalizeRemoteWorkerSecretDeliveryAuthority,
type RemoteWorkerSecretDeliveryAuthority,
@@ -82,7 +83,7 @@ export class ClusterMountedSecretProvider
this.reader = new PrivateProjectedFileReader({
rootDirectory: options.rootDirectory,
minimumBytes: 0,
maximumBytes: MAX_REMOTE_SECRET_VALUE_BYTES,
maximumBytes: MAX_REMOTE_ENVIRONMENT_BUNDLE_VALUE_BYTES,
access: 'private_material',
});
} catch (error) {
@@ -116,6 +117,7 @@ export class ClusterMountedSecretProvider
const buffers: Buffer[] = [];
try {
const values = [];
const environmentBundles = [];
let totalBytes = 0;
for (const secretRef of normalized.secretRefs) {
const bytes = await this.reader
@@ -127,6 +129,9 @@ export class ClusterMountedSecretProvider
);
});
buffers.push(bytes);
if (bytes.byteLength > MAX_REMOTE_SECRET_VALUE_BYTES) {
throw new ClusterMountedSecretProviderError('material_unavailable');
}
totalBytes += bytes.byteLength;
if (totalBytes > MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES) {
throw new ClusterMountedSecretProviderError('material_unavailable');
@@ -138,9 +143,27 @@ export class ClusterMountedSecretProvider
}),
);
}
for (const secretRef of normalized.environmentBundleRefs) {
const bytes = await this.reader
.read(clusterMountedSecretFileName(secretRef))
.catch((error) => {
throw new ClusterMountedSecretProviderError(
'material_unavailable',
{ cause: error },
);
});
buffers.push(bytes);
environmentBundles.push(
Object.freeze({
secretRef,
value: secretValue(bytes),
}),
);
}
let disposed = false;
return Object.freeze({
values: Object.freeze(values),
environmentBundles: Object.freeze(environmentBundles),
dispose() {
if (disposed) return;
disposed = true;
packages/ql3-cluster-control/src/remote-execution/remoteWorkerSecretDeliveryService.ts
+11 -3
View File
@@ -66,7 +66,9 @@ export class ClusterRemoteWorkerSecretDeliveryService {
authorized.offerId !== command.offerId ||
authorized.leaseGeneration !== command.leaseGeneration ||
authorized.leaseVersion !== command.expectedLeaseVersion ||
JSON.stringify(authorized.secretRefs) !== JSON.stringify(command.secretRefs)
JSON.stringify(authorized.secretRefs) !== JSON.stringify(command.secretRefs) ||
JSON.stringify(authorized.environmentBundleRefs) !==
JSON.stringify(command.environmentBundleRefs)
) throw new InvalidRemoteWorkerSecretDeliveryError(
'repository authority does not match command',
);
@@ -88,7 +90,8 @@ export class ClusterRemoteWorkerSecretDeliveryService {
if (
typeof resolution !== 'object' ||
Array.isArray(resolution) ||
Object.keys(resolution).some((key) => key !== 'values' && key !== 'dispose') ||
Object.keys(resolution).some((key) =>
key !== 'values' && key !== 'environmentBundles' && key !== 'dispose') ||
(resolution.dispose !== undefined &&
typeof resolution.dispose !== 'function')
) throw new InvalidRemoteWorkerSecretDeliveryError(
@@ -100,13 +103,18 @@ export class ClusterRemoteWorkerSecretDeliveryService {
offerId: authorized.offerId,
executionDigest: authorized.executionDigest,
values: resolution.values,
}, authorized.secretRefs);
environmentBundles: resolution.environmentBundles,
}, {
secretRefs: authorized.secretRefs,
environmentBundleRefs: authorized.environmentBundleRefs,
});
return Object.freeze({
runId: body.runId,
attemptId: body.attemptId,
offerId: body.offerId,
executionDigest: body.executionDigest,
values: body.values,
environmentBundles: body.environmentBundles,
...(resolution.dispose === undefined
? {}
: { dispose: resolution.dispose }),
packages/ql3-cluster-control/src/worker-ingress/workerIngressConfig.ts
+2 -2
View File
@@ -519,9 +519,9 @@ export function loadClusterWorkerIngressConfig(
maxResponseBytes: integerValue(
environment,
'QL3_WORKER_INGRESS_MAX_RESPONSE_BYTES',
64 * 1024,
256 * 1024,
1024,
64 * 1024,
256 * 1024,
),
maxInFlightRequests: integerValue(
environment,
packages/ql3-cluster-control/src/worker-ingress/workerIngressPipeline.ts
+6 -2
View File
@@ -465,7 +465,7 @@ export function createWorkerIngressAdmissionPipeline(
'schema', 'runId', 'attemptId', 'projectId', 'taskId',
'taskRevision', 'executionDigest', 'workerGeneration',
'offerId', 'leaseGeneration', 'leaseToken',
'expectedLeaseVersion', 'secretRefs',
'expectedLeaseVersion', 'secretRefs', 'environmentBundleRefs',
]);
if (value.schema !== REMOTE_SECRET_DELIVERY_SCHEMA) {
throw failure(400, 'invalid_worker_request');
@@ -486,12 +486,16 @@ export function createWorkerIngressAdmissionPipeline(
leaseToken: value.leaseToken as string,
expectedLeaseVersion: value.expectedLeaseVersion as number,
secretRefs: value.secretRefs as string[],
environmentBundleRefs: value.environmentBundleRefs as string[],
},
);
try {
const responseBody = createRemoteWorkerSecretDeliveryResponseBody(
delivered,
value.secretRefs as string[],
{
secretRefs: value.secretRefs as string[],
environmentBundleRefs: value.environmentBundleRefs as string[],
},
);
if (
responseBody.runId !== value.runId ||
packages/ql3-cluster-control/test/mountedSecretProvider.test.cjs
+40 -4
View File
@@ -30,6 +30,11 @@ const VERSIONED_SECRET_REF = createSecretRef({
name: 'certificate',
version: 3,
});
const ENVIRONMENT_BUNDLE_REF = createSecretRef({
projectId: 'project-1',
name: 'legacy-env-bundle',
version: 4,
});
function authority(secretRefs = [SECRET_REF]) {
return {
@@ -46,6 +51,7 @@ function authority(secretRefs = [SECRET_REF]) {
leaseGeneration: 1,
leaseVersion: 1,
secretRefs,
environmentBundleRefs: [],
};
}
@@ -58,10 +64,7 @@ test('maps canonical SecretRef to a stable path-free Kubernetes key', () => {
const first = clusterMountedSecretFileName(SECRET_REF);
assert.match(first, /^[0-9a-f]{64}$/);
assert.equal(clusterMountedSecretFileName(SECRET_REF), first);
assert.notEqual(
clusterMountedSecretFileName(VERSIONED_SECRET_REF),
first,
);
assert.notEqual(clusterMountedSecretFileName(VERSIONED_SECRET_REF), first);
assert.throws(
() => clusterMountedSecretFileName('not-a-secret-ref'),
ClusterMountedSecretProviderError,
@@ -82,6 +85,7 @@ test('resolves every request again and observes atomic material rotation', async
assert.deepEqual(first.values, [
{ secretRef: SECRET_REF, value: 'generation-one' },
]);
assert.deepEqual(first.environmentBundles, []);
await first.dispose();
const replacement = `${file}.replacement`;
@@ -155,3 +159,35 @@ test('fails readiness for a missing or symlinked provider root', async (t) => {
ClusterMountedSecretProviderError,
);
});
test('delivers one larger opaque environment bundle without widening normal Secrets', async (t) => {
const root = await mkdtemp(path.join(os.tmpdir(), 'ql3-mounted-bundle-'));
t.after(() => rm(root, { recursive: true, force: true }));
await chmod(root, 0o700);
const value = JSON.stringify({
schema: 'qinglong/environment-bundle@v1',
entries: [{ name: 'LEGACY_VALUE', value: 'x'.repeat(20 * 1024) }],
});
await privateFile(
path.join(root, clusterMountedSecretFileName(ENVIRONMENT_BUNDLE_REF)),
value,
);
const provider = await createClusterMountedSecretProvider({
rootDirectory: root,
});
const resolution = await provider.resolve({
...authority([]),
environmentBundleRefs: [ENVIRONMENT_BUNDLE_REF],
});
assert.deepEqual(resolution.values, []);
assert.deepEqual(resolution.environmentBundles, [
{ secretRef: ENVIRONMENT_BUNDLE_REF, value },
]);
await resolution.dispose();
await privateFile(
path.join(root, clusterMountedSecretFileName(SECRET_REF)),
'x'.repeat(16 * 1024 + 1),
);
await assert.rejects(provider.resolve(authority()), /material_unavailable/);
});
packages/ql3-cluster-control/test/remoteWorkerSecretDeliveryService.test.cjs
+5 -1
View File
@@ -21,7 +21,7 @@ function command() {
taskId: 'task-1', taskRevision: 'revision-1', executionDigest: DIGEST,
offerId: 'offer-1', leaseGeneration: 3,
leaseToken: 'worker_generated_lease_capability_0000000000000001',
expectedLeaseVersion: 4, secretRefs: [SECRET_REF],
expectedLeaseVersion: 4, secretRefs: [SECRET_REF], environmentBundleRefs: [],
};
}
@@ -47,6 +47,7 @@ test('resolves plaintext only after repository authority succeeds', async () =>
assert.equal('leaseToken' in input, false);
return {
values: [{ secretRef: SECRET_REF, value: 'resolved-value' }],
environmentBundles: [],
dispose() { events.push('dispose'); },
};
},
@@ -56,6 +57,7 @@ test('resolves plaintext only after repository authority succeeds', async () =>
assert.deepEqual(result.values, [
{ secretRef: SECRET_REF, value: 'resolved-value' },
]);
assert.deepEqual(result.environmentBundles, []);
assert.deepEqual(events, ['authorize', 'resolve']);
await result.dispose();
assert.deepEqual(events, ['authorize', 'resolve', 'dispose']);
@@ -101,6 +103,7 @@ test('disposes malformed provider output and converts it to unavailable', async
async resolve() {
return {
values: [{ secretRef: SECRET_REF, value: 'x'.repeat(17 * 1024) }],
environmentBundles: [],
dispose() { disposed += 1; },
};
},
@@ -120,6 +123,7 @@ test('rejects extensible provider output and still invokes valid cleanup', async
async resolve() {
return {
values: [{ secretRef: SECRET_REF, value: 'resolved-value' }],
environmentBundles: [],
dispose() { disposed += 1; },
diagnostic: 'must-not-cross-boundary',
};
packages/ql3-cluster-control/test/workerIngressConfig.test.cjs
+1 -1
View File
@@ -93,7 +93,7 @@ test('builds exact bounded Worker ingress and least-privilege Pool config', asyn
host: '127.0.0.1',
port: 5901,
maxBodyBytes: 65_536,
maxResponseBytes: 65_536,
maxResponseBytes: 262_144,
maxInFlightRequests: 32,
authenticationRateWindowMs: 60_000,
authenticationRatePerPeer: 20,
packages/ql3-cluster-control/test/workerIngressPipeline.test.cjs
+10 -4
View File
@@ -592,6 +592,7 @@ test('binds one Secret batch to path identity and never echoes capabilities', as
offerId: command.offerId,
executionDigest: command.executionDigest,
values: [{ secretRef, value: 'resolved-value' }],
environmentBundles: [],
dispose() { disposed += 1; },
};
},
@@ -599,18 +600,20 @@ test('binds one Secret batch to path identity and never echoes capabilities', as
});
const leaseToken = 'worker_generated_lease_capability_0000000000000001';
const body = {
schema: 'qinglong/remote-secret-delivery@v1',
schema: 'qinglong/remote-secret-delivery@v2',
runId: 'run-1', attemptId: 'attempt-1', projectId: 'project-1',
taskId: 'task-1', taskRevision: 'revision-1', executionDigest,
workerGeneration: 2, offerId: 'offer-1', leaseGeneration: 3,
leaseToken, expectedLeaseVersion: 4, secretRefs: [secretRef],
environmentBundleRefs: [],
};
const result = await (await pipeline.prepare(metadata('secrets'))).handle(body);
assert.equal(result.statusCode, 200);
assert.equal(result.body.schema, 'qinglong/remote-secret-delivery@v1');
assert.equal(result.body.schema, 'qinglong/remote-secret-delivery@v2');
assert.deepEqual(result.body.values, [
{ secretRef, value: 'resolved-value' },
]);
assert.deepEqual(result.body.environmentBundles, []);
assert.equal(JSON.stringify(result.body).includes(leaseToken), false);
const { schema: _schema, ...commandBody } = body;
assert.deepEqual(observed, {
@@ -634,13 +637,14 @@ test('maps stale Secret delivery authority to conflict before any response', asy
});
await assert.rejects(
(await pipeline.prepare(metadata('secrets'))).handle({
schema: 'qinglong/remote-secret-delivery@v1',
schema: 'qinglong/remote-secret-delivery@v2',
runId: 'run-1', attemptId: 'attempt-1', projectId: 'project-1',
taskId: 'task-1', taskRevision: 'revision-1',
executionDigest: 'c'.repeat(64), workerGeneration: 2,
offerId: 'offer-1', leaseGeneration: 3,
leaseToken: 'worker_generated_lease_capability_0000000000000001',
expectedLeaseVersion: 4, secretRefs: [secretRef],
environmentBundleRefs: [],
}),
(error) =>
error.statusCode === 409 && error.code === 'worker_secret_delivery_fenced',
@@ -656,19 +660,21 @@ test('rejects a Secret service response whose authority drifts', async () => {
runId: 'run-other', attemptId: 'attempt-1', offerId: 'offer-1',
executionDigest: 'c'.repeat(64),
values: [{ secretRef, value: 'must-not-escape' }],
environmentBundles: [],
};
},
},
});
await assert.rejects(
(await pipeline.prepare(metadata('secrets'))).handle({
schema: 'qinglong/remote-secret-delivery@v1',
schema: 'qinglong/remote-secret-delivery@v2',
runId: 'run-1', attemptId: 'attempt-1', projectId: 'project-1',
taskId: 'task-1', taskRevision: 'revision-1',
executionDigest: 'c'.repeat(64), workerGeneration: 2,
offerId: 'offer-1', leaseGeneration: 3,
leaseToken: 'worker_generated_lease_capability_0000000000000001',
expectedLeaseVersion: 4, secretRefs: [secretRef],
environmentBundleRefs: [],
}),
(error) =>
error.statusCode === 503 && error.code === 'worker_ingress_unavailable',