open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-23 03:18:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): add opaque cluster environment bundle delivery

Browse Source
This commit is contained in:
whyour
2026-08-24 19:31:03 +08:00
parent cf2c0ec7b3
commit 4abf125ce9
36 changed files with 1682 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-worker-runtime/src/remote-execution/executionContextMaterializer.ts
+135 -48
View File
@@ -1,8 +1,10 @@
// Remote Execution owns bounded Secret and Artifact context materialization.
import {
MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES,
MAX_LOCAL_DISPATCH_ENVIRONMENT_ENTRIES,
MAX_LOCAL_DISPATCH_SECRET_REFS,
} from '@qinglong/runtime-core/local-dispatch';
import { parseEnvironmentBundle } from '@qinglong/runtime-core/environment-bundle';
import type { ClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import { createClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import { assertRunDispatchId } from '@qinglong/runtime-core/run-dispatch-lease';
@@ -17,20 +19,27 @@ export interface WorkerRemoteSecretResolution {
secretRef: string;
value: string;
}>[];
readonly environmentBundles: readonly Readonly<{
secretRef: string;
value: string;
}>[];
readonly dispose?: () => Promise<void> | void;
}
export interface WorkerRemoteSecretEnvironmentProvider {
resolve(request: Readonly<{
projectId: string;
taskId: string;
taskRevision: string;
runId: string;
attemptId: string;
offerId: string;
executionDigest: string;
secretRefs: readonly string[];
}>): Promise<WorkerRemoteSecretResolution | undefined>;
resolve(
request: Readonly<{
projectId: string;
taskId: string;
taskRevision: string;
runId: string;
attemptId: string;
offerId: string;
executionDigest: string;
secretRefs: readonly string[];
environmentBundleRefs: readonly string[];
}>,
): Promise<WorkerRemoteSecretResolution | undefined>;
}
export interface WorkerRemoteLogArtifactPreparation {
@@ -42,12 +51,14 @@ export interface WorkerRemoteLogArtifactPreparation {
}
export interface WorkerRemoteLogArtifactAllocator {
prepare(request: Readonly<{
projectId: string;
runId: string;
attemptId: string;
offerId: string;
}>): Promise<WorkerRemoteLogArtifactPreparation | undefined>;
prepare(
request: Readonly<{
projectId: string;
runId: string;
attemptId: string;
offerId: string;
}>,
): Promise<WorkerRemoteLogArtifactPreparation | undefined>;
}
export interface BoundedWorkerRemoteExecutionContextMaterializerOptions {
@@ -86,11 +97,14 @@ function environmentValue(value: unknown): string {
async function disposeQuietly(
operation: (() => Promise<void> | void) | undefined,
): Promise<void> {
await Promise.resolve().then(() => operation?.()).catch(() => undefined);
await Promise.resolve()
.then(() => operation?.())
.catch(() => undefined);
}
export class BoundedWorkerRemoteExecutionContextMaterializer
implements WorkerRemoteExecutionContextMaterializer {
implements WorkerRemoteExecutionContextMaterializer
{
private readonly artifacts: WorkerRemoteLogArtifactAllocator;
private readonly secrets?: WorkerRemoteSecretEnvironmentProvider;
@@ -109,9 +123,11 @@ export class BoundedWorkerRemoteExecutionContextMaterializer
this.secrets = options.secrets;
}
async prepare(input: Readonly<{
offer: ClusterRemoteExecutionOffer;
}>): Promise<MaterializedWorkerRemoteExecutionContext> {
async prepare(
input: Readonly<{
offer: ClusterRemoteExecutionOffer;
}>,
): Promise<MaterializedWorkerRemoteExecutionContext> {
let offer: ClusterRemoteExecutionOffer;
try {
offer = createClusterRemoteExecutionOffer(input?.offer);
@@ -122,9 +138,17 @@ export class BoundedWorkerRemoteExecutionContextMaterializer
}
const bindings = offer.executionRevision.environment;
const secretRefs = Object.freeze([
...new Set(bindings.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [])),
...new Set(
bindings.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [],
),
),
]);
const environmentBundleRefs = Object.freeze(
offer.executionRevision.environmentBundleRef === undefined
? []
: [offer.executionRevision.environmentBundleRef],
);
if (secretRefs.length > MAX_LOCAL_DISPATCH_SECRET_REFS) {
throw new WorkerRemoteExecutionMaterializationError(
'environment_budget_exceeded',
@@ -132,23 +156,26 @@ export class BoundedWorkerRemoteExecutionContextMaterializer
}
let secretResolution: WorkerRemoteSecretResolution | undefined;
const secretByRef = new Map<string, string>();
if (secretRefs.length > 0) {
if (secretRefs.length > 0 || environmentBundleRefs.length > 0) {
if (!this.secrets) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_unavailable',
);
}
try {
secretResolution = await this.secrets.resolve(Object.freeze({
projectId: offer.candidate.projectId,
taskId: offer.candidate.taskId,
taskRevision: offer.candidate.taskRevision,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
executionDigest: offer.executionDigest,
secretRefs,
}));
secretResolution = await this.secrets.resolve(
Object.freeze({
projectId: offer.candidate.projectId,
taskId: offer.candidate.taskId,
taskRevision: offer.candidate.taskRevision,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
executionDigest: offer.executionDigest,
secretRefs,
environmentBundleRefs,
}),
);
} catch {
throw new WorkerRemoteExecutionMaterializationError(
'secret_unavailable',
@@ -160,10 +187,17 @@ export class BoundedWorkerRemoteExecutionContextMaterializer
);
}
if (
Object.keys(secretResolution).some((key) =>
key !== 'values' && key !== 'dispose') ||
Object.keys(secretResolution).some(
(key) =>
key !== 'values' &&
key !== 'environmentBundles' &&
key !== 'dispose',
) ||
!Array.isArray(secretResolution.values) ||
secretResolution.values.length !== secretRefs.length ||
!Array.isArray(secretResolution.environmentBundles) ||
secretResolution.environmentBundles.length !==
environmentBundleRefs.length ||
(secretResolution.dispose !== undefined &&
typeof secretResolution.dispose !== 'function')
) {
@@ -198,36 +232,89 @@ export class BoundedWorkerRemoteExecutionContextMaterializer
let environmentBytes = 0;
let environment: MaterializedWorkerRemoteExecutionContext['environment'];
try {
environment = Object.freeze(bindings.map((binding) => {
const value = binding.kind === 'public'
? binding.value
: secretByRef.get(binding.secretRef);
const names = new Set<string>();
const materialized = bindings.map((binding) => {
const value =
binding.kind === 'public'
? binding.value
: secretByRef.get(binding.secretRef);
if (value === undefined) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
environmentBytes += Buffer.byteLength(binding.name, 'utf8') +
environmentBytes +=
Buffer.byteLength(binding.name, 'utf8') +
Buffer.byteLength(value, 'utf8');
names.add(binding.name);
if (environmentBytes > MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES) {
throw new WorkerRemoteExecutionMaterializationError(
'environment_budget_exceeded',
);
}
return Object.freeze({ name: binding.name, value });
}));
});
for (const entry of secretResolution?.environmentBundles ?? []) {
if (
!entry ||
typeof entry !== 'object' ||
Object.keys(entry).length !== 2 ||
!Object.hasOwn(entry, 'secretRef') ||
!Object.hasOwn(entry, 'value') ||
typeof entry.secretRef !== 'string' ||
!environmentBundleRefs.includes(entry.secretRef) ||
typeof entry.value !== 'string'
) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
let bundle;
try {
bundle = parseEnvironmentBundle(entry.value);
} catch {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
for (const binding of bundle.entries) {
if (names.has(binding.name)) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
names.add(binding.name);
environmentBytes +=
Buffer.byteLength(binding.name, 'utf8') +
Buffer.byteLength(binding.value, 'utf8');
if (
materialized.length >= MAX_LOCAL_DISPATCH_ENVIRONMENT_ENTRIES ||
environmentBytes > MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES
) {
throw new WorkerRemoteExecutionMaterializationError(
'environment_budget_exceeded',
);
}
materialized.push(
Object.freeze({ name: binding.name, value: binding.value }),
);
}
}
environment = Object.freeze(materialized);
} catch (error) {
await disposeQuietly(secretResolution?.dispose);
throw error;
}
let artifact: WorkerRemoteLogArtifactPreparation | undefined;
try {
artifact = await this.artifacts.prepare(Object.freeze({
projectId: offer.candidate.projectId,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
}));
artifact = await this.artifacts.prepare(
Object.freeze({
projectId: offer.candidate.projectId,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
}),
);
} catch {
await disposeQuietly(secretResolution?.dispose);
throw new WorkerRemoteExecutionMaterializationError(
packages/ql3-worker-runtime/src/remote-execution/transport/remoteSecretHttpsProvider.ts
+37 -11
View File
@@ -36,7 +36,8 @@ export interface WorkerRemoteSecretHttpsProviderOptions {
}
export class WorkerRemoteSecretHttpsProvider
implements WorkerRemoteSecretEnvironmentProvider {
implements WorkerRemoteSecretEnvironmentProvider
{
private readonly client: Pick<WorkerIngressHttpsClient, 'postJson'>;
private readonly inbox: Pick<WorkerRemoteExecutionInbox, 'readOffer'>;
@@ -45,13 +46,15 @@ export class WorkerRemoteSecretHttpsProvider
!options ||
typeof options.client?.postJson !== 'function' ||
typeof options.inbox?.readOffer !== 'function'
) throw new WorkerRemoteSecretHttpsProviderError('invalid_configuration');
)
throw new WorkerRemoteSecretHttpsProviderError('invalid_configuration');
this.client = options.client;
this.inbox = options.inbox;
}
async resolve(request: Parameters<WorkerRemoteSecretEnvironmentProvider['resolve']>[0])
: Promise<WorkerRemoteSecretResolution | undefined> {
async resolve(
request: Parameters<WorkerRemoteSecretEnvironmentProvider['resolve']>[0],
): Promise<WorkerRemoteSecretResolution | undefined> {
let record;
try {
record = await this.inbox.readOffer(request.offerId);
@@ -68,9 +71,17 @@ export class WorkerRemoteSecretHttpsProvider
throw new WorkerRemoteSecretHttpsProviderError('authority_mismatch');
}
const expectedRefs = Object.freeze([
...new Set(offer.executionRevision.environment.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [])),
...new Set(
offer.executionRevision.environment.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [],
),
),
]);
const expectedEnvironmentBundleRefs = Object.freeze(
offer.executionRevision.environmentBundleRef === undefined
? []
: [offer.executionRevision.environmentBundleRef],
);
if (
offer.offerId !== request.offerId ||
offer.executionDigest !== request.executionDigest ||
@@ -79,10 +90,14 @@ export class WorkerRemoteSecretHttpsProvider
offer.candidate.taskRevision !== request.taskRevision ||
offer.candidate.runId !== request.runId ||
offer.candidate.attemptId !== request.attemptId ||
JSON.stringify(expectedRefs) !== JSON.stringify(request.secretRefs)
) throw new WorkerRemoteSecretHttpsProviderError('authority_mismatch');
JSON.stringify(expectedRefs) !== JSON.stringify(request.secretRefs) ||
JSON.stringify(expectedEnvironmentBundleRefs) !==
JSON.stringify(request.environmentBundleRefs)
)
throw new WorkerRemoteSecretHttpsProviderError('authority_mismatch');
const path = `/api/v3/worker-ingress/workers/${offer.worker.workerId}` +
const path =
`/api/v3/worker-ingress/workers/${offer.worker.workerId}` +
`/sessions/${offer.worker.sessionId}/secrets`;
const body = createRemoteWorkerSecretDeliveryRequestBody({
workerId: offer.worker.workerId,
@@ -99,6 +114,7 @@ export class WorkerRemoteSecretHttpsProvider
leaseToken: offer.leaseToken,
expectedLeaseVersion: offer.lease.version,
secretRefs: expectedRefs,
environmentBundleRefs: expectedEnvironmentBundleRefs,
});
let serialized: Uint8Array;
try {
@@ -119,11 +135,21 @@ export class WorkerRemoteSecretHttpsProvider
offerId: offer.offerId,
executionDigest: offer.executionDigest,
secretRefs: expectedRefs,
environmentBundleRefs: expectedEnvironmentBundleRefs,
});
const values = Object.freeze(delivered.values.map((entry) =>
Object.freeze({ secretRef: entry.secretRef, value: entry.value })));
const values = Object.freeze(
delivered.values.map((entry) =>
Object.freeze({ secretRef: entry.secretRef, value: entry.value }),
),
);
const environmentBundles = Object.freeze(
delivered.environmentBundles.map((entry) =>
Object.freeze({ secretRef: entry.secretRef, value: entry.value }),
),
);
return Object.freeze({
values,
environmentBundles,
dispose() {
// JavaScript strings cannot be zeroized. Drop all retained references;
// the transport bytes were already scrubbed by the parser.
packages/ql3-worker-runtime/src/remote-execution/transport/workerIngressHttpsClient.ts
+3 -1
View File
@@ -17,7 +17,9 @@ const MAX_TLS_MATERIAL_BYTES = 1024 * 1024;
const MAX_REQUEST_BYTES = 4096;
const HARD_MAX_REQUEST_BYTES = 64 * 1024;
const HARD_MAX_STREAM_REQUEST_BYTES = 64 * 1024 * 1024 + 4 * 1024 + 4;
const MAX_RESPONSE_BYTES = 128 * 1024;
// JSON routes retain their own smaller requested caps. The shared ceiling must
// also admit one bounded environment bundle response.
const MAX_RESPONSE_BYTES = 256 * 1024;
const CREDENTIAL_POOL_KEY = Symbol('qinglong.worker-ingress-credential-pool-key');
export const WORKER_INGRESS_ARTIFACT_CONTENT_TYPE =
packages/ql3-worker-runtime/test/executionContextMaterializer.test.cjs
+129 -22
View File
@@ -11,9 +11,11 @@ const {
const {
digestRunDispatchLeaseToken,
} = require('@qinglong/runtime-core/run-dispatch-lease');
const { createSecretRef } = require('@qinglong/runtime-core/secret-reference');
const {
createSecretRef,
} = require('@qinglong/runtime-core/secret-reference');
ENVIRONMENT_BUNDLE_SCHEMA,
serializeEnvironmentBundle,
} = require('@qinglong/runtime-core/environment-bundle');
const {
BoundedWorkerRemoteExecutionContextMaterializer,
} = require('../dist/remote-execution/remoteOfferDeliveryEntrypoint');
@@ -27,7 +29,7 @@ function secret(name) {
return createSecretRef({ projectId: 'project-1', name });
}
function offer(environment) {
function offer(environment, environmentBundleRef) {
const executionRevision = createClusterTaskExecutionRevision({
projectId: 'project-1',
taskId: 'task-1',
@@ -38,6 +40,7 @@ function offer(environment) {
planSchema: 'qinglong/command-execution@v1',
command: { kind: 'argv', file: '/bin/true', args: [] },
environment,
...(environmentBundleRef === undefined ? {} : { environmentBundleRef }),
createdAtMs: 1,
});
return createClusterRemoteExecutionOffer({
@@ -100,7 +103,10 @@ test('resolves deduplicated Secrets before allocating one Attempt log', async ()
secretRequest = request;
return {
values: [{ secretRef, value: 'resolved-value' }],
dispose() { events.push('dispose-secrets'); },
environmentBundles: [],
dispose() {
events.push('dispose-secrets');
},
};
},
},
@@ -110,8 +116,13 @@ test('resolves deduplicated Secrets before allocating one Attempt log', async ()
artifactRequest = request;
return {
logArtifactId: 'remote-log-1',
takeOutput() { events.push('take-output'); return output; },
release() { events.push('release-artifact'); },
takeOutput() {
events.push('take-output');
return output;
},
release() {
events.push('release-artifact');
},
};
},
},
@@ -129,6 +140,7 @@ test('resolves deduplicated Secrets before allocating one Attempt log', async ()
offerId: 'offer-materializer-1',
executionDigest: acceptedOffer.executionDigest,
secretRefs: [secretRef],
environmentBundleRefs: [],
});
assert.deepEqual(artifactRequest, {
projectId: 'project-1',
@@ -136,7 +148,10 @@ test('resolves deduplicated Secrets before allocating one Attempt log', async ()
attemptId: 'attempt-1',
offerId: 'offer-materializer-1',
});
assert.equal(JSON.stringify([secretRequest, artifactRequest]).includes(LEASE_TOKEN), false);
assert.equal(
JSON.stringify([secretRequest, artifactRequest]).includes(LEASE_TOKEN),
false,
);
assert.deepEqual(context.environment, [
{ name: 'PUBLIC', value: 'visible' },
{ name: 'SECRET_A', value: 'resolved-value' },
@@ -149,18 +164,26 @@ test('resolves deduplicated Secrets before allocating one Attempt log', async ()
await context.dispose();
await context.dispose();
assert.deepEqual(events.slice(2).sort(), [
'dispose-secrets', 'release-artifact', 'take-output',
'dispose-secrets',
'release-artifact',
'take-output',
]);
});
test('fails before Artifact allocation when Secret authority is unavailable', async () => {
let artifacts = 0;
const materializer = new BoundedWorkerRemoteExecutionContextMaterializer({
artifacts: { async prepare() { artifacts += 1; } },
artifacts: {
async prepare() {
artifacts += 1;
},
},
});
await assert.rejects(
materializer.prepare({
offer: offer([{ name: 'SECRET', kind: 'secret', secretRef: secret('one') }]),
offer: offer([
{ name: 'SECRET', kind: 'secret', secretRef: secret('one') },
]),
}),
/secret_unavailable/,
);
@@ -179,11 +202,18 @@ test('disposes malformed Secret and Artifact responses without exposing values',
{ secretRef, value: 'first' },
{ secretRef, value: 'duplicate' },
],
dispose() { disposedSecrets += 1; },
environmentBundles: [],
dispose() {
disposedSecrets += 1;
},
};
},
},
artifacts: { async prepare() { throw new Error('must not allocate'); } },
artifacts: {
async prepare() {
throw new Error('must not allocate');
},
},
});
await assert.rejects(
malformedSecrets.prepare({
@@ -196,16 +226,20 @@ test('disposes malformed Secret and Artifact responses without exposing values',
);
assert.equal(disposedSecrets, 1);
const malformedArtifact = new BoundedWorkerRemoteExecutionContextMaterializer({
artifacts: {
async prepare() {
return {
logArtifactId: 'x'.repeat(37),
release() { releasedArtifact += 1; },
};
const malformedArtifact = new BoundedWorkerRemoteExecutionContextMaterializer(
{
artifacts: {
async prepare() {
return {
logArtifactId: 'x'.repeat(37),
release() {
releasedArtifact += 1;
},
};
},
},
},
});
);
await assert.rejects(
malformedArtifact.prepare({
offer: offer([{ name: 'PUBLIC', kind: 'public', value: 'visible' }]),
@@ -231,11 +265,18 @@ test('enforces the resolved environment byte budget before Artifact allocation',
secretRef,
value: 'x'.repeat(16 * 1024),
})),
dispose() { disposed += 1; },
environmentBundles: [],
dispose() {
disposed += 1;
},
};
},
},
artifacts: { async prepare() { artifacts += 1; } },
artifacts: {
async prepare() {
artifacts += 1;
},
},
});
await assert.rejects(
materializer.prepare({ offer: offer(bindings) }),
@@ -244,3 +285,69 @@ test('enforces the resolved environment byte budget before Artifact allocation',
assert.equal(disposed, 1);
assert.equal(artifacts, 0);
});
test('expands one opaque bundle in memory and rejects name collisions', async () => {
const environmentBundleRef = createSecretRef({
projectId: 'project-1',
name: 'legacy-env-bundle',
version: 2,
});
const bundle = serializeEnvironmentBundle({
schema: ENVIRONMENT_BUNDLE_SCHEMA,
entries: [
{ name: 'LEGACY_TOKEN', value: 'private' },
{ name: 'LEGACY_MODE', value: 'compat' },
],
});
const materializer = new BoundedWorkerRemoteExecutionContextMaterializer({
secrets: {
async resolve(request) {
assert.deepEqual(request.secretRefs, []);
assert.deepEqual(request.environmentBundleRefs, [environmentBundleRef]);
return {
values: [],
environmentBundles: [
{ secretRef: environmentBundleRef, value: bundle },
],
};
},
},
artifacts: {
async prepare() {
return {
logArtifactId: 'remote-log-bundle',
takeOutput() {
return {
logArtifactId: 'remote-log-bundle',
async write() {},
async close() {},
};
},
async release() {},
};
},
},
});
const context = await materializer.prepare({
offer: offer(
[{ name: 'PUBLIC', kind: 'public', value: 'visible' }],
environmentBundleRef,
),
});
assert.deepEqual(context.environment, [
{ name: 'PUBLIC', value: 'visible' },
{ name: 'LEGACY_MODE', value: 'compat' },
{ name: 'LEGACY_TOKEN', value: 'private' },
]);
await context.dispose();
await assert.rejects(
materializer.prepare({
offer: offer(
[{ name: 'LEGACY_MODE', kind: 'public', value: 'current' }],
environmentBundleRef,
),
}),
/secret_response_invalid/,
);
});
packages/ql3-worker-runtime/test/remoteSecretHttpsProvider.test.cjs
+86 -31
View File
@@ -24,31 +24,52 @@ const SECRET_REF = createSecretRef({ projectId: 'project-1', name: 'token' });
function acceptedOffer() {
const executionRevision = createClusterTaskExecutionRevision({
projectId: 'project-1', taskId: 'task-1', taskRevision: TASK_REVISION,
sourceRevision: 1, sourceContentDigest: SOURCE_DIGEST,
executorType: 'remote_worker', planSchema: 'qinglong/command-execution@v1',
projectId: 'project-1',
taskId: 'task-1',
taskRevision: TASK_REVISION,
sourceRevision: 1,
sourceContentDigest: SOURCE_DIGEST,
executorType: 'remote_worker',
planSchema: 'qinglong/command-execution@v1',
command: { kind: 'argv', file: '/bin/true', args: [] },
environment: [{ name: 'TOKEN', kind: 'secret', secretRef: SECRET_REF }],
createdAtMs: 1,
});
return createClusterRemoteExecutionOffer({
offerId: 'offer-1', deliveryKind: 'new_claim',
offerId: 'offer-1',
deliveryKind: 'new_claim',
executionDigest: executionRevision.contentDigest,
candidate: {
runId: 'run-1', attemptId: 'attempt-1', projectId: 'project-1',
taskId: 'task-1', taskRevision: TASK_REVISION, priority: 1,
queuedAtMs: 10, attemptCreatedAtMs: 11, attemptNumber: 1,
runId: 'run-1',
attemptId: 'attempt-1',
projectId: 'project-1',
taskId: 'task-1',
taskRevision: TASK_REVISION,
priority: 1,
queuedAtMs: 10,
attemptCreatedAtMs: 11,
attemptNumber: 1,
executorType: 'remote_worker',
},
worker: { workerId: 'edge-1', sessionId: SESSION_ID, generation: 2 },
lease: {
attemptId: 'attempt-1', runId: 'run-1', status: 'leased', version: 4,
leaseGeneration: 3, workerId: 'edge-1', workerSessionId: SESSION_ID,
workerGeneration: 2, leaseTokenDigest: digestRunDispatchLeaseToken(LEASE_TOKEN),
acquiredAtMs: 20, renewedAtMs: 20, expiresAtMs: 30_020,
attemptId: 'attempt-1',
runId: 'run-1',
status: 'leased',
version: 4,
leaseGeneration: 3,
workerId: 'edge-1',
workerSessionId: SESSION_ID,
workerGeneration: 2,
leaseTokenDigest: digestRunDispatchLeaseToken(LEASE_TOKEN),
acquiredAtMs: 20,
renewedAtMs: 20,
expiresAtMs: 30_020,
updatedAtMs: 20,
},
leaseToken: LEASE_TOKEN, executionRevision, placementScore: 0,
leaseToken: LEASE_TOKEN,
executionRevision,
placementScore: 0,
});
}
@@ -62,6 +83,7 @@ function requestFor(offer) {
offerId: offer.offerId,
executionDigest: offer.executionDigest,
secretRefs: [SECRET_REF],
environmentBundleRefs: [],
};
}
@@ -78,12 +100,17 @@ test('rehydrates lease authority from inbox and delivers one exact Secret batch'
client: {
async postJson(request) {
transport = request;
return Buffer.from(JSON.stringify({
schema: 'qinglong/remote-secret-delivery@v1',
runId: 'run-1', attemptId: 'attempt-1', offerId: 'offer-1',
executionDigest: offer.executionDigest,
values: [{ secretRef: SECRET_REF, value: 'resolved-value' }],
}));
return Buffer.from(
JSON.stringify({
schema: 'qinglong/remote-secret-delivery@v2',
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
executionDigest: offer.executionDigest,
values: [{ secretRef: SECRET_REF, value: 'resolved-value' }],
environmentBundles: [],
}),
);
},
},
});
@@ -91,7 +118,11 @@ test('rehydrates lease authority from inbox and delivers one exact Secret batch'
assert.deepEqual(resolution.values, [
{ secretRef: SECRET_REF, value: 'resolved-value' },
]);
assert.equal(transport.path.endsWith(`/sessions/${SESSION_ID}/secrets`), true);
assert.deepEqual(resolution.environmentBundles, []);
assert.equal(
transport.path.endsWith(`/sessions/${SESSION_ID}/secrets`),
true,
);
assert.equal(transport.body.leaseToken, LEASE_TOKEN);
assert.equal(transport.maximumRequestBytes, 64 * 1024);
assert.equal(JSON.stringify(requestFor(offer)).includes(LEASE_TOKEN), false);
@@ -102,9 +133,15 @@ test('rejects a stale inbox identity before sending the capability', async () =>
let calls = 0;
const provider = new WorkerRemoteSecretHttpsProvider({
inbox: {
async readOffer() { return { state: 'starting_acknowledged', offer }; },
async readOffer() {
return { state: 'starting_acknowledged', offer };
},
},
client: {
async postJson() {
calls += 1;
},
},
client: { async postJson() { calls += 1; } },
});
await assert.rejects(
provider.resolve({ ...requestFor(offer), executionDigest: 'b'.repeat(64) }),
@@ -117,16 +154,23 @@ test('rejects response authority drift and does not return plaintext', async ()
const offer = acceptedOffer();
const provider = new WorkerRemoteSecretHttpsProvider({
inbox: {
async readOffer() { return { state: 'starting_acknowledged', offer }; },
async readOffer() {
return { state: 'starting_acknowledged', offer };
},
},
client: {
async postJson() {
return Buffer.from(JSON.stringify({
schema: 'qinglong/remote-secret-delivery@v1',
runId: 'run-other', attemptId: 'attempt-1', offerId: 'offer-1',
executionDigest: offer.executionDigest,
values: [{ secretRef: SECRET_REF, value: 'must-not-escape' }],
}));
return Buffer.from(
JSON.stringify({
schema: 'qinglong/remote-secret-delivery@v2',
runId: 'run-other',
attemptId: 'attempt-1',
offerId: 'offer-1',
executionDigest: offer.executionDigest,
values: [{ secretRef: SECRET_REF, value: 'must-not-escape' }],
environmentBundles: [],
}),
);
},
},
});
@@ -138,10 +182,21 @@ test('does not fetch Secrets before starting ACK or after the launch barrier', a
for (const state of ['accepted', 'launching']) {
let calls = 0;
const provider = new WorkerRemoteSecretHttpsProvider({
inbox: { async readOffer() { return { state, offer }; } },
client: { async postJson() { calls += 1; } },
inbox: {
async readOffer() {
return { state, offer };
},
},
client: {
async postJson() {
calls += 1;
},
},
});
await assert.rejects(provider.resolve(requestFor(offer)), /offer_unavailable/);
await assert.rejects(
provider.resolve(requestFor(offer)),
/offer_unavailable/,
);
assert.equal(calls, 0);
}
});
packages/ql3-worker-runtime/test/workerIngressHttpsClient.test.cjs
+10 -1
View File
@@ -267,11 +267,20 @@ test('disposes provider-owned credential material on success and rejection', asy
rejected.postJson({
path: COMPLETION_PATH,
body: {},
maximumResponseBytes: 1024,
maximumResponseBytes: 256 * 1024,
}),
/credentials_unavailable/,
);
assert.equal(rejectedDisposals, 1);
await assert.rejects(
rejected.postJson({
path: COMPLETION_PATH,
body: {},
maximumResponseBytes: 256 * 1024 + 1,
}),
/request_rejected/,
);
assert.equal(rejectedDisposals, 1);
} finally {
rejected.close();
}