open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-22 19:29:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): add opaque cluster environment bundle delivery

Browse Source
This commit is contained in:
whyour
2026-08-24 19:31:03 +08:00
parent cf2c0ec7b3
commit 4abf125ce9
36 changed files with 1682 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-cluster-control/src/remote-execution/mountedSecretProvider.ts
+24 -1
View File
@@ -1,6 +1,7 @@
// Remote Execution owns mounted Secret resolution for authenticated delivery.
import {
MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES,
MAX_REMOTE_ENVIRONMENT_BUNDLE_VALUE_BYTES,
MAX_REMOTE_SECRET_VALUE_BYTES,
normalizeRemoteWorkerSecretDeliveryAuthority,
type RemoteWorkerSecretDeliveryAuthority,
@@ -82,7 +83,7 @@ export class ClusterMountedSecretProvider
this.reader = new PrivateProjectedFileReader({
rootDirectory: options.rootDirectory,
minimumBytes: 0,
maximumBytes: MAX_REMOTE_SECRET_VALUE_BYTES,
maximumBytes: MAX_REMOTE_ENVIRONMENT_BUNDLE_VALUE_BYTES,
access: 'private_material',
});
} catch (error) {
@@ -116,6 +117,7 @@ export class ClusterMountedSecretProvider
const buffers: Buffer[] = [];
try {
const values = [];
const environmentBundles = [];
let totalBytes = 0;
for (const secretRef of normalized.secretRefs) {
const bytes = await this.reader
@@ -127,6 +129,9 @@ export class ClusterMountedSecretProvider
);
});
buffers.push(bytes);
if (bytes.byteLength > MAX_REMOTE_SECRET_VALUE_BYTES) {
throw new ClusterMountedSecretProviderError('material_unavailable');
}
totalBytes += bytes.byteLength;
if (totalBytes > MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES) {
throw new ClusterMountedSecretProviderError('material_unavailable');
@@ -138,9 +143,27 @@ export class ClusterMountedSecretProvider
}),
);
}
for (const secretRef of normalized.environmentBundleRefs) {
const bytes = await this.reader
.read(clusterMountedSecretFileName(secretRef))
.catch((error) => {
throw new ClusterMountedSecretProviderError(
'material_unavailable',
{ cause: error },
);
});
buffers.push(bytes);
environmentBundles.push(
Object.freeze({
secretRef,
value: secretValue(bytes),
}),
);
}
let disposed = false;
return Object.freeze({
values: Object.freeze(values),
environmentBundles: Object.freeze(environmentBundles),
dispose() {
if (disposed) return;
disposed = true;
packages/ql3-cluster-control/src/remote-execution/remoteWorkerSecretDeliveryService.ts
+11 -3
View File
@@ -66,7 +66,9 @@ export class ClusterRemoteWorkerSecretDeliveryService {
authorized.offerId !== command.offerId ||
authorized.leaseGeneration !== command.leaseGeneration ||
authorized.leaseVersion !== command.expectedLeaseVersion ||
JSON.stringify(authorized.secretRefs) !== JSON.stringify(command.secretRefs)
JSON.stringify(authorized.secretRefs) !== JSON.stringify(command.secretRefs) ||
JSON.stringify(authorized.environmentBundleRefs) !==
JSON.stringify(command.environmentBundleRefs)
) throw new InvalidRemoteWorkerSecretDeliveryError(
'repository authority does not match command',
);
@@ -88,7 +90,8 @@ export class ClusterRemoteWorkerSecretDeliveryService {
if (
typeof resolution !== 'object' ||
Array.isArray(resolution) ||
Object.keys(resolution).some((key) => key !== 'values' && key !== 'dispose') ||
Object.keys(resolution).some((key) =>
key !== 'values' && key !== 'environmentBundles' && key !== 'dispose') ||
(resolution.dispose !== undefined &&
typeof resolution.dispose !== 'function')
) throw new InvalidRemoteWorkerSecretDeliveryError(
@@ -100,13 +103,18 @@ export class ClusterRemoteWorkerSecretDeliveryService {
offerId: authorized.offerId,
executionDigest: authorized.executionDigest,
values: resolution.values,
}, authorized.secretRefs);
environmentBundles: resolution.environmentBundles,
}, {
secretRefs: authorized.secretRefs,
environmentBundleRefs: authorized.environmentBundleRefs,
});
return Object.freeze({
runId: body.runId,
attemptId: body.attemptId,
offerId: body.offerId,
executionDigest: body.executionDigest,
values: body.values,
environmentBundles: body.environmentBundles,
...(resolution.dispose === undefined
? {}
: { dispose: resolution.dispose }),
packages/ql3-cluster-control/src/worker-ingress/workerIngressConfig.ts
+2 -2
View File
@@ -519,9 +519,9 @@ export function loadClusterWorkerIngressConfig(
maxResponseBytes: integerValue(
environment,
'QL3_WORKER_INGRESS_MAX_RESPONSE_BYTES',
64 * 1024,
256 * 1024,
1024,
64 * 1024,
256 * 1024,
),
maxInFlightRequests: integerValue(
environment,
packages/ql3-cluster-control/src/worker-ingress/workerIngressPipeline.ts
+6 -2
View File
@@ -465,7 +465,7 @@ export function createWorkerIngressAdmissionPipeline(
'schema', 'runId', 'attemptId', 'projectId', 'taskId',
'taskRevision', 'executionDigest', 'workerGeneration',
'offerId', 'leaseGeneration', 'leaseToken',
'expectedLeaseVersion', 'secretRefs',
'expectedLeaseVersion', 'secretRefs', 'environmentBundleRefs',
]);
if (value.schema !== REMOTE_SECRET_DELIVERY_SCHEMA) {
throw failure(400, 'invalid_worker_request');
@@ -486,12 +486,16 @@ export function createWorkerIngressAdmissionPipeline(
leaseToken: value.leaseToken as string,
expectedLeaseVersion: value.expectedLeaseVersion as number,
secretRefs: value.secretRefs as string[],
environmentBundleRefs: value.environmentBundleRefs as string[],
},
);
try {
const responseBody = createRemoteWorkerSecretDeliveryResponseBody(
delivered,
value.secretRefs as string[],
{
secretRefs: value.secretRefs as string[],
environmentBundleRefs: value.environmentBundleRefs as string[],
},
);
if (
responseBody.runId !== value.runId ||
packages/ql3-cluster-control/test/mountedSecretProvider.test.cjs
+40 -4
View File
@@ -30,6 +30,11 @@ const VERSIONED_SECRET_REF = createSecretRef({
name: 'certificate',
version: 3,
});
const ENVIRONMENT_BUNDLE_REF = createSecretRef({
projectId: 'project-1',
name: 'legacy-env-bundle',
version: 4,
});
function authority(secretRefs = [SECRET_REF]) {
return {
@@ -46,6 +51,7 @@ function authority(secretRefs = [SECRET_REF]) {
leaseGeneration: 1,
leaseVersion: 1,
secretRefs,
environmentBundleRefs: [],
};
}
@@ -58,10 +64,7 @@ test('maps canonical SecretRef to a stable path-free Kubernetes key', () => {
const first = clusterMountedSecretFileName(SECRET_REF);
assert.match(first, /^[0-9a-f]{64}$/);
assert.equal(clusterMountedSecretFileName(SECRET_REF), first);
assert.notEqual(
clusterMountedSecretFileName(VERSIONED_SECRET_REF),
first,
);
assert.notEqual(clusterMountedSecretFileName(VERSIONED_SECRET_REF), first);
assert.throws(
() => clusterMountedSecretFileName('not-a-secret-ref'),
ClusterMountedSecretProviderError,
@@ -82,6 +85,7 @@ test('resolves every request again and observes atomic material rotation', async
assert.deepEqual(first.values, [
{ secretRef: SECRET_REF, value: 'generation-one' },
]);
assert.deepEqual(first.environmentBundles, []);
await first.dispose();
const replacement = `${file}.replacement`;
@@ -155,3 +159,35 @@ test('fails readiness for a missing or symlinked provider root', async (t) => {
ClusterMountedSecretProviderError,
);
});
test('delivers one larger opaque environment bundle without widening normal Secrets', async (t) => {
const root = await mkdtemp(path.join(os.tmpdir(), 'ql3-mounted-bundle-'));
t.after(() => rm(root, { recursive: true, force: true }));
await chmod(root, 0o700);
const value = JSON.stringify({
schema: 'qinglong/environment-bundle@v1',
entries: [{ name: 'LEGACY_VALUE', value: 'x'.repeat(20 * 1024) }],
});
await privateFile(
path.join(root, clusterMountedSecretFileName(ENVIRONMENT_BUNDLE_REF)),
value,
);
const provider = await createClusterMountedSecretProvider({
rootDirectory: root,
});
const resolution = await provider.resolve({
...authority([]),
environmentBundleRefs: [ENVIRONMENT_BUNDLE_REF],
});
assert.deepEqual(resolution.values, []);
assert.deepEqual(resolution.environmentBundles, [
{ secretRef: ENVIRONMENT_BUNDLE_REF, value },
]);
await resolution.dispose();
await privateFile(
path.join(root, clusterMountedSecretFileName(SECRET_REF)),
'x'.repeat(16 * 1024 + 1),
);
await assert.rejects(provider.resolve(authority()), /material_unavailable/);
});
packages/ql3-cluster-control/test/remoteWorkerSecretDeliveryService.test.cjs
+5 -1
View File
@@ -21,7 +21,7 @@ function command() {
taskId: 'task-1', taskRevision: 'revision-1', executionDigest: DIGEST,
offerId: 'offer-1', leaseGeneration: 3,
leaseToken: 'worker_generated_lease_capability_0000000000000001',
expectedLeaseVersion: 4, secretRefs: [SECRET_REF],
expectedLeaseVersion: 4, secretRefs: [SECRET_REF], environmentBundleRefs: [],
};
}
@@ -47,6 +47,7 @@ test('resolves plaintext only after repository authority succeeds', async () =>
assert.equal('leaseToken' in input, false);
return {
values: [{ secretRef: SECRET_REF, value: 'resolved-value' }],
environmentBundles: [],
dispose() { events.push('dispose'); },
};
},
@@ -56,6 +57,7 @@ test('resolves plaintext only after repository authority succeeds', async () =>
assert.deepEqual(result.values, [
{ secretRef: SECRET_REF, value: 'resolved-value' },
]);
assert.deepEqual(result.environmentBundles, []);
assert.deepEqual(events, ['authorize', 'resolve']);
await result.dispose();
assert.deepEqual(events, ['authorize', 'resolve', 'dispose']);
@@ -101,6 +103,7 @@ test('disposes malformed provider output and converts it to unavailable', async
async resolve() {
return {
values: [{ secretRef: SECRET_REF, value: 'x'.repeat(17 * 1024) }],
environmentBundles: [],
dispose() { disposed += 1; },
};
},
@@ -120,6 +123,7 @@ test('rejects extensible provider output and still invokes valid cleanup', async
async resolve() {
return {
values: [{ secretRef: SECRET_REF, value: 'resolved-value' }],
environmentBundles: [],
dispose() { disposed += 1; },
diagnostic: 'must-not-cross-boundary',
};
packages/ql3-cluster-control/test/workerIngressConfig.test.cjs
+1 -1
View File
@@ -93,7 +93,7 @@ test('builds exact bounded Worker ingress and least-privilege Pool config', asyn
host: '127.0.0.1',
port: 5901,
maxBodyBytes: 65_536,
maxResponseBytes: 65_536,
maxResponseBytes: 262_144,
maxInFlightRequests: 32,
authenticationRateWindowMs: 60_000,
authenticationRatePerPeer: 20,
packages/ql3-cluster-control/test/workerIngressPipeline.test.cjs
+10 -4
View File
@@ -592,6 +592,7 @@ test('binds one Secret batch to path identity and never echoes capabilities', as
offerId: command.offerId,
executionDigest: command.executionDigest,
values: [{ secretRef, value: 'resolved-value' }],
environmentBundles: [],
dispose() { disposed += 1; },
};
},
@@ -599,18 +600,20 @@ test('binds one Secret batch to path identity and never echoes capabilities', as
});
const leaseToken = 'worker_generated_lease_capability_0000000000000001';
const body = {
schema: 'qinglong/remote-secret-delivery@v1',
schema: 'qinglong/remote-secret-delivery@v2',
runId: 'run-1', attemptId: 'attempt-1', projectId: 'project-1',
taskId: 'task-1', taskRevision: 'revision-1', executionDigest,
workerGeneration: 2, offerId: 'offer-1', leaseGeneration: 3,
leaseToken, expectedLeaseVersion: 4, secretRefs: [secretRef],
environmentBundleRefs: [],
};
const result = await (await pipeline.prepare(metadata('secrets'))).handle(body);
assert.equal(result.statusCode, 200);
assert.equal(result.body.schema, 'qinglong/remote-secret-delivery@v1');
assert.equal(result.body.schema, 'qinglong/remote-secret-delivery@v2');
assert.deepEqual(result.body.values, [
{ secretRef, value: 'resolved-value' },
]);
assert.deepEqual(result.body.environmentBundles, []);
assert.equal(JSON.stringify(result.body).includes(leaseToken), false);
const { schema: _schema, ...commandBody } = body;
assert.deepEqual(observed, {
@@ -634,13 +637,14 @@ test('maps stale Secret delivery authority to conflict before any response', asy
});
await assert.rejects(
(await pipeline.prepare(metadata('secrets'))).handle({
schema: 'qinglong/remote-secret-delivery@v1',
schema: 'qinglong/remote-secret-delivery@v2',
runId: 'run-1', attemptId: 'attempt-1', projectId: 'project-1',
taskId: 'task-1', taskRevision: 'revision-1',
executionDigest: 'c'.repeat(64), workerGeneration: 2,
offerId: 'offer-1', leaseGeneration: 3,
leaseToken: 'worker_generated_lease_capability_0000000000000001',
expectedLeaseVersion: 4, secretRefs: [secretRef],
environmentBundleRefs: [],
}),
(error) =>
error.statusCode === 409 && error.code === 'worker_secret_delivery_fenced',
@@ -656,19 +660,21 @@ test('rejects a Secret service response whose authority drifts', async () => {
runId: 'run-other', attemptId: 'attempt-1', offerId: 'offer-1',
executionDigest: 'c'.repeat(64),
values: [{ secretRef, value: 'must-not-escape' }],
environmentBundles: [],
};
},
},
});
await assert.rejects(
(await pipeline.prepare(metadata('secrets'))).handle({
schema: 'qinglong/remote-secret-delivery@v1',
schema: 'qinglong/remote-secret-delivery@v2',
runId: 'run-1', attemptId: 'attempt-1', projectId: 'project-1',
taskId: 'task-1', taskRevision: 'revision-1',
executionDigest: 'c'.repeat(64), workerGeneration: 2,
offerId: 'offer-1', leaseGeneration: 3,
leaseToken: 'worker_generated_lease_capability_0000000000000001',
expectedLeaseVersion: 4, secretRefs: [secretRef],
environmentBundleRefs: [],
}),
(error) =>
error.statusCode === 503 && error.code === 'worker_ingress_unavailable',
packages/ql3-cluster-postgres/src/automation/taskDefinitionRepository.ts
+30 -25
View File
@@ -77,9 +77,7 @@ function unavailable(): TaskDefinitionUnavailableError {
return new TaskDefinitionUnavailableError();
}
function taskDefinitionRecord(
row: TaskDefinitionRow,
): TaskDefinitionRecord {
function taskDefinitionRecord(row: TaskDefinitionRow): TaskDefinitionRecord {
try {
const description = row.description;
if (description !== null && typeof description !== 'string') {
@@ -128,6 +126,9 @@ function executionPlanJson(
return Object.freeze({
command: revision.command,
environment: revision.environment,
...(revision.environmentBundleRef === undefined
? {}
: { environmentBundleRef: revision.environmentBundleRef }),
...(revision.workingDirectory === undefined
? {}
: { workingDirectory: revision.workingDirectory }),
@@ -140,7 +141,9 @@ function executionPlanJson(
});
}
function executionRevision(row: TaskDefinitionRow): ClusterTaskExecutionRevision {
function executionRevision(
row: TaskDefinitionRow,
): ClusterTaskExecutionRevision {
try {
const plan = postgresRequiredJsonObject(row.planJson, unavailable);
const keys = Object.keys(plan);
@@ -149,9 +152,14 @@ function executionRevision(row: TaskDefinitionRow): ClusterTaskExecutionRevision
!keys.includes('environment') ||
keys.some(
(key) =>
!['command', 'environment', 'placement', 'timeoutMs', 'workingDirectory'].includes(
key,
),
![
'command',
'environment',
'environmentBundleRef',
'placement',
'timeoutMs',
'workingDirectory',
].includes(key),
)
) {
throw unavailable();
@@ -176,6 +184,9 @@ function executionRevision(row: TaskDefinitionRow): ClusterTaskExecutionRevision
command: plan.command as ClusterTaskExecutionRevision['command'],
environment:
plan.environment as ClusterTaskExecutionRevision['environment'],
...(plan.environmentBundleRef === undefined
? {}
: { environmentBundleRef: plan.environmentBundleRef as string }),
...(plan.workingDirectory === undefined
? {}
: { workingDirectory: plan.workingDirectory as string }),
@@ -185,10 +196,9 @@ function executionRevision(row: TaskDefinitionRow): ClusterTaskExecutionRevision
...(plan.placement === undefined
? {}
: {
placement:
plan.placement as unknown as NonNullable<
ClusterTaskExecutionRevision['placement']
>,
placement: plan.placement as unknown as NonNullable<
ClusterTaskExecutionRevision['placement']
>,
}),
contentDigest: postgresRequiredString(row.contentDigest, unavailable),
createdAtMs: postgresRequiredInteger(row.createdAtMs, unavailable),
@@ -319,7 +329,6 @@ export class PostgresTaskDefinitionSource implements TaskDefinitionSource {
}
}
async findCurrentTaskDefinition(
projectId: string,
taskId: string,
@@ -422,7 +431,9 @@ export class PostgresTaskExecutionRevisionSource
readonly sourceRevision: number;
}): Promise<ClusterTaskExecutionRevision | null> {
if (!identity || typeof identity !== 'object' || Array.isArray(identity)) {
throw new TypeError('Cluster Task execution revision identity is invalid');
throw new TypeError(
'Cluster Task execution revision identity is invalid',
);
}
assertTaskDefinitionIdentifier(identity.projectId, 'projectId');
assertTaskDefinitionIdentifier(identity.taskId, 'taskId');
@@ -447,8 +458,7 @@ export class PostgresTaskDefinitionRepository
{
constructor(
pool: PostgresPool,
private readonly semanticRegistry: TaskSpecSemanticRegistry =
createBuiltInTaskSpecSemanticRegistry(),
private readonly semanticRegistry: TaskSpecSemanticRegistry = createBuiltInTaskSpecSemanticRegistry(),
) {
super(pool);
}
@@ -483,10 +493,7 @@ export class PostgresTaskDefinitionRepository
command.kind === 'command' &&
command.spec.schema === BUILT_IN_COMMAND_TASK_SPEC_SCHEMA
? compileClusterCommandTaskDefinition(
createTaskDefinitionRecord(
command,
command.occurredAtMs,
),
createTaskDefinitionRecord(command, command.occurredAtMs),
this.semanticRegistry,
)
: null;
@@ -560,10 +567,7 @@ export class PostgresTaskDefinitionRepository
WHERE id = $1`,
[command.projectId],
);
if (
project.rows.length !== 1 ||
project.rows[0]?.status !== 'active'
) {
if (project.rows.length !== 1 || project.rows[0]?.status !== 'active') {
throw new TaskDefinitionConflictError();
}
@@ -600,8 +604,9 @@ export class PostgresTaskDefinitionRepository
unavailable,
);
if (
(created ? command.expectedRevision !== null :
currentRevision !== command.expectedRevision) ||
(created
? command.expectedRevision !== null
: currentRevision !== command.expectedRevision) ||
command.occurredAtMs < previousUpdatedAtMs
) {
throw new TaskDefinitionConflictError();
packages/ql3-cluster-postgres/src/worker-credential/remoteWorkerSecretDeliveryRepository.ts
+56 -16
View File
@@ -27,7 +27,8 @@ function text(row: Row, key: string): string {
function integer(row: Row, key: string): number {
const raw = row[key];
const value = typeof raw === 'string' && /^\d+$/.test(raw) ? Number(raw) : raw;
const value =
typeof raw === 'string' && /^\d+$/.test(raw) ? Number(raw) : raw;
if (typeof value !== 'number' || !Number.isSafeInteger(value)) {
throw new RemoteWorkerSecretDeliveryUnavailableError();
}
@@ -44,10 +45,19 @@ function executionRevision(row: Row): ClusterTaskExecutionRevision {
if (
!keys.includes('command') ||
!keys.includes('environment') ||
keys.some((key) =>
!['command', 'environment', 'placement', 'timeoutMs', 'workingDirectory']
.includes(key))
) throw new RemoteWorkerSecretDeliveryUnavailableError();
keys.some(
(key) =>
![
'command',
'environment',
'environmentBundleRef',
'placement',
'timeoutMs',
'workingDirectory',
].includes(key),
)
)
throw new RemoteWorkerSecretDeliveryUnavailableError();
return normalizeClusterTaskExecutionRevision({
projectId: text(row, 'revisionProjectId'),
taskId: text(row, 'revisionTaskId'),
@@ -57,7 +67,11 @@ function executionRevision(row: Row): ClusterTaskExecutionRevision {
executorType: text(row, 'revisionExecutorType') as 'remote_worker',
planSchema: text(row, 'planSchema') as 'qinglong/command-execution@v1',
command: value.command as ClusterTaskExecutionRevision['command'],
environment: value.environment as ClusterTaskExecutionRevision['environment'],
environment:
value.environment as ClusterTaskExecutionRevision['environment'],
...(value.environmentBundleRef === undefined
? {}
: { environmentBundleRef: value.environmentBundleRef as string }),
...(value.workingDirectory === undefined
? {}
: { workingDirectory: value.workingDirectory as string }),
@@ -66,7 +80,11 @@ function executionRevision(row: Row): ClusterTaskExecutionRevision {
: { timeoutMs: value.timeoutMs as number }),
...(value.placement === undefined
? {}
: { placement: value.placement as NonNullable<ClusterTaskExecutionRevision['placement']> }),
: {
placement: value.placement as NonNullable<
ClusterTaskExecutionRevision['placement']
>,
}),
contentDigest: text(row, 'revisionContentDigest'),
createdAtMs: integer(row, 'revisionCreatedAtMs'),
});
@@ -80,7 +98,8 @@ async function begin(client: PostgresClient): Promise<void> {
}
export class PostgresRemoteWorkerSecretDeliveryAuthorityRepository
implements RemoteWorkerSecretDeliveryAuthorityRepository {
implements RemoteWorkerSecretDeliveryAuthorityRepository
{
constructor(private readonly pool: PostgresPool) {
if (!pool || typeof pool.connect !== 'function') {
throw new TypeError('PostgreSQL remote Secret delivery pool is invalid');
@@ -156,7 +175,9 @@ export class PostgresRemoteWorkerSecretDeliveryAuthorityRepository
}
const row = result.rows[0];
if (!row) {
throw new RemoteWorkerSecretDeliveryFenceRejectedError('authority_mismatch');
throw new RemoteWorkerSecretDeliveryFenceRejectedError(
'authority_mismatch',
);
}
const observedAtMs = integer(row, 'observedAtMs');
const tokenDigest = digestRunDispatchLeaseToken(command.leaseToken);
@@ -192,25 +213,38 @@ export class PostgresRemoteWorkerSecretDeliveryAuthorityRepository
row.leaseOfferId === command.offerId &&
integer(row, 'leaseExpiresAtMs') > observedAtMs;
if (!matches) {
throw new RemoteWorkerSecretDeliveryFenceRejectedError('authority_mismatch');
throw new RemoteWorkerSecretDeliveryFenceRejectedError(
'authority_mismatch',
);
}
let revision: ClusterTaskExecutionRevision;
try {
revision = executionRevision(row);
} catch (error) {
if (error instanceof RemoteWorkerSecretDeliveryUnavailableError) throw error;
if (error instanceof RemoteWorkerSecretDeliveryUnavailableError)
throw error;
throw new RemoteWorkerSecretDeliveryUnavailableError();
}
const expectedRefs = Object.freeze([
...new Set(revision.environment.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [])),
...new Set(
revision.environment.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [],
),
),
]);
const expectedEnvironmentBundleRefs = Object.freeze(
revision.environmentBundleRef === undefined
? []
: [revision.environmentBundleRef],
);
if (
revision.projectId !== command.projectId ||
revision.taskId !== command.taskId ||
revision.taskRevision !== command.taskRevision ||
revision.contentDigest !== command.executionDigest ||
JSON.stringify(expectedRefs) !== JSON.stringify(command.secretRefs)
JSON.stringify(expectedRefs) !== JSON.stringify(command.secretRefs) ||
JSON.stringify(expectedEnvironmentBundleRefs) !==
JSON.stringify(command.environmentBundleRefs)
) {
throw new RemoteWorkerSecretDeliveryFenceRejectedError(
'secret_scope_mismatch',
@@ -230,15 +264,21 @@ export class PostgresRemoteWorkerSecretDeliveryAuthorityRepository
leaseGeneration: command.leaseGeneration,
leaseVersion: command.expectedLeaseVersion,
secretRefs: expectedRefs,
environmentBundleRefs: expectedEnvironmentBundleRefs,
});
await client.query('COMMIT');
return authority;
} catch (error) {
try { await client.query('ROLLBACK'); } catch { /* preserve root */ }
try {
await client.query('ROLLBACK');
} catch {
/* preserve root */
}
if (
error instanceof RemoteWorkerSecretDeliveryFenceRejectedError ||
error instanceof RemoteWorkerSecretDeliveryUnavailableError
) throw error;
)
throw error;
throw new RemoteWorkerSecretDeliveryUnavailableError();
} finally {
client.release();
packages/ql3-cluster-postgres/test/definitionRepositories.test.cjs
+29
View File
@@ -17,6 +17,7 @@ const {
const {
compileClusterCommandTaskDefinition,
} = require('@qinglong/runtime-core/cluster-execution-revision');
const { createSecretRef } = require('@qinglong/runtime-core/secret-reference');
const {
TaskDefinitionAdministrationAuthorizationFenceConflictError,
TaskDefinitionAdministrationMutationConflictError,
@@ -447,6 +448,34 @@ test('publishes TaskDefinition atomically and replays the exact mutation', async
);
});
test('persists only the pinned environment bundle reference in an execution plan', async () => {
const environmentBundleRef = createSecretRef({
projectId: TASK_COMMAND.projectId,
name: 'legacy-env-bundle',
version: 7,
});
const command = {
...TASK_COMMAND,
taskId: 'task-bundle-00001',
mutationId: '123e4567-e89b-42d3-a456-426614174021',
spec: {
...TASK_COMMAND.spec,
config: { ...TASK_COMMAND.spec.config, environmentBundleRef },
},
};
const fixture = appendPool('task');
await new PostgresTaskDefinitionRepository(
fixture.pool,
).appendTaskDefinitionRevision(command);
const insert = fixture.queries.find(({ text }) =>
text.includes('INSERT INTO "ql3"."task_execution_revisions"'),
);
const plan = JSON.parse(insert.values[7]);
assert.equal(plan.environmentBundleRef, environmentBundleRef);
assert.deepEqual(plan.environment, []);
assert.equal(JSON.stringify(plan).includes('LEGACY_ENV_NAME'), false);
});
test('runs TaskDefinition transaction hooks for create and replay before COMMIT', async () => {
const createdFixture = appendPool('task');
const createdHook = [];
packages/ql3-cluster-postgres/test/remoteWorkerSecretDeliveryRepository.test.cjs
+109 -30
View File
@@ -18,12 +18,21 @@ const SOURCE_DIGEST = 'a'.repeat(64);
const TASK_REVISION = `qltd:v1:1:${SOURCE_DIGEST}`;
const LEASE_TOKEN = 'worker_generated_lease_capability_0000000000000001';
const SECRET_REF = createSecretRef({ projectId: 'project-1', name: 'token' });
const ENVIRONMENT_BUNDLE_REF = createSecretRef({
projectId: 'project-1',
name: 'legacy-env-bundle',
version: 4,
});
function revision() {
return createClusterTaskExecutionRevision({
projectId: 'project-1', taskId: 'task-1', taskRevision: TASK_REVISION,
sourceRevision: 1, sourceContentDigest: SOURCE_DIGEST,
executorType: 'remote_worker', planSchema: 'qinglong/command-execution@v1',
projectId: 'project-1',
taskId: 'task-1',
taskRevision: TASK_REVISION,
sourceRevision: 1,
sourceContentDigest: SOURCE_DIGEST,
executorType: 'remote_worker',
planSchema: 'qinglong/command-execution@v1',
command: { kind: 'argv', file: '/bin/true', args: [] },
environment: [{ name: 'TOKEN', kind: 'secret', secretRef: SECRET_REF }],
createdAtMs: 1,
@@ -32,40 +41,74 @@ function revision() {
function command(executionDigest, overrides = {}) {
return {
workerId: 'edge-1', workerSessionId: SESSION_ID, workerGeneration: 2,
runId: 'run-1', attemptId: 'attempt-1', projectId: 'project-1',
taskId: 'task-1', taskRevision: TASK_REVISION, executionDigest,
offerId: 'offer-1', leaseGeneration: 3, leaseToken: LEASE_TOKEN,
expectedLeaseVersion: 4, secretRefs: [SECRET_REF], ...overrides,
workerId: 'edge-1',
workerSessionId: SESSION_ID,
workerGeneration: 2,
runId: 'run-1',
attemptId: 'attempt-1',
projectId: 'project-1',
taskId: 'task-1',
taskRevision: TASK_REVISION,
executionDigest,
offerId: 'offer-1',
leaseGeneration: 3,
leaseToken: LEASE_TOKEN,
expectedLeaseVersion: 4,
secretRefs: [SECRET_REF],
environmentBundleRefs: [],
...overrides,
};
}
function authorityRow(plan, overrides = {}) {
return {
observedAtMs: '1000', runId: 'run-1', runProjectId: 'project-1',
runTaskId: 'task-1', runTaskRevision: TASK_REVISION,
runStatus: 'dispatching', executionOwner: 'runtime',
cancelRequestedAtMs: null, attemptStatus: 'starting',
attemptExecutorType: 'remote_worker', attemptWorkerId: 'edge-1',
attemptWorkerSessionId: SESSION_ID, attemptWorkerGeneration: 2,
observedAtMs: '1000',
runId: 'run-1',
runProjectId: 'project-1',
runTaskId: 'task-1',
runTaskRevision: TASK_REVISION,
runStatus: 'dispatching',
executionOwner: 'runtime',
cancelRequestedAtMs: null,
attemptStatus: 'starting',
attemptExecutorType: 'remote_worker',
attemptWorkerId: 'edge-1',
attemptWorkerSessionId: SESSION_ID,
attemptWorkerGeneration: 2,
attemptLeaseTokenDigest: digestRunDispatchLeaseToken(LEASE_TOKEN),
attemptLeaseGeneration: 3, attemptLeaseVersion: 4,
attemptOfferId: 'offer-1', sessionId: SESSION_ID, sessionGeneration: 2,
sessionStatus: 'online', sessionExpiresAtMs: '5000',
leaseRunId: 'run-1', leaseStatus: 'leased', leaseVersion: 4,
leaseGeneration: 3, leaseWorkerId: 'edge-1',
leaseWorkerSessionId: SESSION_ID, leaseWorkerGeneration: 2,
attemptLeaseGeneration: 3,
attemptLeaseVersion: 4,
attemptOfferId: 'offer-1',
sessionId: SESSION_ID,
sessionGeneration: 2,
sessionStatus: 'online',
sessionExpiresAtMs: '5000',
leaseRunId: 'run-1',
leaseStatus: 'leased',
leaseVersion: 4,
leaseGeneration: 3,
leaseWorkerId: 'edge-1',
leaseWorkerSessionId: SESSION_ID,
leaseWorkerGeneration: 2,
leaseTokenDigest: digestRunDispatchLeaseToken(LEASE_TOKEN),
leaseOfferId: 'offer-1', leaseExpiresAtMs: '5000',
revisionProjectId: plan.projectId, revisionTaskId: plan.taskId,
leaseOfferId: 'offer-1',
leaseExpiresAtMs: '5000',
revisionProjectId: plan.projectId,
revisionTaskId: plan.taskId,
sourceRevision: plan.sourceRevision,
revisionTaskRevision: plan.taskRevision,
sourceContentDigest: plan.sourceContentDigest,
revisionExecutorType: plan.executorType, planSchema: plan.planSchema,
revisionExecutorType: plan.executorType,
planSchema: plan.planSchema,
planJson: {
command: plan.command,
environment: plan.environment,
...(plan.workingDirectory === undefined ? {} : { workingDirectory: plan.workingDirectory }),
...(plan.environmentBundleRef === undefined
? {}
: { environmentBundleRef: plan.environmentBundleRef }),
...(plan.workingDirectory === undefined
? {}
: { workingDirectory: plan.workingDirectory }),
...(plan.timeoutMs === undefined ? {} : { timeoutMs: plan.timeoutMs }),
...(plan.placement === undefined ? {} : { placement: plan.placement }),
},
@@ -84,11 +127,15 @@ function fixture(row) {
if (sql.includes('FROM observation')) return { rows: row ? [row] : [] };
return { rows: [] };
},
release() { released += 1; },
release() {
released += 1;
},
};
return {
repository: new PostgresRemoteWorkerSecretDeliveryAuthorityRepository({
async connect() { return client; },
async connect() {
return client;
},
}),
queries,
released: () => released,
@@ -100,9 +147,13 @@ test('authorizes exact Session, Lease and immutable execution revision fences',
const { repository, queries, released } = fixture(authorityRow(plan));
const result = await repository.authorize(command(plan.contentDigest));
assert.deepEqual(result.secretRefs, [SECRET_REF]);
assert.deepEqual(result.environmentBundleRefs, []);
assert.equal(result.executionDigest, plan.contentDigest);
assert.equal('leaseToken' in result, false);
assert.equal(queries.some(({ sql }) => sql.includes('pg_advisory_xact_lock')), true);
assert.equal(
queries.some(({ sql }) => sql.includes('pg_advisory_xact_lock')),
true,
);
assert.equal(queries.at(-1).sql, 'COMMIT');
assert.equal(released(), 1);
});
@@ -129,9 +180,11 @@ test('rejects partial Secret scope and execution digest drift', async () => {
const extra = createSecretRef({ projectId: 'project-1', name: 'other' });
const partial = fixture(authorityRow(plan));
await assert.rejects(
partial.repository.authorize(command(plan.contentDigest, {
secretRefs: [extra],
})),
partial.repository.authorize(
command(plan.contentDigest, {
secretRefs: [extra],
}),
),
/secret_scope_mismatch/,
);
const digestDrift = fixture(authorityRow(plan));
@@ -140,3 +193,29 @@ test('rejects partial Secret scope and execution digest drift', async () => {
/secret_scope_mismatch/,
);
});
test('authorizes a bundle-only execution without exposing environment names', async () => {
const plan = createClusterTaskExecutionRevision({
projectId: 'project-1',
taskId: 'task-1',
taskRevision: TASK_REVISION,
sourceRevision: 1,
sourceContentDigest: SOURCE_DIGEST,
executorType: 'remote_worker',
planSchema: 'qinglong/command-execution@v1',
command: { kind: 'argv', file: '/bin/true', args: [] },
environment: [],
environmentBundleRef: ENVIRONMENT_BUNDLE_REF,
createdAtMs: 1,
});
const { repository } = fixture(authorityRow(plan));
const result = await repository.authorize(
command(plan.contentDigest, {
secretRefs: [],
environmentBundleRefs: [ENVIRONMENT_BUNDLE_REF],
}),
);
assert.deepEqual(result.secretRefs, []);
assert.deepEqual(result.environmentBundleRefs, [ENVIRONMENT_BUNDLE_REF]);
assert.equal(JSON.stringify(result).includes('LEGACY_VALUE'), false);
});
packages/ql3-runtime-core/package.json
+8
View File
@@ -110,6 +110,9 @@
"secret-projection": [
"dist/secret/secretProjection.d.ts"
],
"environment-bundle": [
"dist/secret/environmentBundle.d.ts"
],
"plugin-package-task-reconciliation": [
"dist/plugin-package/pluginPackageTaskReconciliation.d.ts"
],
@@ -470,6 +473,11 @@
"require": "./dist/secret/secretProjection.js",
"default": "./dist/secret/secretProjection.js"
},
"./environment-bundle": {
"types": "./dist/secret/environmentBundle.d.ts",
"require": "./dist/secret/environmentBundle.js",
"default": "./dist/secret/environmentBundle.js"
},
"./plugin-package-task-reconciliation": {
"types": "./dist/plugin-package/pluginPackageTaskReconciliation.d.ts",
"require": "./dist/plugin-package/pluginPackageTaskReconciliation.js",
packages/ql3-runtime-core/src/migration/clusterLegacyEnvMigrationPlan.ts
+2 -1
View File
@@ -7,6 +7,7 @@ export const CLUSTER_LEGACY_ENV_MIGRATION_PLAN_SCHEMA =
export const MAX_CLUSTER_LEGACY_ENV_SOURCE_ROWS = 100_000;
export const MAX_CLUSTER_LEGACY_ENV_TASKS = 100_000;
export const MAX_CLUSTER_LEGACY_ENV_TRIGGERS = 500_000;
export const MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BINDINGS = 256;
export const MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BYTES = 64 * 1024;
export const MAX_CLUSTER_LEGACY_ENV_MIGRATION_PLAN_JSON_BYTES = 8 * 1024;
@@ -207,7 +208,7 @@ function sourceEvidence(
const effectiveBindingCount = count(
value.effectiveBindingCount,
'effectiveBindingCount',
MAX_CLUSTER_LEGACY_ENV_SOURCE_ROWS,
MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BINDINGS,
);
if (
sourceRowCount < 1 ||
packages/ql3-runtime-core/src/remote-execution/remoteSecretDelivery.ts
+221 -69
View File
@@ -1,14 +1,19 @@
import { digestRunDispatchLeaseToken, assertRunDispatchId } from '../run/runDispatchLease';
import {
digestRunDispatchLeaseToken,
assertRunDispatchId,
} from '../run/runDispatchLease';
import { parseSecretRef } from '../secret/secretReference';
import { assertWorkerId, assertWorkerSessionId } from '../worker/workerSession';
export const REMOTE_SECRET_DELIVERY_SCHEMA =
'qinglong/remote-secret-delivery@v1';
'qinglong/remote-secret-delivery@v2';
export const MAX_REMOTE_SECRET_DELIVERY_REFS = 64;
export const MAX_REMOTE_ENVIRONMENT_BUNDLE_REFS = 1;
export const MAX_REMOTE_SECRET_DELIVERY_REQUEST_BYTES = 64 * 1024;
export const MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES = 128 * 1024;
export const MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES = 256 * 1024;
export const MAX_REMOTE_SECRET_VALUE_BYTES = 16 * 1024;
export const MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES = 64 * 1024;
export const MAX_REMOTE_ENVIRONMENT_BUNDLE_VALUE_BYTES = 96 * 1024;
export interface RemoteWorkerSecretDeliveryCommand {
readonly workerId: string;
@@ -25,6 +30,7 @@ export interface RemoteWorkerSecretDeliveryCommand {
readonly leaseToken: string;
readonly expectedLeaseVersion: number;
readonly secretRefs: readonly string[];
readonly environmentBundleRefs: readonly string[];
}
export type RemoteWorkerSecretDeliveryRequestBody = Readonly<
@@ -47,6 +53,7 @@ export interface RemoteWorkerSecretDeliveryAuthority {
readonly leaseGeneration: number;
readonly leaseVersion: number;
readonly secretRefs: readonly string[];
readonly environmentBundleRefs: readonly string[];
}
export interface RemoteWorkerSecretValue {
@@ -56,6 +63,7 @@ export interface RemoteWorkerSecretValue {
export interface RemoteWorkerSecretResolution {
readonly values: readonly RemoteWorkerSecretValue[];
readonly environmentBundles: readonly RemoteWorkerSecretValue[];
readonly dispose?: () => Promise<void> | void;
}
@@ -77,6 +85,7 @@ export interface RemoteWorkerSecretDeliveryResult {
readonly offerId: string;
readonly executionDigest: string;
readonly values: readonly RemoteWorkerSecretValue[];
readonly environmentBundles: readonly RemoteWorkerSecretValue[];
readonly dispose?: () => Promise<void> | void;
}
@@ -134,7 +143,8 @@ function exactKeys(
if (
actual.length !== sorted.length ||
actual.some((key, index) => key !== sorted[index])
) invalid(`${label} shape is invalid`);
)
invalid(`${label} shape is invalid`);
}
function identifier(label: string, value: unknown, maximum = 128): string {
@@ -143,7 +153,8 @@ function identifier(label: string, value: unknown, maximum = 128): string {
value.length < 1 ||
Buffer.byteLength(value, 'utf8') > maximum ||
/[\u0000-\u001f\u007f]/.test(value)
) return invalid(`${label} is invalid`);
)
return invalid(`${label} is invalid`);
return value;
}
@@ -152,19 +163,18 @@ function positiveInteger(label: string, value: unknown, minimum = 1): number {
!Number.isSafeInteger(value) ||
(value as number) < minimum ||
(value as number) > 2_147_483_647
) return invalid(`${label} is invalid`);
)
return invalid(`${label} is invalid`);
return value as number;
}
function normalizeSecretRefs(
value: unknown,
projectId: string,
maximum: number,
): readonly string[] {
if (
!Array.isArray(value) ||
value.length < 1 ||
value.length > MAX_REMOTE_SECRET_DELIVERY_REFS
) return invalid('secretRefs are invalid');
if (!Array.isArray(value) || value.length > maximum)
return invalid('secretRefs are invalid');
const seen = new Set<string>();
const refs = value.map((entry) => {
if (typeof entry !== 'string' || seen.has(entry)) {
@@ -188,11 +198,27 @@ export function normalizeRemoteWorkerSecretDeliveryCommand(
value: RemoteWorkerSecretDeliveryCommand,
): Readonly<RemoteWorkerSecretDeliveryCommand> {
const command = object(value, 'command');
exactKeys(command, [
'attemptId', 'executionDigest', 'expectedLeaseVersion', 'leaseGeneration',
'leaseToken', 'offerId', 'projectId', 'runId', 'secretRefs', 'taskId',
'taskRevision', 'workerGeneration', 'workerId', 'workerSessionId',
], 'command');
exactKeys(
command,
[
'attemptId',
'environmentBundleRefs',
'executionDigest',
'expectedLeaseVersion',
'leaseGeneration',
'leaseToken',
'offerId',
'projectId',
'runId',
'secretRefs',
'taskId',
'taskRevision',
'workerGeneration',
'workerId',
'workerSessionId',
],
'command',
);
try {
assertWorkerId(command.workerId as string);
assertWorkerSessionId(command.workerSessionId as string);
@@ -206,7 +232,10 @@ export function normalizeRemoteWorkerSecretDeliveryCommand(
const normalized = Object.freeze({
workerId: command.workerId as string,
workerSessionId: command.workerSessionId as string,
workerGeneration: positiveInteger('workerGeneration', command.workerGeneration),
workerGeneration: positiveInteger(
'workerGeneration',
command.workerGeneration,
),
runId: command.runId as string,
attemptId: command.attemptId as string,
projectId,
@@ -214,13 +243,38 @@ export function normalizeRemoteWorkerSecretDeliveryCommand(
taskRevision: identifier('taskRevision', command.taskRevision),
executionDigest: identifier('executionDigest', command.executionDigest, 64),
offerId: command.offerId as string,
leaseGeneration: positiveInteger('leaseGeneration', command.leaseGeneration),
leaseGeneration: positiveInteger(
'leaseGeneration',
command.leaseGeneration,
),
leaseToken: identifier('leaseToken', command.leaseToken, 128),
expectedLeaseVersion: positiveInteger(
'expectedLeaseVersion', command.expectedLeaseVersion, 0,
'expectedLeaseVersion',
command.expectedLeaseVersion,
0,
),
secretRefs: normalizeSecretRefs(
command.secretRefs,
projectId,
MAX_REMOTE_SECRET_DELIVERY_REFS,
),
environmentBundleRefs: normalizeSecretRefs(
command.environmentBundleRefs,
projectId,
MAX_REMOTE_ENVIRONMENT_BUNDLE_REFS,
),
secretRefs: normalizeSecretRefs(command.secretRefs, projectId),
});
if (
normalized.secretRefs.length + normalized.environmentBundleRefs.length <
1
)
return invalid('Secret reference set is empty');
if (
normalized.secretRefs.some((reference) =>
normalized.environmentBundleRefs.includes(reference),
)
)
return invalid('Secret reference roles overlap');
if (!/^[0-9a-f]{64}$/.test(normalized.executionDigest)) {
return invalid('executionDigest is invalid');
}
@@ -236,11 +290,26 @@ export function normalizeRemoteWorkerSecretDeliveryAuthority(
value: RemoteWorkerSecretDeliveryAuthority,
): Readonly<RemoteWorkerSecretDeliveryAuthority> {
const authority = object(value, 'authority');
exactKeys(authority, [
'attemptId', 'executionDigest', 'leaseGeneration', 'leaseVersion',
'offerId', 'projectId', 'runId', 'secretRefs', 'taskId', 'taskRevision',
'workerGeneration', 'workerId', 'workerSessionId',
], 'authority');
exactKeys(
authority,
[
'attemptId',
'environmentBundleRefs',
'executionDigest',
'leaseGeneration',
'leaseVersion',
'offerId',
'projectId',
'runId',
'secretRefs',
'taskId',
'taskRevision',
'workerGeneration',
'workerId',
'workerSessionId',
],
'authority',
);
try {
assertWorkerId(authority.workerId as string);
assertWorkerSessionId(authority.workerSessionId as string);
@@ -255,7 +324,8 @@ export function normalizeRemoteWorkerSecretDeliveryAuthority(
workerId: authority.workerId as string,
workerSessionId: authority.workerSessionId as string,
workerGeneration: positiveInteger(
'workerGeneration', authority.workerGeneration,
'workerGeneration',
authority.workerGeneration,
),
runId: authority.runId as string,
attemptId: authority.attemptId as string,
@@ -263,15 +333,38 @@ export function normalizeRemoteWorkerSecretDeliveryAuthority(
taskId: identifier('taskId', authority.taskId),
taskRevision: identifier('taskRevision', authority.taskRevision),
executionDigest: identifier(
'executionDigest', authority.executionDigest, 64,
'executionDigest',
authority.executionDigest,
64,
),
offerId: authority.offerId as string,
leaseGeneration: positiveInteger(
'leaseGeneration', authority.leaseGeneration,
'leaseGeneration',
authority.leaseGeneration,
),
leaseVersion: positiveInteger('leaseVersion', authority.leaseVersion, 0),
secretRefs: normalizeSecretRefs(authority.secretRefs, projectId),
secretRefs: normalizeSecretRefs(
authority.secretRefs,
projectId,
MAX_REMOTE_SECRET_DELIVERY_REFS,
),
environmentBundleRefs: normalizeSecretRefs(
authority.environmentBundleRefs,
projectId,
MAX_REMOTE_ENVIRONMENT_BUNDLE_REFS,
),
});
if (
normalized.secretRefs.length + normalized.environmentBundleRefs.length <
1
)
return invalid('Secret reference set is empty');
if (
normalized.secretRefs.some((reference) =>
normalized.environmentBundleRefs.includes(reference),
)
)
return invalid('Secret reference roles overlap');
if (!/^[0-9a-f]{64}$/.test(normalized.executionDigest)) {
return invalid('executionDigest is invalid');
}
@@ -282,59 +375,98 @@ export function createRemoteWorkerSecretDeliveryRequestBody(
command: RemoteWorkerSecretDeliveryCommand,
): RemoteWorkerSecretDeliveryRequestBody {
const normalized = normalizeRemoteWorkerSecretDeliveryCommand(command);
const { workerId: _workerId, workerSessionId: _sessionId, ...request } = normalized;
const {
workerId: _workerId,
workerSessionId: _sessionId,
...request
} = normalized;
return Object.freeze({ schema: REMOTE_SECRET_DELIVERY_SCHEMA, ...request });
}
function normalizeValues(
value: unknown,
expectedRefs: readonly string[],
maximumValueBytes: number,
maximumTotalBytes: number,
label: string,
): readonly RemoteWorkerSecretValue[] {
if (!Array.isArray(value) || value.length !== expectedRefs.length) {
return invalid('Secret values are invalid');
return invalid(`${label} are invalid`);
}
let totalValueBytes = 0;
return Object.freeze(value.map((entry, index) => {
const item = object(entry, `values[${index}]`);
exactKeys(item, ['secretRef', 'value'], `values[${index}]`);
if (
item.secretRef !== expectedRefs[index] ||
typeof item.value !== 'string' ||
item.value.includes('\0') ||
Buffer.byteLength(item.value, 'utf8') > MAX_REMOTE_SECRET_VALUE_BYTES
) return invalid(`values[${index}] is invalid`);
totalValueBytes += Buffer.byteLength(item.value, 'utf8');
if (totalValueBytes > MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES) {
return invalid('Secret value byte budget exceeded');
}
return Object.freeze({
secretRef: item.secretRef as string,
value: item.value,
});
}));
return Object.freeze(
value.map((entry, index) => {
const item = object(entry, `values[${index}]`);
exactKeys(item, ['secretRef', 'value'], `values[${index}]`);
if (
item.secretRef !== expectedRefs[index] ||
typeof item.value !== 'string' ||
item.value.includes('\0') ||
Buffer.byteLength(item.value, 'utf8') > maximumValueBytes
)
return invalid(`values[${index}] is invalid`);
totalValueBytes += Buffer.byteLength(item.value, 'utf8');
if (totalValueBytes > maximumTotalBytes) {
return invalid('Secret value byte budget exceeded');
}
return Object.freeze({
secretRef: item.secretRef as string,
value: item.value,
});
}),
);
}
export function createRemoteWorkerSecretDeliveryResponseBody(
result: Readonly<RemoteWorkerSecretDeliveryResult>,
expectedRefs: readonly string[],
expected: Readonly<{
secretRefs: readonly string[];
environmentBundleRefs: readonly string[];
}>,
): RemoteWorkerSecretDeliveryResponseBody {
const value = object(result, 'result');
const allowed = ['attemptId', 'dispose', 'executionDigest', 'offerId', 'runId', 'values'];
const allowed = [
'attemptId',
'dispose',
'environmentBundles',
'executionDigest',
'offerId',
'runId',
'values',
];
if (Object.keys(value).some((key) => !allowed.includes(key))) {
return invalid('result shape is invalid');
}
const runId = identifier('runId', value.runId, 36);
const attemptId = identifier('attemptId', value.attemptId, 36);
const offerId = identifier('offerId', value.offerId, 128);
const executionDigest = identifier('executionDigest', value.executionDigest, 64);
if (!/^[0-9a-f]{64}$/.test(executionDigest)) invalid('executionDigest is invalid');
const executionDigest = identifier(
'executionDigest',
value.executionDigest,
64,
);
if (!/^[0-9a-f]{64}$/.test(executionDigest))
invalid('executionDigest is invalid');
return Object.freeze({
schema: REMOTE_SECRET_DELIVERY_SCHEMA,
runId,
attemptId,
offerId,
executionDigest,
values: normalizeValues(value.values, expectedRefs),
values: normalizeValues(
value.values,
expected.secretRefs,
MAX_REMOTE_SECRET_VALUE_BYTES,
MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES,
'Secret values',
),
environmentBundles: normalizeValues(
value.environmentBundles,
expected.environmentBundleRefs,
MAX_REMOTE_ENVIRONMENT_BUNDLE_VALUE_BYTES,
MAX_REMOTE_ENVIRONMENT_BUNDLE_VALUE_BYTES,
'environment bundles',
),
});
}
@@ -346,15 +478,18 @@ export function parseRemoteWorkerSecretDeliveryResponse(
offerId: string;
executionDigest: string;
secretRefs: readonly string[];
environmentBundleRefs: readonly string[];
}>,
): Readonly<RemoteWorkerSecretDeliveryResult> {
const bytes = typeof serialized === 'string'
? Buffer.from(serialized, 'utf8')
: Buffer.from(serialized);
const bytes =
typeof serialized === 'string'
? Buffer.from(serialized, 'utf8')
: Buffer.from(serialized);
if (
bytes.byteLength < 2 ||
bytes.byteLength > MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES
) return invalid('response byte size is outside the allowed range');
)
return invalid('response byte size is outside the allowed range');
let parsed: unknown;
try {
parsed = JSON.parse(bytes.toString('utf8')) as unknown;
@@ -364,30 +499,47 @@ export function parseRemoteWorkerSecretDeliveryResponse(
bytes.fill(0);
}
const response = object(parsed, 'response');
exactKeys(response, [
'attemptId', 'executionDigest', 'offerId', 'runId', 'schema', 'values',
], 'response');
exactKeys(
response,
[
'attemptId',
'environmentBundles',
'executionDigest',
'offerId',
'runId',
'schema',
'values',
],
'response',
);
if (response.schema !== REMOTE_SECRET_DELIVERY_SCHEMA) {
return invalid('response schema is invalid');
}
const result = createRemoteWorkerSecretDeliveryResponseBody({
runId: response.runId as string,
attemptId: response.attemptId as string,
offerId: response.offerId as string,
executionDigest: response.executionDigest as string,
values: response.values as readonly RemoteWorkerSecretValue[],
}, expected.secretRefs);
const result = createRemoteWorkerSecretDeliveryResponseBody(
{
runId: response.runId as string,
attemptId: response.attemptId as string,
offerId: response.offerId as string,
executionDigest: response.executionDigest as string,
values: response.values as readonly RemoteWorkerSecretValue[],
environmentBundles:
response.environmentBundles as readonly RemoteWorkerSecretValue[],
},
expected,
);
if (
result.runId !== expected.runId ||
result.attemptId !== expected.attemptId ||
result.offerId !== expected.offerId ||
result.executionDigest !== expected.executionDigest
) return invalid('response authority does not match request');
)
return invalid('response authority does not match request');
return Object.freeze({
runId: result.runId,
attemptId: result.attemptId,
offerId: result.offerId,
executionDigest: result.executionDigest,
values: result.values,
environmentBundles: result.environmentBundles,
});
}
packages/ql3-runtime-core/src/secret/environmentBundle.ts
+164
View File
@@ -0,0 +1,164 @@
export const ENVIRONMENT_BUNDLE_SCHEMA =
'qinglong/environment-bundle@v1' as const;
export const MAX_ENVIRONMENT_BUNDLE_ENTRIES = 256;
export const MAX_ENVIRONMENT_BUNDLE_VALUE_BYTES = 16 * 1024;
export const MAX_ENVIRONMENT_BUNDLE_TOTAL_BYTES = 64 * 1024;
export const MAX_ENVIRONMENT_BUNDLE_ENCODED_BYTES = 96 * 1024;
const ENVIRONMENT_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/;
export interface EnvironmentBundleEntry {
readonly name: string;
readonly value: string;
}
export interface EnvironmentBundle {
readonly schema: typeof ENVIRONMENT_BUNDLE_SCHEMA;
readonly entries: readonly EnvironmentBundleEntry[];
}
export class InvalidEnvironmentBundleError extends TypeError {
readonly code = 'ENVIRONMENT_BUNDLE_INVALID';
constructor(message: string) {
super(`Environment bundle is invalid: ${message}`);
this.name = 'InvalidEnvironmentBundleError';
}
}
function invalid(message: string): never {
throw new InvalidEnvironmentBundleError(message);
}
function dataObject(value: unknown, label: string): Record<string, unknown> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
(Object.getPrototypeOf(value) !== Object.prototype &&
Object.getPrototypeOf(value) !== null)
) {
return invalid(`${label} must be an object`);
}
const descriptors = Object.getOwnPropertyDescriptors(value);
if (
Object.values(descriptors).some(
(descriptor) =>
descriptor.get !== undefined ||
descriptor.set !== undefined ||
descriptor.enumerable !== true,
)
) {
return invalid(`${label} must contain enumerable data properties`);
}
return value as Record<string, unknown>;
}
function exactKeys(
value: object,
expected: readonly string[],
label: string,
): void {
const actual = Reflect.ownKeys(value);
const canonical = [...expected].sort();
if (
actual.some((key) => typeof key !== 'string') ||
actual.length !== canonical.length ||
actual
.map(String)
.sort()
.some((key, index) => key !== canonical[index])
) {
return invalid(`${label} shape is invalid`);
}
}
export function normalizeEnvironmentBundle(
value: EnvironmentBundle,
): Readonly<EnvironmentBundle> {
const bundle = dataObject(value, 'bundle');
exactKeys(bundle, ['entries', 'schema'], 'bundle');
if (bundle.schema !== ENVIRONMENT_BUNDLE_SCHEMA) {
return invalid('schema is invalid');
}
if (
!Array.isArray(bundle.entries) ||
bundle.entries.length < 1 ||
bundle.entries.length > MAX_ENVIRONMENT_BUNDLE_ENTRIES
) {
return invalid('entry count is invalid');
}
const names = new Set<string>();
let totalBytes = 0;
const entries = bundle.entries.map((value, index) => {
const entry = dataObject(value, `entries[${index}]`);
exactKeys(entry, ['name', 'value'], `entries[${index}]`);
if (
typeof entry.name !== 'string' ||
!ENVIRONMENT_NAME_PATTERN.test(entry.name) ||
entry.name.startsWith('QL3_') ||
names.has(entry.name)
) {
return invalid(`entries[${index}].name is invalid or duplicated`);
}
if (
typeof entry.value !== 'string' ||
entry.value.includes('\0') ||
Buffer.byteLength(entry.value, 'utf8') >
MAX_ENVIRONMENT_BUNDLE_VALUE_BYTES
) {
return invalid(`entries[${index}].value is invalid`);
}
names.add(entry.name);
totalBytes +=
Buffer.byteLength(entry.name, 'utf8') +
Buffer.byteLength(entry.value, 'utf8');
if (totalBytes > MAX_ENVIRONMENT_BUNDLE_TOTAL_BYTES) {
return invalid('environment byte budget exceeded');
}
return Object.freeze({ name: entry.name, value: entry.value });
});
entries.sort((left, right) =>
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
);
const normalized = Object.freeze({
schema: ENVIRONMENT_BUNDLE_SCHEMA,
entries: Object.freeze(entries),
});
if (
Buffer.byteLength(JSON.stringify(normalized), 'utf8') >
MAX_ENVIRONMENT_BUNDLE_ENCODED_BYTES
) {
return invalid('encoded byte budget exceeded');
}
return normalized;
}
export function serializeEnvironmentBundle(value: EnvironmentBundle): string {
return JSON.stringify(normalizeEnvironmentBundle(value));
}
export function parseEnvironmentBundle(
serialized: string | Uint8Array,
): Readonly<EnvironmentBundle> {
const bytes =
typeof serialized === 'string'
? Buffer.from(serialized, 'utf8')
: Buffer.from(serialized);
if (
bytes.byteLength < 2 ||
bytes.byteLength > MAX_ENVIRONMENT_BUNDLE_ENCODED_BYTES
) {
bytes.fill(0);
return invalid('encoded byte size is outside the allowed range');
}
let parsed: unknown;
try {
parsed = JSON.parse(bytes.toString('utf8')) as unknown;
} catch {
return invalid('payload is not valid JSON');
} finally {
bytes.fill(0);
}
return normalizeEnvironmentBundle(parsed as EnvironmentBundle);
}
packages/ql3-runtime-core/src/task-definition/clusterExecutionRevision.ts
+27 -10
View File
@@ -32,6 +32,7 @@ export interface ClusterTaskExecutionRevisionContent {
readonly planSchema: typeof CLUSTER_EXECUTION_PLAN_SCHEMA;
readonly command: LocalDispatchCommand;
readonly environment: readonly LocalExecutionEnvironmentBinding[];
readonly environmentBundleRef?: string;
readonly workingDirectory?: string;
readonly timeoutMs?: number;
readonly placement?: RemoteWorkerPlacementSpec;
@@ -79,9 +80,7 @@ function revision(value: unknown): number {
(value as number) < 1 ||
(value as number) > 2_147_483_647
) {
throw new InvalidClusterExecutionRevisionError(
'sourceRevision is invalid',
);
throw new InvalidClusterExecutionRevisionError('sourceRevision is invalid');
}
return value as number;
}
@@ -103,6 +102,7 @@ function normalizeContent(
'command',
'createdAtMs',
'environment',
'environmentBundleRef',
'executorType',
'planSchema',
'placement',
@@ -155,6 +155,7 @@ function normalizeContent(
}
let command: LocalDispatchCommand;
let environment: readonly LocalExecutionEnvironmentBinding[];
let environmentBundleRef: string | undefined;
try {
command = normalizeLocalDispatchCommand(value.command);
environment = createLocalExecutionContextRecipe({
@@ -169,6 +170,16 @@ function normalizeContent(
throw new Error('cross-project Secret reference');
}
}
if (value.environmentBundleRef !== undefined) {
const reference = parseSecretRef(value.environmentBundleRef);
if (
reference.projectId !== projectId ||
reference.version === undefined
) {
throw new Error('invalid environment bundle Secret reference');
}
environmentBundleRef = value.environmentBundleRef;
}
} catch {
throw new InvalidClusterExecutionRevisionError(
'command or environment is invalid',
@@ -200,9 +211,10 @@ function normalizeContent(
timeoutMs = value.timeoutMs;
}
const createdAtMs = timestamp(value.createdAtMs);
const placement = value.placement === undefined
? undefined
: effectiveRemoteWorkerPlacement(value.placement);
const placement =
value.placement === undefined
? undefined
: effectiveRemoteWorkerPlacement(value.placement);
const normalized = Object.freeze({
projectId,
taskId,
@@ -213,20 +225,22 @@ function normalizeContent(
planSchema: CLUSTER_EXECUTION_PLAN_SCHEMA,
command,
environment,
...(environmentBundleRef === undefined ? {} : { environmentBundleRef }),
...(workingDirectory === undefined ? {} : { workingDirectory }),
...(timeoutMs === undefined ? {} : { timeoutMs }),
...(placement === undefined ? {} : { placement }),
createdAtMs,
});
if (Buffer.byteLength(JSON.stringify(normalized), 'utf8') > MAX_CLUSTER_EXECUTION_PLAN_BYTES) {
if (
Buffer.byteLength(JSON.stringify(normalized), 'utf8') >
MAX_CLUSTER_EXECUTION_PLAN_BYTES
) {
throw new InvalidClusterExecutionRevisionError('plan byte budget exceeded');
}
return normalized;
}
function digest(
content: ClusterTaskExecutionRevisionContent,
): string {
function digest(content: ClusterTaskExecutionRevisionContent): string {
const { createdAtMs: _createdAtMs, ...immutable } = content;
return createHash('sha256')
.update('qinglong.cluster-task-execution-revision.v1\0', 'utf8')
@@ -273,6 +287,9 @@ export function compileClusterCommandTaskDefinition(
planSchema: CLUSTER_EXECUTION_PLAN_SCHEMA,
command: plan.command,
environment: plan.environment,
...(plan.environmentBundleRef === undefined
? {}
: { environmentBundleRef: plan.environmentBundleRef }),
...(plan.workingDirectory === undefined
? {}
: { workingDirectory: plan.workingDirectory }),
packages/ql3-runtime-core/src/task-definition/taskDefinitionExecutionCompiler.ts
+12 -4
View File
@@ -36,6 +36,7 @@ export interface CommandTaskExecutionPlan {
readonly sourceContentDigest: string;
readonly command: LocalDispatchCommand;
readonly environment: readonly LocalExecutionEnvironmentBinding[];
readonly environmentBundleRef?: string;
readonly workingDirectory?: string;
readonly timeoutMs?: number;
readonly placement?: RemoteWorkerPlacementSpec;
@@ -112,13 +113,13 @@ export function parseTaskDefinitionRevisionRef(
return Object.freeze({ revision, contentDigest });
}
function canonicalRecord(definition: TaskDefinitionRecord): TaskDefinitionRecord {
function canonicalRecord(
definition: TaskDefinitionRecord,
): TaskDefinitionRecord {
try {
return normalizeTaskDefinitionRecord(definition);
} catch {
throw new InvalidTaskDefinitionCompilationError(
'source record is invalid',
);
throw new InvalidTaskDefinitionCompilationError('source record is invalid');
}
}
@@ -172,6 +173,7 @@ export function compileCommandTaskDefinition(
const config = semanticSpec.config as unknown as Readonly<{
command: LocalDispatchCommand;
environment: readonly LocalExecutionEnvironmentBinding[];
environmentBundleRef?: string;
workingDirectory?: string;
timeoutMs?: number;
placement?: RemoteWorkerPlacementSpec;
@@ -188,6 +190,9 @@ export function compileCommandTaskDefinition(
sourceContentDigest: source.contentDigest,
command: config.command,
environment: config.environment,
...(config.environmentBundleRef === undefined
? {}
: { environmentBundleRef: config.environmentBundleRef }),
...(config.workingDirectory === undefined
? {}
: { workingDirectory: config.workingDirectory }),
@@ -202,6 +207,9 @@ export function compileLocalCommandTaskDefinition(
semanticRegistry: TaskSpecSemanticRegistry,
): LocalCommandTaskExecutionPlan {
const source = compileCommandTaskDefinition(definition, semanticRegistry);
if (source.environmentBundleRef !== undefined) {
throw new UnsupportedTaskDefinitionCompilationError();
}
const contextRecipe = createLocalExecutionContextRecipe({
environment: source.environment,
createdAtMs: source.createdAtMs,
packages/ql3-runtime-core/src/task-definition/taskSpecSemantic.ts
+41 -16
View File
@@ -196,9 +196,7 @@ function normalizeEnvironment(
);
});
if (bytes > MAX_COMMAND_TASK_ENVIRONMENT_BYTES) {
throw new InvalidTaskSpecSemanticError(
'environment byte budget exceeded',
);
throw new InvalidTaskSpecSemanticError('environment byte budget exceeded');
}
environment.sort((left, right) =>
(left as { name: string }).name.localeCompare(
@@ -215,14 +213,46 @@ function normalizeCommandConfig(
exactKeys(
config,
['command'],
['environment', 'placement', 'timeoutMs', 'workingDirectory'],
[
'environment',
'environmentBundleRef',
'placement',
'timeoutMs',
'workingDirectory',
],
'command config',
);
const command = normalizeCommand(config.command);
const environment = normalizeEnvironment(config.environment ?? [], context.projectId);
const placement = config.placement === undefined
? undefined
: normalizeRemoteWorkerPlacement(config.placement);
const environment = normalizeEnvironment(
config.environment ?? [],
context.projectId,
);
let environmentBundleRef: string | undefined;
if (config.environmentBundleRef !== undefined) {
environmentBundleRef = boundedText(
config.environmentBundleRef,
'environmentBundleRef',
512,
);
let reference;
try {
reference = parseSecretRef(environmentBundleRef);
} catch {
throw new InvalidTaskSpecSemanticError('environmentBundleRef is invalid');
}
if (
reference.projectId !== context.projectId ||
reference.version === undefined
) {
throw new InvalidTaskSpecSemanticError(
'environmentBundleRef must pin a version in the same Project',
);
}
}
const placement =
config.placement === undefined
? undefined
: normalizeRemoteWorkerPlacement(config.placement);
let workingDirectory: string | undefined;
if (config.workingDirectory !== undefined) {
workingDirectory = boundedText(
@@ -250,6 +280,7 @@ function normalizeCommandConfig(
return Object.freeze({
command,
environment,
...(environmentBundleRef === undefined ? {} : { environmentBundleRef }),
...(placement === undefined
? {}
: { placement: placement as unknown as TaskDefinitionJson }),
@@ -268,10 +299,7 @@ const BUILT_IN_DESCRIPTORS: readonly TaskSpecSemanticDescriptor[] =
]);
export class TaskSpecSemanticRegistry {
readonly #descriptors: ReadonlyMap<
string,
TaskSpecSemanticDescriptor
>;
readonly #descriptors: ReadonlyMap<string, TaskSpecSemanticDescriptor>;
readonly #metadata: readonly TaskSpecSemanticMetadata[];
constructor(descriptors: readonly TaskSpecSemanticDescriptor[]) {
@@ -393,8 +421,5 @@ export function createTaskSpecSemanticRegistry(
'extension descriptor uses the reserved qinglong namespace',
);
}
return new TaskSpecSemanticRegistry([
...BUILT_IN_DESCRIPTORS,
...extensions,
]);
return new TaskSpecSemanticRegistry([...BUILT_IN_DESCRIPTORS, ...extensions]);
}
packages/ql3-runtime-core/test/clusterExecutionRevision.test.cjs
+76 -21
View File
@@ -44,15 +44,55 @@ function definition() {
});
return {
registry,
record: createTaskDefinitionRecord({
...command,
spec: registry.normalize({
projectId: command.projectId,
taskId: command.taskId,
kind: command.kind,
spec: command.spec,
}),
}, 90),
record: createTaskDefinitionRecord(
{
...command,
spec: registry.normalize({
projectId: command.projectId,
taskId: command.taskId,
kind: command.kind,
spec: command.spec,
}),
},
90,
),
};
}
function definitionWithBundle() {
const input = definition();
const environmentBundleRef = createSecretRef({
projectId: 'default',
name: 'legacy-env-bundle',
version: 4,
});
const spec = input.registry.normalize({
projectId: input.record.projectId,
taskId: input.record.taskId,
kind: input.record.kind,
spec: {
...input.record.spec,
config: { ...input.record.spec.config, environmentBundleRef },
},
});
return {
registry: input.registry,
environmentBundleRef,
record: createTaskDefinitionRecord(
{
projectId: input.record.projectId,
taskId: input.record.taskId,
expectedRevision: null,
mutationId: input.record.mutationId,
name: input.record.name,
kind: input.record.kind,
spec,
labels: input.record.labels,
enabled: input.record.enabled,
occurredAtMs: input.record.updatedAtMs,
},
input.record.createdAtMs,
),
};
}
@@ -70,6 +110,17 @@ test('compiles one digest-bound remote Worker execution revision', () => {
assert.deepEqual(normalizeClusterTaskExecutionRevision(revision), revision);
});
test('carries only the pinned environment bundle reference into Cluster plans', () => {
const input = definitionWithBundle();
const revision = compileClusterCommandTaskDefinition(
input.record,
input.registry,
);
assert.equal(revision.environmentBundleRef, input.environmentBundleRef);
assert.equal(JSON.stringify(revision).includes('legacy env value'), false);
assert.deepEqual(normalizeClusterTaskExecutionRevision(revision), revision);
});
test('rejects digest drift and cross-Project Secret references', () => {
const input = definition();
const revision = compileClusterCommandTaskDefinition(
@@ -77,21 +128,25 @@ test('rejects digest drift and cross-Project Secret references', () => {
input.registry,
);
assert.throws(
() => normalizeClusterTaskExecutionRevision({
...revision,
contentDigest: '0'.repeat(64),
}),
() =>
normalizeClusterTaskExecutionRevision({
...revision,
contentDigest: '0'.repeat(64),
}),
InvalidClusterExecutionRevisionError,
);
assert.throws(
() => normalizeClusterTaskExecutionRevision({
...revision,
environment: [{
kind: 'secret',
name: 'TOKEN',
secretRef: createSecretRef({ projectId: 'another', name: 'TOKEN' }),
}],
}),
() =>
normalizeClusterTaskExecutionRevision({
...revision,
environment: [
{
kind: 'secret',
name: 'TOKEN',
secretRef: createSecretRef({ projectId: 'another', name: 'TOKEN' }),
},
],
}),
InvalidClusterExecutionRevisionError,
);
});
packages/ql3-runtime-core/test/clusterLegacyEnvMigrationPlan.test.cjs
+10
View File
@@ -4,6 +4,7 @@ const { test } = require('node:test');
const {
CLUSTER_LEGACY_ENV_MIGRATION_PLAN_SCHEMA,
MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BYTES,
MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BINDINGS,
MAX_CLUSTER_LEGACY_ENV_SOURCE_ROWS,
MAX_CLUSTER_LEGACY_ENV_TASKS,
MAX_CLUSTER_LEGACY_ENV_TRIGGERS,
@@ -111,6 +112,15 @@ test('enforces source consistency and router-safe bounded targets', () => {
sourceRowCount: MAX_CLUSTER_LEGACY_ENV_SOURCE_ROWS + 1,
},
},
{
source: {
...intent().source,
sourceRowCount: MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BINDINGS + 1,
activeRowCount: MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BINDINGS + 1,
disabledRowCount: 0,
effectiveBindingCount: MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BINDINGS + 1,
},
},
{
target: {
...intent().target,
packages/ql3-runtime-core/test/environmentBundle.test.cjs
+58
View File
@@ -0,0 +1,58 @@
'use strict';
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
ENVIRONMENT_BUNDLE_SCHEMA,
InvalidEnvironmentBundleError,
parseEnvironmentBundle,
serializeEnvironmentBundle,
} = require('../dist/secret/environmentBundle');
test('canonicalizes one opaque environment bundle without external authority', () => {
const serialized = serializeEnvironmentBundle({
schema: ENVIRONMENT_BUNDLE_SCHEMA,
entries: [
{ name: 'TOKEN', value: 'secret' },
{ name: 'EMPTY', value: '' },
],
});
assert.deepEqual(parseEnvironmentBundle(serialized), {
schema: ENVIRONMENT_BUNDLE_SCHEMA,
entries: [
{ name: 'EMPTY', value: '' },
{ name: 'TOKEN', value: 'secret' },
],
});
});
test('rejects duplicate, reserved, widened and over-budget bundle entries', () => {
const values = [
{ schema: ENVIRONMENT_BUNDLE_SCHEMA, entries: [] },
{
schema: ENVIRONMENT_BUNDLE_SCHEMA,
entries: [
{ name: 'TOKEN', value: 'a' },
{ name: 'TOKEN', value: 'b' },
],
},
{
schema: ENVIRONMENT_BUNDLE_SCHEMA,
entries: [{ name: 'QL3_TOKEN', value: 'a' }],
},
{
schema: ENVIRONMENT_BUNDLE_SCHEMA,
entries: [{ name: 'TOKEN', value: 'x'.repeat(16 * 1024 + 1) }],
},
{
schema: ENVIRONMENT_BUNDLE_SCHEMA,
entries: [{ name: 'TOKEN', value: 'a', secretRef: 'forbidden' }],
},
];
for (const value of values) {
assert.throws(
() => serializeEnvironmentBundle(value),
InvalidEnvironmentBundleError,
);
}
});
packages/ql3-runtime-core/test/remoteSecretDelivery.test.cjs
+104 -36
View File
@@ -14,6 +14,11 @@ const { createSecretRef } = require('../dist/secret/secretReference');
const SESSION_ID = '018f0000-0000-7000-8000-000000000001';
const DIGEST = 'a'.repeat(64);
const SECRET_REF = createSecretRef({ projectId: 'project-1', name: 'token' });
const BUNDLE_REF = createSecretRef({
projectId: 'project-1',
name: 'legacy-env-bundle',
version: 7,
});
function command(overrides = {}) {
return {
@@ -31,13 +36,14 @@ function command(overrides = {}) {
leaseToken: 'worker_generated_lease_capability_0000000000000001',
expectedLeaseVersion: 4,
secretRefs: [SECRET_REF],
environmentBundleRefs: [],
...overrides,
};
}
test('creates a versioned request without duplicating path-bound identity', () => {
const body = createRemoteWorkerSecretDeliveryRequestBody(command());
assert.equal(body.schema, 'qinglong/remote-secret-delivery@v1');
assert.equal(body.schema, 'qinglong/remote-secret-delivery@v2');
assert.equal('workerId' in body, false);
assert.equal('workerSessionId' in body, false);
assert.deepEqual(body.secretRefs, [SECRET_REF]);
@@ -45,67 +51,129 @@ test('creates a versioned request without duplicating path-bound identity', () =
});
test('parses only an exact authority and ordered Secret set', () => {
const response = createRemoteWorkerSecretDeliveryResponseBody({
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
executionDigest: DIGEST,
values: [{ secretRef: SECRET_REF, value: 'private-value' }],
}, [SECRET_REF]);
const response = createRemoteWorkerSecretDeliveryResponseBody(
{
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
executionDigest: DIGEST,
values: [{ secretRef: SECRET_REF, value: 'private-value' }],
environmentBundles: [],
},
{ secretRefs: [SECRET_REF], environmentBundleRefs: [] },
);
const parsed = parseRemoteWorkerSecretDeliveryResponse(
JSON.stringify(response),
{
runId: 'run-1', attemptId: 'attempt-1', offerId: 'offer-1',
executionDigest: DIGEST, secretRefs: [SECRET_REF],
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
executionDigest: DIGEST,
secretRefs: [SECRET_REF],
environmentBundleRefs: [],
},
);
assert.deepEqual(parsed.values, [
{ secretRef: SECRET_REF, value: 'private-value' },
]);
assert.throws(
() => parseRemoteWorkerSecretDeliveryResponse(JSON.stringify(response), {
runId: 'run-other', attemptId: 'attempt-1', offerId: 'offer-1',
executionDigest: DIGEST, secretRefs: [SECRET_REF],
}),
() =>
parseRemoteWorkerSecretDeliveryResponse(JSON.stringify(response), {
runId: 'run-other',
attemptId: 'attempt-1',
offerId: 'offer-1',
executionDigest: DIGEST,
secretRefs: [SECRET_REF],
environmentBundleRefs: [],
}),
/authority does not match/,
);
});
test('rejects duplicate, cross-project and oversized delivery input', () => {
assert.throws(
() => normalizeRemoteWorkerSecretDeliveryCommand(command({
secretRefs: [SECRET_REF, SECRET_REF],
})),
() =>
normalizeRemoteWorkerSecretDeliveryCommand(
command({
secretRefs: [SECRET_REF, SECRET_REF],
}),
),
/secretRefs are invalid/,
);
const foreign = createSecretRef({ projectId: 'project-2', name: 'token' });
assert.throws(
() => normalizeRemoteWorkerSecretDeliveryCommand(command({
secretRefs: [foreign],
})),
() =>
normalizeRemoteWorkerSecretDeliveryCommand(
command({
secretRefs: [foreign],
}),
),
/project is invalid/,
);
assert.throws(
() => parseRemoteWorkerSecretDeliveryResponse(
Buffer.alloc(MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES + 1),
{
runId: 'run-1', attemptId: 'attempt-1', offerId: 'offer-1',
executionDigest: DIGEST, secretRefs: [SECRET_REF],
},
),
() =>
parseRemoteWorkerSecretDeliveryResponse(
Buffer.alloc(MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES + 1),
{
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
executionDigest: DIGEST,
secretRefs: [SECRET_REF],
environmentBundleRefs: [],
},
),
/byte size/,
);
const refs = Array.from({ length: 5 }, (_, index) =>
createSecretRef({ projectId: 'project-1', name: `item-${index}` }));
createSecretRef({ projectId: 'project-1', name: `item-${index}` }),
);
assert.throws(
() => createRemoteWorkerSecretDeliveryResponseBody({
runId: 'run-1', attemptId: 'attempt-1', offerId: 'offer-1',
executionDigest: DIGEST,
values: refs.map((secretRef) => ({
secretRef,
value: 'x'.repeat(16 * 1024),
})),
}, refs),
() =>
createRemoteWorkerSecretDeliveryResponseBody(
{
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
executionDigest: DIGEST,
values: refs.map((secretRef) => ({
secretRef,
value: 'x'.repeat(16 * 1024),
})),
environmentBundles: [],
},
{ secretRefs: refs, environmentBundleRefs: [] },
),
/byte budget/,
);
});
test('keeps one environment bundle in a distinct bounded authority role', () => {
const normalized = normalizeRemoteWorkerSecretDeliveryCommand(
command({
secretRefs: [],
environmentBundleRefs: [BUNDLE_REF],
}),
);
assert.deepEqual(normalized.environmentBundleRefs, [BUNDLE_REF]);
assert.throws(
() =>
normalizeRemoteWorkerSecretDeliveryCommand(
command({
secretRefs: [BUNDLE_REF],
environmentBundleRefs: [BUNDLE_REF],
}),
),
/roles overlap/,
);
assert.throws(
() =>
normalizeRemoteWorkerSecretDeliveryCommand(
command({
secretRefs: [],
environmentBundleRefs: [],
}),
),
/set is empty/,
);
});
packages/ql3-runtime-core/test/taskSpecSemantic.test.cjs
+57 -6
View File
@@ -1,6 +1,7 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const { createLocalSecretRef } = require('../dist/secret/localSecret');
const { createSecretRef } = require('../dist/secret/secretReference');
const {
BUILT_IN_COMMAND_TASK_SPEC_SCHEMA,
InvalidTaskSpecSemanticError,
@@ -231,17 +232,67 @@ test('canonicalizes an optional bounded Remote Worker PlacementSpec in command s
preferred: [{ labels: { tier: 'edge' }, weight: 5 }],
});
assert.throws(
() => registry.normalize(context({
() =>
registry.normalize(
context({
spec: {
schema: BUILT_IN_COMMAND_TASK_SPEC_SCHEMA,
config: {
command: { kind: 'argv', file: '/bin/echo', args: [] },
placement: {
required: {
runtimes: [{ name: 'node', versionRange: 'not-semver' }],
},
},
},
},
}),
),
InvalidTaskSpecSemanticError,
);
});
test('accepts only a same-Project version-pinned environment bundle reference', () => {
const registry = createBuiltInTaskSpecSemanticRegistry();
const environmentBundleRef = createSecretRef({
projectId: 'default',
name: 'legacy-env-bundle',
version: 3,
});
const normalized = registry.normalize(
context({
spec: {
schema: BUILT_IN_COMMAND_TASK_SPEC_SCHEMA,
config: {
command: { kind: 'argv', file: '/bin/echo', args: [] },
placement: {
required: { runtimes: [{ name: 'node', versionRange: 'not-semver' }] },
},
environmentBundleRef,
},
},
})),
InvalidTaskSpecSemanticError,
}),
);
assert.equal(normalized.config.environmentBundleRef, environmentBundleRef);
for (const invalidRef of [
createSecretRef({ projectId: 'default', name: 'legacy-env-bundle' }),
createSecretRef({
projectId: 'other',
name: 'legacy-env-bundle',
version: 3,
}),
]) {
assert.throws(
() =>
registry.normalize(
context({
spec: {
schema: BUILT_IN_COMMAND_TASK_SPEC_SCHEMA,
config: {
command: { kind: 'argv', file: '/bin/echo', args: [] },
environmentBundleRef: invalidRef,
},
},
}),
),
/environmentBundleRef/,
);
}
});
packages/ql3-worker-runtime/src/remote-execution/executionContextMaterializer.ts
+135 -48
View File
@@ -1,8 +1,10 @@
// Remote Execution owns bounded Secret and Artifact context materialization.
import {
MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES,
MAX_LOCAL_DISPATCH_ENVIRONMENT_ENTRIES,
MAX_LOCAL_DISPATCH_SECRET_REFS,
} from '@qinglong/runtime-core/local-dispatch';
import { parseEnvironmentBundle } from '@qinglong/runtime-core/environment-bundle';
import type { ClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import { createClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import { assertRunDispatchId } from '@qinglong/runtime-core/run-dispatch-lease';
@@ -17,20 +19,27 @@ export interface WorkerRemoteSecretResolution {
secretRef: string;
value: string;
}>[];
readonly environmentBundles: readonly Readonly<{
secretRef: string;
value: string;
}>[];
readonly dispose?: () => Promise<void> | void;
}
export interface WorkerRemoteSecretEnvironmentProvider {
resolve(request: Readonly<{
projectId: string;
taskId: string;
taskRevision: string;
runId: string;
attemptId: string;
offerId: string;
executionDigest: string;
secretRefs: readonly string[];
}>): Promise<WorkerRemoteSecretResolution | undefined>;
resolve(
request: Readonly<{
projectId: string;
taskId: string;
taskRevision: string;
runId: string;
attemptId: string;
offerId: string;
executionDigest: string;
secretRefs: readonly string[];
environmentBundleRefs: readonly string[];
}>,
): Promise<WorkerRemoteSecretResolution | undefined>;
}
export interface WorkerRemoteLogArtifactPreparation {
@@ -42,12 +51,14 @@ export interface WorkerRemoteLogArtifactPreparation {
}
export interface WorkerRemoteLogArtifactAllocator {
prepare(request: Readonly<{
projectId: string;
runId: string;
attemptId: string;
offerId: string;
}>): Promise<WorkerRemoteLogArtifactPreparation | undefined>;
prepare(
request: Readonly<{
projectId: string;
runId: string;
attemptId: string;
offerId: string;
}>,
): Promise<WorkerRemoteLogArtifactPreparation | undefined>;
}
export interface BoundedWorkerRemoteExecutionContextMaterializerOptions {
@@ -86,11 +97,14 @@ function environmentValue(value: unknown): string {
async function disposeQuietly(
operation: (() => Promise<void> | void) | undefined,
): Promise<void> {
await Promise.resolve().then(() => operation?.()).catch(() => undefined);
await Promise.resolve()
.then(() => operation?.())
.catch(() => undefined);
}
export class BoundedWorkerRemoteExecutionContextMaterializer
implements WorkerRemoteExecutionContextMaterializer {
implements WorkerRemoteExecutionContextMaterializer
{
private readonly artifacts: WorkerRemoteLogArtifactAllocator;
private readonly secrets?: WorkerRemoteSecretEnvironmentProvider;
@@ -109,9 +123,11 @@ export class BoundedWorkerRemoteExecutionContextMaterializer
this.secrets = options.secrets;
}
async prepare(input: Readonly<{
offer: ClusterRemoteExecutionOffer;
}>): Promise<MaterializedWorkerRemoteExecutionContext> {
async prepare(
input: Readonly<{
offer: ClusterRemoteExecutionOffer;
}>,
): Promise<MaterializedWorkerRemoteExecutionContext> {
let offer: ClusterRemoteExecutionOffer;
try {
offer = createClusterRemoteExecutionOffer(input?.offer);
@@ -122,9 +138,17 @@ export class BoundedWorkerRemoteExecutionContextMaterializer
}
const bindings = offer.executionRevision.environment;
const secretRefs = Object.freeze([
...new Set(bindings.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [])),
...new Set(
bindings.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [],
),
),
]);
const environmentBundleRefs = Object.freeze(
offer.executionRevision.environmentBundleRef === undefined
? []
: [offer.executionRevision.environmentBundleRef],
);
if (secretRefs.length > MAX_LOCAL_DISPATCH_SECRET_REFS) {
throw new WorkerRemoteExecutionMaterializationError(
'environment_budget_exceeded',
@@ -132,23 +156,26 @@ export class BoundedWorkerRemoteExecutionContextMaterializer
}
let secretResolution: WorkerRemoteSecretResolution | undefined;
const secretByRef = new Map<string, string>();
if (secretRefs.length > 0) {
if (secretRefs.length > 0 || environmentBundleRefs.length > 0) {
if (!this.secrets) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_unavailable',
);
}
try {
secretResolution = await this.secrets.resolve(Object.freeze({
projectId: offer.candidate.projectId,
taskId: offer.candidate.taskId,
taskRevision: offer.candidate.taskRevision,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
executionDigest: offer.executionDigest,
secretRefs,
}));
secretResolution = await this.secrets.resolve(
Object.freeze({
projectId: offer.candidate.projectId,
taskId: offer.candidate.taskId,
taskRevision: offer.candidate.taskRevision,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
executionDigest: offer.executionDigest,
secretRefs,
environmentBundleRefs,
}),
);
} catch {
throw new WorkerRemoteExecutionMaterializationError(
'secret_unavailable',
@@ -160,10 +187,17 @@ export class BoundedWorkerRemoteExecutionContextMaterializer
);
}
if (
Object.keys(secretResolution).some((key) =>
key !== 'values' && key !== 'dispose') ||
Object.keys(secretResolution).some(
(key) =>
key !== 'values' &&
key !== 'environmentBundles' &&
key !== 'dispose',
) ||
!Array.isArray(secretResolution.values) ||
secretResolution.values.length !== secretRefs.length ||
!Array.isArray(secretResolution.environmentBundles) ||
secretResolution.environmentBundles.length !==
environmentBundleRefs.length ||
(secretResolution.dispose !== undefined &&
typeof secretResolution.dispose !== 'function')
) {
@@ -198,36 +232,89 @@ export class BoundedWorkerRemoteExecutionContextMaterializer
let environmentBytes = 0;
let environment: MaterializedWorkerRemoteExecutionContext['environment'];
try {
environment = Object.freeze(bindings.map((binding) => {
const value = binding.kind === 'public'
? binding.value
: secretByRef.get(binding.secretRef);
const names = new Set<string>();
const materialized = bindings.map((binding) => {
const value =
binding.kind === 'public'
? binding.value
: secretByRef.get(binding.secretRef);
if (value === undefined) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
environmentBytes += Buffer.byteLength(binding.name, 'utf8') +
environmentBytes +=
Buffer.byteLength(binding.name, 'utf8') +
Buffer.byteLength(value, 'utf8');
names.add(binding.name);
if (environmentBytes > MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES) {
throw new WorkerRemoteExecutionMaterializationError(
'environment_budget_exceeded',
);
}
return Object.freeze({ name: binding.name, value });
}));
});
for (const entry of secretResolution?.environmentBundles ?? []) {
if (
!entry ||
typeof entry !== 'object' ||
Object.keys(entry).length !== 2 ||
!Object.hasOwn(entry, 'secretRef') ||
!Object.hasOwn(entry, 'value') ||
typeof entry.secretRef !== 'string' ||
!environmentBundleRefs.includes(entry.secretRef) ||
typeof entry.value !== 'string'
) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
let bundle;
try {
bundle = parseEnvironmentBundle(entry.value);
} catch {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
for (const binding of bundle.entries) {
if (names.has(binding.name)) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
names.add(binding.name);
environmentBytes +=
Buffer.byteLength(binding.name, 'utf8') +
Buffer.byteLength(binding.value, 'utf8');
if (
materialized.length >= MAX_LOCAL_DISPATCH_ENVIRONMENT_ENTRIES ||
environmentBytes > MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES
) {
throw new WorkerRemoteExecutionMaterializationError(
'environment_budget_exceeded',
);
}
materialized.push(
Object.freeze({ name: binding.name, value: binding.value }),
);
}
}
environment = Object.freeze(materialized);
} catch (error) {
await disposeQuietly(secretResolution?.dispose);
throw error;
}
let artifact: WorkerRemoteLogArtifactPreparation | undefined;
try {
artifact = await this.artifacts.prepare(Object.freeze({
projectId: offer.candidate.projectId,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
}));
artifact = await this.artifacts.prepare(
Object.freeze({
projectId: offer.candidate.projectId,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
}),
);
} catch {
await disposeQuietly(secretResolution?.dispose);
throw new WorkerRemoteExecutionMaterializationError(
packages/ql3-worker-runtime/src/remote-execution/transport/remoteSecretHttpsProvider.ts
+37 -11
View File
@@ -36,7 +36,8 @@ export interface WorkerRemoteSecretHttpsProviderOptions {
}
export class WorkerRemoteSecretHttpsProvider
implements WorkerRemoteSecretEnvironmentProvider {
implements WorkerRemoteSecretEnvironmentProvider
{
private readonly client: Pick<WorkerIngressHttpsClient, 'postJson'>;
private readonly inbox: Pick<WorkerRemoteExecutionInbox, 'readOffer'>;
@@ -45,13 +46,15 @@ export class WorkerRemoteSecretHttpsProvider
!options ||
typeof options.client?.postJson !== 'function' ||
typeof options.inbox?.readOffer !== 'function'
) throw new WorkerRemoteSecretHttpsProviderError('invalid_configuration');
)
throw new WorkerRemoteSecretHttpsProviderError('invalid_configuration');
this.client = options.client;
this.inbox = options.inbox;
}
async resolve(request: Parameters<WorkerRemoteSecretEnvironmentProvider['resolve']>[0])
: Promise<WorkerRemoteSecretResolution | undefined> {
async resolve(
request: Parameters<WorkerRemoteSecretEnvironmentProvider['resolve']>[0],
): Promise<WorkerRemoteSecretResolution | undefined> {
let record;
try {
record = await this.inbox.readOffer(request.offerId);
@@ -68,9 +71,17 @@ export class WorkerRemoteSecretHttpsProvider
throw new WorkerRemoteSecretHttpsProviderError('authority_mismatch');
}
const expectedRefs = Object.freeze([
...new Set(offer.executionRevision.environment.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [])),
...new Set(
offer.executionRevision.environment.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [],
),
),
]);
const expectedEnvironmentBundleRefs = Object.freeze(
offer.executionRevision.environmentBundleRef === undefined
? []
: [offer.executionRevision.environmentBundleRef],
);
if (
offer.offerId !== request.offerId ||
offer.executionDigest !== request.executionDigest ||
@@ -79,10 +90,14 @@ export class WorkerRemoteSecretHttpsProvider
offer.candidate.taskRevision !== request.taskRevision ||
offer.candidate.runId !== request.runId ||
offer.candidate.attemptId !== request.attemptId ||
JSON.stringify(expectedRefs) !== JSON.stringify(request.secretRefs)
) throw new WorkerRemoteSecretHttpsProviderError('authority_mismatch');
JSON.stringify(expectedRefs) !== JSON.stringify(request.secretRefs) ||
JSON.stringify(expectedEnvironmentBundleRefs) !==
JSON.stringify(request.environmentBundleRefs)
)
throw new WorkerRemoteSecretHttpsProviderError('authority_mismatch');
const path = `/api/v3/worker-ingress/workers/${offer.worker.workerId}` +
const path =
`/api/v3/worker-ingress/workers/${offer.worker.workerId}` +
`/sessions/${offer.worker.sessionId}/secrets`;
const body = createRemoteWorkerSecretDeliveryRequestBody({
workerId: offer.worker.workerId,
@@ -99,6 +114,7 @@ export class WorkerRemoteSecretHttpsProvider
leaseToken: offer.leaseToken,
expectedLeaseVersion: offer.lease.version,
secretRefs: expectedRefs,
environmentBundleRefs: expectedEnvironmentBundleRefs,
});
let serialized: Uint8Array;
try {
@@ -119,11 +135,21 @@ export class WorkerRemoteSecretHttpsProvider
offerId: offer.offerId,
executionDigest: offer.executionDigest,
secretRefs: expectedRefs,
environmentBundleRefs: expectedEnvironmentBundleRefs,
});
const values = Object.freeze(delivered.values.map((entry) =>
Object.freeze({ secretRef: entry.secretRef, value: entry.value })));
const values = Object.freeze(
delivered.values.map((entry) =>
Object.freeze({ secretRef: entry.secretRef, value: entry.value }),
),
);
const environmentBundles = Object.freeze(
delivered.environmentBundles.map((entry) =>
Object.freeze({ secretRef: entry.secretRef, value: entry.value }),
),
);
return Object.freeze({
values,
environmentBundles,
dispose() {
// JavaScript strings cannot be zeroized. Drop all retained references;
// the transport bytes were already scrubbed by the parser.
packages/ql3-worker-runtime/src/remote-execution/transport/workerIngressHttpsClient.ts
+3 -1
View File
@@ -17,7 +17,9 @@ const MAX_TLS_MATERIAL_BYTES = 1024 * 1024;
const MAX_REQUEST_BYTES = 4096;
const HARD_MAX_REQUEST_BYTES = 64 * 1024;
const HARD_MAX_STREAM_REQUEST_BYTES = 64 * 1024 * 1024 + 4 * 1024 + 4;
const MAX_RESPONSE_BYTES = 128 * 1024;
// JSON routes retain their own smaller requested caps. The shared ceiling must
// also admit one bounded environment bundle response.
const MAX_RESPONSE_BYTES = 256 * 1024;
const CREDENTIAL_POOL_KEY = Symbol('qinglong.worker-ingress-credential-pool-key');
export const WORKER_INGRESS_ARTIFACT_CONTENT_TYPE =
packages/ql3-worker-runtime/test/executionContextMaterializer.test.cjs
+129 -22
View File
@@ -11,9 +11,11 @@ const {
const {
digestRunDispatchLeaseToken,
} = require('@qinglong/runtime-core/run-dispatch-lease');
const { createSecretRef } = require('@qinglong/runtime-core/secret-reference');
const {
createSecretRef,
} = require('@qinglong/runtime-core/secret-reference');
ENVIRONMENT_BUNDLE_SCHEMA,
serializeEnvironmentBundle,
} = require('@qinglong/runtime-core/environment-bundle');
const {
BoundedWorkerRemoteExecutionContextMaterializer,
} = require('../dist/remote-execution/remoteOfferDeliveryEntrypoint');
@@ -27,7 +29,7 @@ function secret(name) {
return createSecretRef({ projectId: 'project-1', name });
}
function offer(environment) {
function offer(environment, environmentBundleRef) {
const executionRevision = createClusterTaskExecutionRevision({
projectId: 'project-1',
taskId: 'task-1',
@@ -38,6 +40,7 @@ function offer(environment) {
planSchema: 'qinglong/command-execution@v1',
command: { kind: 'argv', file: '/bin/true', args: [] },
environment,
...(environmentBundleRef === undefined ? {} : { environmentBundleRef }),
createdAtMs: 1,
});
return createClusterRemoteExecutionOffer({
@@ -100,7 +103,10 @@ test('resolves deduplicated Secrets before allocating one Attempt log', async ()
secretRequest = request;
return {
values: [{ secretRef, value: 'resolved-value' }],
dispose() { events.push('dispose-secrets'); },
environmentBundles: [],
dispose() {
events.push('dispose-secrets');
},
};
},
},
@@ -110,8 +116,13 @@ test('resolves deduplicated Secrets before allocating one Attempt log', async ()
artifactRequest = request;
return {
logArtifactId: 'remote-log-1',
takeOutput() { events.push('take-output'); return output; },
release() { events.push('release-artifact'); },
takeOutput() {
events.push('take-output');
return output;
},
release() {
events.push('release-artifact');
},
};
},
},
@@ -129,6 +140,7 @@ test('resolves deduplicated Secrets before allocating one Attempt log', async ()
offerId: 'offer-materializer-1',
executionDigest: acceptedOffer.executionDigest,
secretRefs: [secretRef],
environmentBundleRefs: [],
});
assert.deepEqual(artifactRequest, {
projectId: 'project-1',
@@ -136,7 +148,10 @@ test('resolves deduplicated Secrets before allocating one Attempt log', async ()
attemptId: 'attempt-1',
offerId: 'offer-materializer-1',
});
assert.equal(JSON.stringify([secretRequest, artifactRequest]).includes(LEASE_TOKEN), false);
assert.equal(
JSON.stringify([secretRequest, artifactRequest]).includes(LEASE_TOKEN),
false,
);
assert.deepEqual(context.environment, [
{ name: 'PUBLIC', value: 'visible' },
{ name: 'SECRET_A', value: 'resolved-value' },
@@ -149,18 +164,26 @@ test('resolves deduplicated Secrets before allocating one Attempt log', async ()
await context.dispose();
await context.dispose();
assert.deepEqual(events.slice(2).sort(), [
'dispose-secrets', 'release-artifact', 'take-output',
'dispose-secrets',
'release-artifact',
'take-output',
]);
});
test('fails before Artifact allocation when Secret authority is unavailable', async () => {
let artifacts = 0;
const materializer = new BoundedWorkerRemoteExecutionContextMaterializer({
artifacts: { async prepare() { artifacts += 1; } },
artifacts: {
async prepare() {
artifacts += 1;
},
},
});
await assert.rejects(
materializer.prepare({
offer: offer([{ name: 'SECRET', kind: 'secret', secretRef: secret('one') }]),
offer: offer([
{ name: 'SECRET', kind: 'secret', secretRef: secret('one') },
]),
}),
/secret_unavailable/,
);
@@ -179,11 +202,18 @@ test('disposes malformed Secret and Artifact responses without exposing values',
{ secretRef, value: 'first' },
{ secretRef, value: 'duplicate' },
],
dispose() { disposedSecrets += 1; },
environmentBundles: [],
dispose() {
disposedSecrets += 1;
},
};
},
},
artifacts: { async prepare() { throw new Error('must not allocate'); } },
artifacts: {
async prepare() {
throw new Error('must not allocate');
},
},
});
await assert.rejects(
malformedSecrets.prepare({
@@ -196,16 +226,20 @@ test('disposes malformed Secret and Artifact responses without exposing values',
);
assert.equal(disposedSecrets, 1);
const malformedArtifact = new BoundedWorkerRemoteExecutionContextMaterializer({
artifacts: {
async prepare() {
return {
logArtifactId: 'x'.repeat(37),
release() { releasedArtifact += 1; },
};
const malformedArtifact = new BoundedWorkerRemoteExecutionContextMaterializer(
{
artifacts: {
async prepare() {
return {
logArtifactId: 'x'.repeat(37),
release() {
releasedArtifact += 1;
},
};
},
},
},
});
);
await assert.rejects(
malformedArtifact.prepare({
offer: offer([{ name: 'PUBLIC', kind: 'public', value: 'visible' }]),
@@ -231,11 +265,18 @@ test('enforces the resolved environment byte budget before Artifact allocation',
secretRef,
value: 'x'.repeat(16 * 1024),
})),
dispose() { disposed += 1; },
environmentBundles: [],
dispose() {
disposed += 1;
},
};
},
},
artifacts: { async prepare() { artifacts += 1; } },
artifacts: {
async prepare() {
artifacts += 1;
},
},
});
await assert.rejects(
materializer.prepare({ offer: offer(bindings) }),
@@ -244,3 +285,69 @@ test('enforces the resolved environment byte budget before Artifact allocation',
assert.equal(disposed, 1);
assert.equal(artifacts, 0);
});
test('expands one opaque bundle in memory and rejects name collisions', async () => {
const environmentBundleRef = createSecretRef({
projectId: 'project-1',
name: 'legacy-env-bundle',
version: 2,
});
const bundle = serializeEnvironmentBundle({
schema: ENVIRONMENT_BUNDLE_SCHEMA,
entries: [
{ name: 'LEGACY_TOKEN', value: 'private' },
{ name: 'LEGACY_MODE', value: 'compat' },
],
});
const materializer = new BoundedWorkerRemoteExecutionContextMaterializer({
secrets: {
async resolve(request) {
assert.deepEqual(request.secretRefs, []);
assert.deepEqual(request.environmentBundleRefs, [environmentBundleRef]);
return {
values: [],
environmentBundles: [
{ secretRef: environmentBundleRef, value: bundle },
],
};
},
},
artifacts: {
async prepare() {
return {
logArtifactId: 'remote-log-bundle',
takeOutput() {
return {
logArtifactId: 'remote-log-bundle',
async write() {},
async close() {},
};
},
async release() {},
};
},
},
});
const context = await materializer.prepare({
offer: offer(
[{ name: 'PUBLIC', kind: 'public', value: 'visible' }],
environmentBundleRef,
),
});
assert.deepEqual(context.environment, [
{ name: 'PUBLIC', value: 'visible' },
{ name: 'LEGACY_MODE', value: 'compat' },
{ name: 'LEGACY_TOKEN', value: 'private' },
]);
await context.dispose();
await assert.rejects(
materializer.prepare({
offer: offer(
[{ name: 'LEGACY_MODE', kind: 'public', value: 'current' }],
environmentBundleRef,
),
}),
/secret_response_invalid/,
);
});
packages/ql3-worker-runtime/test/remoteSecretHttpsProvider.test.cjs
+86 -31
View File
@@ -24,31 +24,52 @@ const SECRET_REF = createSecretRef({ projectId: 'project-1', name: 'token' });
function acceptedOffer() {
const executionRevision = createClusterTaskExecutionRevision({
projectId: 'project-1', taskId: 'task-1', taskRevision: TASK_REVISION,
sourceRevision: 1, sourceContentDigest: SOURCE_DIGEST,
executorType: 'remote_worker', planSchema: 'qinglong/command-execution@v1',
projectId: 'project-1',
taskId: 'task-1',
taskRevision: TASK_REVISION,
sourceRevision: 1,
sourceContentDigest: SOURCE_DIGEST,
executorType: 'remote_worker',
planSchema: 'qinglong/command-execution@v1',
command: { kind: 'argv', file: '/bin/true', args: [] },
environment: [{ name: 'TOKEN', kind: 'secret', secretRef: SECRET_REF }],
createdAtMs: 1,
});
return createClusterRemoteExecutionOffer({
offerId: 'offer-1', deliveryKind: 'new_claim',
offerId: 'offer-1',
deliveryKind: 'new_claim',
executionDigest: executionRevision.contentDigest,
candidate: {
runId: 'run-1', attemptId: 'attempt-1', projectId: 'project-1',
taskId: 'task-1', taskRevision: TASK_REVISION, priority: 1,
queuedAtMs: 10, attemptCreatedAtMs: 11, attemptNumber: 1,
runId: 'run-1',
attemptId: 'attempt-1',
projectId: 'project-1',
taskId: 'task-1',
taskRevision: TASK_REVISION,
priority: 1,
queuedAtMs: 10,
attemptCreatedAtMs: 11,
attemptNumber: 1,
executorType: 'remote_worker',
},
worker: { workerId: 'edge-1', sessionId: SESSION_ID, generation: 2 },
lease: {
attemptId: 'attempt-1', runId: 'run-1', status: 'leased', version: 4,
leaseGeneration: 3, workerId: 'edge-1', workerSessionId: SESSION_ID,
workerGeneration: 2, leaseTokenDigest: digestRunDispatchLeaseToken(LEASE_TOKEN),
acquiredAtMs: 20, renewedAtMs: 20, expiresAtMs: 30_020,
attemptId: 'attempt-1',
runId: 'run-1',
status: 'leased',
version: 4,
leaseGeneration: 3,
workerId: 'edge-1',
workerSessionId: SESSION_ID,
workerGeneration: 2,
leaseTokenDigest: digestRunDispatchLeaseToken(LEASE_TOKEN),
acquiredAtMs: 20,
renewedAtMs: 20,
expiresAtMs: 30_020,
updatedAtMs: 20,
},
leaseToken: LEASE_TOKEN, executionRevision, placementScore: 0,
leaseToken: LEASE_TOKEN,
executionRevision,
placementScore: 0,
});
}
@@ -62,6 +83,7 @@ function requestFor(offer) {
offerId: offer.offerId,
executionDigest: offer.executionDigest,
secretRefs: [SECRET_REF],
environmentBundleRefs: [],
};
}
@@ -78,12 +100,17 @@ test('rehydrates lease authority from inbox and delivers one exact Secret batch'
client: {
async postJson(request) {
transport = request;
return Buffer.from(JSON.stringify({
schema: 'qinglong/remote-secret-delivery@v1',
runId: 'run-1', attemptId: 'attempt-1', offerId: 'offer-1',
executionDigest: offer.executionDigest,
values: [{ secretRef: SECRET_REF, value: 'resolved-value' }],
}));
return Buffer.from(
JSON.stringify({
schema: 'qinglong/remote-secret-delivery@v2',
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
executionDigest: offer.executionDigest,
values: [{ secretRef: SECRET_REF, value: 'resolved-value' }],
environmentBundles: [],
}),
);
},
},
});
@@ -91,7 +118,11 @@ test('rehydrates lease authority from inbox and delivers one exact Secret batch'
assert.deepEqual(resolution.values, [
{ secretRef: SECRET_REF, value: 'resolved-value' },
]);
assert.equal(transport.path.endsWith(`/sessions/${SESSION_ID}/secrets`), true);
assert.deepEqual(resolution.environmentBundles, []);
assert.equal(
transport.path.endsWith(`/sessions/${SESSION_ID}/secrets`),
true,
);
assert.equal(transport.body.leaseToken, LEASE_TOKEN);
assert.equal(transport.maximumRequestBytes, 64 * 1024);
assert.equal(JSON.stringify(requestFor(offer)).includes(LEASE_TOKEN), false);
@@ -102,9 +133,15 @@ test('rejects a stale inbox identity before sending the capability', async () =>
let calls = 0;
const provider = new WorkerRemoteSecretHttpsProvider({
inbox: {
async readOffer() { return { state: 'starting_acknowledged', offer }; },
async readOffer() {
return { state: 'starting_acknowledged', offer };
},
},
client: {
async postJson() {
calls += 1;
},
},
client: { async postJson() { calls += 1; } },
});
await assert.rejects(
provider.resolve({ ...requestFor(offer), executionDigest: 'b'.repeat(64) }),
@@ -117,16 +154,23 @@ test('rejects response authority drift and does not return plaintext', async ()
const offer = acceptedOffer();
const provider = new WorkerRemoteSecretHttpsProvider({
inbox: {
async readOffer() { return { state: 'starting_acknowledged', offer }; },
async readOffer() {
return { state: 'starting_acknowledged', offer };
},
},
client: {
async postJson() {
return Buffer.from(JSON.stringify({
schema: 'qinglong/remote-secret-delivery@v1',
runId: 'run-other', attemptId: 'attempt-1', offerId: 'offer-1',
executionDigest: offer.executionDigest,
values: [{ secretRef: SECRET_REF, value: 'must-not-escape' }],
}));
return Buffer.from(
JSON.stringify({
schema: 'qinglong/remote-secret-delivery@v2',
runId: 'run-other',
attemptId: 'attempt-1',
offerId: 'offer-1',
executionDigest: offer.executionDigest,
values: [{ secretRef: SECRET_REF, value: 'must-not-escape' }],
environmentBundles: [],
}),
);
},
},
});
@@ -138,10 +182,21 @@ test('does not fetch Secrets before starting ACK or after the launch barrier', a
for (const state of ['accepted', 'launching']) {
let calls = 0;
const provider = new WorkerRemoteSecretHttpsProvider({
inbox: { async readOffer() { return { state, offer }; } },
client: { async postJson() { calls += 1; } },
inbox: {
async readOffer() {
return { state, offer };
},
},
client: {
async postJson() {
calls += 1;
},
},
});
await assert.rejects(provider.resolve(requestFor(offer)), /offer_unavailable/);
await assert.rejects(
provider.resolve(requestFor(offer)),
/offer_unavailable/,
);
assert.equal(calls, 0);
}
});
packages/ql3-worker-runtime/test/workerIngressHttpsClient.test.cjs
+10 -1
View File
@@ -267,11 +267,20 @@ test('disposes provider-owned credential material on success and rejection', asy
rejected.postJson({
path: COMPLETION_PATH,
body: {},
maximumResponseBytes: 1024,
maximumResponseBytes: 256 * 1024,
}),
/credentials_unavailable/,
);
assert.equal(rejectedDisposals, 1);
await assert.rejects(
rejected.postJson({
path: COMPLETION_PATH,
body: {},
maximumResponseBytes: 256 * 1024 + 1,
}),
/request_rejected/,
);
assert.equal(rejectedDisposals, 1);
} finally {
rejected.close();
}