open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-20 16:07:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): add opaque cluster environment bundle delivery

Browse Source
This commit is contained in:
whyour
2026-08-24 19:31:03 +08:00
parent cf2c0ec7b3
commit 4abf125ce9
36 changed files with 1682 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-runtime-core/test/clusterExecutionRevision.test.cjs
+76 -21
View File
@@ -44,15 +44,55 @@ function definition() {
});
return {
registry,
record: createTaskDefinitionRecord({
...command,
spec: registry.normalize({
projectId: command.projectId,
taskId: command.taskId,
kind: command.kind,
spec: command.spec,
}),
}, 90),
record: createTaskDefinitionRecord(
{
...command,
spec: registry.normalize({
projectId: command.projectId,
taskId: command.taskId,
kind: command.kind,
spec: command.spec,
}),
},
90,
),
};
}
function definitionWithBundle() {
const input = definition();
const environmentBundleRef = createSecretRef({
projectId: 'default',
name: 'legacy-env-bundle',
version: 4,
});
const spec = input.registry.normalize({
projectId: input.record.projectId,
taskId: input.record.taskId,
kind: input.record.kind,
spec: {
...input.record.spec,
config: { ...input.record.spec.config, environmentBundleRef },
},
});
return {
registry: input.registry,
environmentBundleRef,
record: createTaskDefinitionRecord(
{
projectId: input.record.projectId,
taskId: input.record.taskId,
expectedRevision: null,
mutationId: input.record.mutationId,
name: input.record.name,
kind: input.record.kind,
spec,
labels: input.record.labels,
enabled: input.record.enabled,
occurredAtMs: input.record.updatedAtMs,
},
input.record.createdAtMs,
),
};
}
@@ -70,6 +110,17 @@ test('compiles one digest-bound remote Worker execution revision', () => {
assert.deepEqual(normalizeClusterTaskExecutionRevision(revision), revision);
});
test('carries only the pinned environment bundle reference into Cluster plans', () => {
const input = definitionWithBundle();
const revision = compileClusterCommandTaskDefinition(
input.record,
input.registry,
);
assert.equal(revision.environmentBundleRef, input.environmentBundleRef);
assert.equal(JSON.stringify(revision).includes('legacy env value'), false);
assert.deepEqual(normalizeClusterTaskExecutionRevision(revision), revision);
});
test('rejects digest drift and cross-Project Secret references', () => {
const input = definition();
const revision = compileClusterCommandTaskDefinition(
@@ -77,21 +128,25 @@ test('rejects digest drift and cross-Project Secret references', () => {
input.registry,
);
assert.throws(
() => normalizeClusterTaskExecutionRevision({
...revision,
contentDigest: '0'.repeat(64),
}),
() =>
normalizeClusterTaskExecutionRevision({
...revision,
contentDigest: '0'.repeat(64),
}),
InvalidClusterExecutionRevisionError,
);
assert.throws(
() => normalizeClusterTaskExecutionRevision({
...revision,
environment: [{
kind: 'secret',
name: 'TOKEN',
secretRef: createSecretRef({ projectId: 'another', name: 'TOKEN' }),
}],
}),
() =>
normalizeClusterTaskExecutionRevision({
...revision,
environment: [
{
kind: 'secret',
name: 'TOKEN',
secretRef: createSecretRef({ projectId: 'another', name: 'TOKEN' }),
},
],
}),
InvalidClusterExecutionRevisionError,
);
});
packages/ql3-runtime-core/test/clusterLegacyEnvMigrationPlan.test.cjs
+10
View File
@@ -4,6 +4,7 @@ const { test } = require('node:test');
const {
CLUSTER_LEGACY_ENV_MIGRATION_PLAN_SCHEMA,
MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BYTES,
MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BINDINGS,
MAX_CLUSTER_LEGACY_ENV_SOURCE_ROWS,
MAX_CLUSTER_LEGACY_ENV_TASKS,
MAX_CLUSTER_LEGACY_ENV_TRIGGERS,
@@ -111,6 +112,15 @@ test('enforces source consistency and router-safe bounded targets', () => {
sourceRowCount: MAX_CLUSTER_LEGACY_ENV_SOURCE_ROWS + 1,
},
},
{
source: {
...intent().source,
sourceRowCount: MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BINDINGS + 1,
activeRowCount: MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BINDINGS + 1,
disabledRowCount: 0,
effectiveBindingCount: MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BINDINGS + 1,
},
},
{
target: {
...intent().target,
packages/ql3-runtime-core/test/environmentBundle.test.cjs
+58
View File
@@ -0,0 +1,58 @@
'use strict';
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
ENVIRONMENT_BUNDLE_SCHEMA,
InvalidEnvironmentBundleError,
parseEnvironmentBundle,
serializeEnvironmentBundle,
} = require('../dist/secret/environmentBundle');
test('canonicalizes one opaque environment bundle without external authority', () => {
const serialized = serializeEnvironmentBundle({
schema: ENVIRONMENT_BUNDLE_SCHEMA,
entries: [
{ name: 'TOKEN', value: 'secret' },
{ name: 'EMPTY', value: '' },
],
});
assert.deepEqual(parseEnvironmentBundle(serialized), {
schema: ENVIRONMENT_BUNDLE_SCHEMA,
entries: [
{ name: 'EMPTY', value: '' },
{ name: 'TOKEN', value: 'secret' },
],
});
});
test('rejects duplicate, reserved, widened and over-budget bundle entries', () => {
const values = [
{ schema: ENVIRONMENT_BUNDLE_SCHEMA, entries: [] },
{
schema: ENVIRONMENT_BUNDLE_SCHEMA,
entries: [
{ name: 'TOKEN', value: 'a' },
{ name: 'TOKEN', value: 'b' },
],
},
{
schema: ENVIRONMENT_BUNDLE_SCHEMA,
entries: [{ name: 'QL3_TOKEN', value: 'a' }],
},
{
schema: ENVIRONMENT_BUNDLE_SCHEMA,
entries: [{ name: 'TOKEN', value: 'x'.repeat(16 * 1024 + 1) }],
},
{
schema: ENVIRONMENT_BUNDLE_SCHEMA,
entries: [{ name: 'TOKEN', value: 'a', secretRef: 'forbidden' }],
},
];
for (const value of values) {
assert.throws(
() => serializeEnvironmentBundle(value),
InvalidEnvironmentBundleError,
);
}
});
packages/ql3-runtime-core/test/remoteSecretDelivery.test.cjs
+104 -36
View File
@@ -14,6 +14,11 @@ const { createSecretRef } = require('../dist/secret/secretReference');
const SESSION_ID = '018f0000-0000-7000-8000-000000000001';
const DIGEST = 'a'.repeat(64);
const SECRET_REF = createSecretRef({ projectId: 'project-1', name: 'token' });
const BUNDLE_REF = createSecretRef({
projectId: 'project-1',
name: 'legacy-env-bundle',
version: 7,
});
function command(overrides = {}) {
return {
@@ -31,13 +36,14 @@ function command(overrides = {}) {
leaseToken: 'worker_generated_lease_capability_0000000000000001',
expectedLeaseVersion: 4,
secretRefs: [SECRET_REF],
environmentBundleRefs: [],
...overrides,
};
}
test('creates a versioned request without duplicating path-bound identity', () => {
const body = createRemoteWorkerSecretDeliveryRequestBody(command());
assert.equal(body.schema, 'qinglong/remote-secret-delivery@v1');
assert.equal(body.schema, 'qinglong/remote-secret-delivery@v2');
assert.equal('workerId' in body, false);
assert.equal('workerSessionId' in body, false);
assert.deepEqual(body.secretRefs, [SECRET_REF]);
@@ -45,67 +51,129 @@ test('creates a versioned request without duplicating path-bound identity', () =
});
test('parses only an exact authority and ordered Secret set', () => {
const response = createRemoteWorkerSecretDeliveryResponseBody({
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
executionDigest: DIGEST,
values: [{ secretRef: SECRET_REF, value: 'private-value' }],
}, [SECRET_REF]);
const response = createRemoteWorkerSecretDeliveryResponseBody(
{
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
executionDigest: DIGEST,
values: [{ secretRef: SECRET_REF, value: 'private-value' }],
environmentBundles: [],
},
{ secretRefs: [SECRET_REF], environmentBundleRefs: [] },
);
const parsed = parseRemoteWorkerSecretDeliveryResponse(
JSON.stringify(response),
{
runId: 'run-1', attemptId: 'attempt-1', offerId: 'offer-1',
executionDigest: DIGEST, secretRefs: [SECRET_REF],
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
executionDigest: DIGEST,
secretRefs: [SECRET_REF],
environmentBundleRefs: [],
},
);
assert.deepEqual(parsed.values, [
{ secretRef: SECRET_REF, value: 'private-value' },
]);
assert.throws(
() => parseRemoteWorkerSecretDeliveryResponse(JSON.stringify(response), {
runId: 'run-other', attemptId: 'attempt-1', offerId: 'offer-1',
executionDigest: DIGEST, secretRefs: [SECRET_REF],
}),
() =>
parseRemoteWorkerSecretDeliveryResponse(JSON.stringify(response), {
runId: 'run-other',
attemptId: 'attempt-1',
offerId: 'offer-1',
executionDigest: DIGEST,
secretRefs: [SECRET_REF],
environmentBundleRefs: [],
}),
/authority does not match/,
);
});
test('rejects duplicate, cross-project and oversized delivery input', () => {
assert.throws(
() => normalizeRemoteWorkerSecretDeliveryCommand(command({
secretRefs: [SECRET_REF, SECRET_REF],
})),
() =>
normalizeRemoteWorkerSecretDeliveryCommand(
command({
secretRefs: [SECRET_REF, SECRET_REF],
}),
),
/secretRefs are invalid/,
);
const foreign = createSecretRef({ projectId: 'project-2', name: 'token' });
assert.throws(
() => normalizeRemoteWorkerSecretDeliveryCommand(command({
secretRefs: [foreign],
})),
() =>
normalizeRemoteWorkerSecretDeliveryCommand(
command({
secretRefs: [foreign],
}),
),
/project is invalid/,
);
assert.throws(
() => parseRemoteWorkerSecretDeliveryResponse(
Buffer.alloc(MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES + 1),
{
runId: 'run-1', attemptId: 'attempt-1', offerId: 'offer-1',
executionDigest: DIGEST, secretRefs: [SECRET_REF],
},
),
() =>
parseRemoteWorkerSecretDeliveryResponse(
Buffer.alloc(MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES + 1),
{
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
executionDigest: DIGEST,
secretRefs: [SECRET_REF],
environmentBundleRefs: [],
},
),
/byte size/,
);
const refs = Array.from({ length: 5 }, (_, index) =>
createSecretRef({ projectId: 'project-1', name: `item-${index}` }));
createSecretRef({ projectId: 'project-1', name: `item-${index}` }),
);
assert.throws(
() => createRemoteWorkerSecretDeliveryResponseBody({
runId: 'run-1', attemptId: 'attempt-1', offerId: 'offer-1',
executionDigest: DIGEST,
values: refs.map((secretRef) => ({
secretRef,
value: 'x'.repeat(16 * 1024),
})),
}, refs),
() =>
createRemoteWorkerSecretDeliveryResponseBody(
{
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
executionDigest: DIGEST,
values: refs.map((secretRef) => ({
secretRef,
value: 'x'.repeat(16 * 1024),
})),
environmentBundles: [],
},
{ secretRefs: refs, environmentBundleRefs: [] },
),
/byte budget/,
);
});
test('keeps one environment bundle in a distinct bounded authority role', () => {
const normalized = normalizeRemoteWorkerSecretDeliveryCommand(
command({
secretRefs: [],
environmentBundleRefs: [BUNDLE_REF],
}),
);
assert.deepEqual(normalized.environmentBundleRefs, [BUNDLE_REF]);
assert.throws(
() =>
normalizeRemoteWorkerSecretDeliveryCommand(
command({
secretRefs: [BUNDLE_REF],
environmentBundleRefs: [BUNDLE_REF],
}),
),
/roles overlap/,
);
assert.throws(
() =>
normalizeRemoteWorkerSecretDeliveryCommand(
command({
secretRefs: [],
environmentBundleRefs: [],
}),
),
/set is empty/,
);
});
packages/ql3-runtime-core/test/taskSpecSemantic.test.cjs
+57 -6
View File
@@ -1,6 +1,7 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const { createLocalSecretRef } = require('../dist/secret/localSecret');
const { createSecretRef } = require('../dist/secret/secretReference');
const {
BUILT_IN_COMMAND_TASK_SPEC_SCHEMA,
InvalidTaskSpecSemanticError,
@@ -231,17 +232,67 @@ test('canonicalizes an optional bounded Remote Worker PlacementSpec in command s
preferred: [{ labels: { tier: 'edge' }, weight: 5 }],
});
assert.throws(
() => registry.normalize(context({
() =>
registry.normalize(
context({
spec: {
schema: BUILT_IN_COMMAND_TASK_SPEC_SCHEMA,
config: {
command: { kind: 'argv', file: '/bin/echo', args: [] },
placement: {
required: {
runtimes: [{ name: 'node', versionRange: 'not-semver' }],
},
},
},
},
}),
),
InvalidTaskSpecSemanticError,
);
});
test('accepts only a same-Project version-pinned environment bundle reference', () => {
const registry = createBuiltInTaskSpecSemanticRegistry();
const environmentBundleRef = createSecretRef({
projectId: 'default',
name: 'legacy-env-bundle',
version: 3,
});
const normalized = registry.normalize(
context({
spec: {
schema: BUILT_IN_COMMAND_TASK_SPEC_SCHEMA,
config: {
command: { kind: 'argv', file: '/bin/echo', args: [] },
placement: {
required: { runtimes: [{ name: 'node', versionRange: 'not-semver' }] },
},
environmentBundleRef,
},
},
})),
InvalidTaskSpecSemanticError,
}),
);
assert.equal(normalized.config.environmentBundleRef, environmentBundleRef);
for (const invalidRef of [
createSecretRef({ projectId: 'default', name: 'legacy-env-bundle' }),
createSecretRef({
projectId: 'other',
name: 'legacy-env-bundle',
version: 3,
}),
]) {
assert.throws(
() =>
registry.normalize(
context({
spec: {
schema: BUILT_IN_COMMAND_TASK_SPEC_SCHEMA,
config: {
command: { kind: 'argv', file: '/bin/echo', args: [] },
environmentBundleRef: invalidRef,
},
},
}),
),
/environmentBundleRef/,
);
}
});