mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): add opaque cluster environment bundle delivery
This commit is contained in:
@@ -110,6 +110,9 @@
|
||||
"secret-projection": [
|
||||
"dist/secret/secretProjection.d.ts"
|
||||
],
|
||||
"environment-bundle": [
|
||||
"dist/secret/environmentBundle.d.ts"
|
||||
],
|
||||
"plugin-package-task-reconciliation": [
|
||||
"dist/plugin-package/pluginPackageTaskReconciliation.d.ts"
|
||||
],
|
||||
@@ -470,6 +473,11 @@
|
||||
"require": "./dist/secret/secretProjection.js",
|
||||
"default": "./dist/secret/secretProjection.js"
|
||||
},
|
||||
"./environment-bundle": {
|
||||
"types": "./dist/secret/environmentBundle.d.ts",
|
||||
"require": "./dist/secret/environmentBundle.js",
|
||||
"default": "./dist/secret/environmentBundle.js"
|
||||
},
|
||||
"./plugin-package-task-reconciliation": {
|
||||
"types": "./dist/plugin-package/pluginPackageTaskReconciliation.d.ts",
|
||||
"require": "./dist/plugin-package/pluginPackageTaskReconciliation.js",
|
||||
|
||||
@@ -7,6 +7,7 @@ export const CLUSTER_LEGACY_ENV_MIGRATION_PLAN_SCHEMA =
|
||||
export const MAX_CLUSTER_LEGACY_ENV_SOURCE_ROWS = 100_000;
|
||||
export const MAX_CLUSTER_LEGACY_ENV_TASKS = 100_000;
|
||||
export const MAX_CLUSTER_LEGACY_ENV_TRIGGERS = 500_000;
|
||||
export const MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BINDINGS = 256;
|
||||
export const MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BYTES = 64 * 1024;
|
||||
export const MAX_CLUSTER_LEGACY_ENV_MIGRATION_PLAN_JSON_BYTES = 8 * 1024;
|
||||
|
||||
@@ -207,7 +208,7 @@ function sourceEvidence(
|
||||
const effectiveBindingCount = count(
|
||||
value.effectiveBindingCount,
|
||||
'effectiveBindingCount',
|
||||
MAX_CLUSTER_LEGACY_ENV_SOURCE_ROWS,
|
||||
MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BINDINGS,
|
||||
);
|
||||
if (
|
||||
sourceRowCount < 1 ||
|
||||
|
||||
@@ -1,14 +1,19 @@
|
||||
import { digestRunDispatchLeaseToken, assertRunDispatchId } from '../run/runDispatchLease';
|
||||
import {
|
||||
digestRunDispatchLeaseToken,
|
||||
assertRunDispatchId,
|
||||
} from '../run/runDispatchLease';
|
||||
import { parseSecretRef } from '../secret/secretReference';
|
||||
import { assertWorkerId, assertWorkerSessionId } from '../worker/workerSession';
|
||||
|
||||
export const REMOTE_SECRET_DELIVERY_SCHEMA =
|
||||
'qinglong/remote-secret-delivery@v1';
|
||||
'qinglong/remote-secret-delivery@v2';
|
||||
export const MAX_REMOTE_SECRET_DELIVERY_REFS = 64;
|
||||
export const MAX_REMOTE_ENVIRONMENT_BUNDLE_REFS = 1;
|
||||
export const MAX_REMOTE_SECRET_DELIVERY_REQUEST_BYTES = 64 * 1024;
|
||||
export const MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES = 128 * 1024;
|
||||
export const MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES = 256 * 1024;
|
||||
export const MAX_REMOTE_SECRET_VALUE_BYTES = 16 * 1024;
|
||||
export const MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES = 64 * 1024;
|
||||
export const MAX_REMOTE_ENVIRONMENT_BUNDLE_VALUE_BYTES = 96 * 1024;
|
||||
|
||||
export interface RemoteWorkerSecretDeliveryCommand {
|
||||
readonly workerId: string;
|
||||
@@ -25,6 +30,7 @@ export interface RemoteWorkerSecretDeliveryCommand {
|
||||
readonly leaseToken: string;
|
||||
readonly expectedLeaseVersion: number;
|
||||
readonly secretRefs: readonly string[];
|
||||
readonly environmentBundleRefs: readonly string[];
|
||||
}
|
||||
|
||||
export type RemoteWorkerSecretDeliveryRequestBody = Readonly<
|
||||
@@ -47,6 +53,7 @@ export interface RemoteWorkerSecretDeliveryAuthority {
|
||||
readonly leaseGeneration: number;
|
||||
readonly leaseVersion: number;
|
||||
readonly secretRefs: readonly string[];
|
||||
readonly environmentBundleRefs: readonly string[];
|
||||
}
|
||||
|
||||
export interface RemoteWorkerSecretValue {
|
||||
@@ -56,6 +63,7 @@ export interface RemoteWorkerSecretValue {
|
||||
|
||||
export interface RemoteWorkerSecretResolution {
|
||||
readonly values: readonly RemoteWorkerSecretValue[];
|
||||
readonly environmentBundles: readonly RemoteWorkerSecretValue[];
|
||||
readonly dispose?: () => Promise<void> | void;
|
||||
}
|
||||
|
||||
@@ -77,6 +85,7 @@ export interface RemoteWorkerSecretDeliveryResult {
|
||||
readonly offerId: string;
|
||||
readonly executionDigest: string;
|
||||
readonly values: readonly RemoteWorkerSecretValue[];
|
||||
readonly environmentBundles: readonly RemoteWorkerSecretValue[];
|
||||
readonly dispose?: () => Promise<void> | void;
|
||||
}
|
||||
|
||||
@@ -134,7 +143,8 @@ function exactKeys(
|
||||
if (
|
||||
actual.length !== sorted.length ||
|
||||
actual.some((key, index) => key !== sorted[index])
|
||||
) invalid(`${label} shape is invalid`);
|
||||
)
|
||||
invalid(`${label} shape is invalid`);
|
||||
}
|
||||
|
||||
function identifier(label: string, value: unknown, maximum = 128): string {
|
||||
@@ -143,7 +153,8 @@ function identifier(label: string, value: unknown, maximum = 128): string {
|
||||
value.length < 1 ||
|
||||
Buffer.byteLength(value, 'utf8') > maximum ||
|
||||
/[\u0000-\u001f\u007f]/.test(value)
|
||||
) return invalid(`${label} is invalid`);
|
||||
)
|
||||
return invalid(`${label} is invalid`);
|
||||
return value;
|
||||
}
|
||||
|
||||
@@ -152,19 +163,18 @@ function positiveInteger(label: string, value: unknown, minimum = 1): number {
|
||||
!Number.isSafeInteger(value) ||
|
||||
(value as number) < minimum ||
|
||||
(value as number) > 2_147_483_647
|
||||
) return invalid(`${label} is invalid`);
|
||||
)
|
||||
return invalid(`${label} is invalid`);
|
||||
return value as number;
|
||||
}
|
||||
|
||||
function normalizeSecretRefs(
|
||||
value: unknown,
|
||||
projectId: string,
|
||||
maximum: number,
|
||||
): readonly string[] {
|
||||
if (
|
||||
!Array.isArray(value) ||
|
||||
value.length < 1 ||
|
||||
value.length > MAX_REMOTE_SECRET_DELIVERY_REFS
|
||||
) return invalid('secretRefs are invalid');
|
||||
if (!Array.isArray(value) || value.length > maximum)
|
||||
return invalid('secretRefs are invalid');
|
||||
const seen = new Set<string>();
|
||||
const refs = value.map((entry) => {
|
||||
if (typeof entry !== 'string' || seen.has(entry)) {
|
||||
@@ -188,11 +198,27 @@ export function normalizeRemoteWorkerSecretDeliveryCommand(
|
||||
value: RemoteWorkerSecretDeliveryCommand,
|
||||
): Readonly<RemoteWorkerSecretDeliveryCommand> {
|
||||
const command = object(value, 'command');
|
||||
exactKeys(command, [
|
||||
'attemptId', 'executionDigest', 'expectedLeaseVersion', 'leaseGeneration',
|
||||
'leaseToken', 'offerId', 'projectId', 'runId', 'secretRefs', 'taskId',
|
||||
'taskRevision', 'workerGeneration', 'workerId', 'workerSessionId',
|
||||
], 'command');
|
||||
exactKeys(
|
||||
command,
|
||||
[
|
||||
'attemptId',
|
||||
'environmentBundleRefs',
|
||||
'executionDigest',
|
||||
'expectedLeaseVersion',
|
||||
'leaseGeneration',
|
||||
'leaseToken',
|
||||
'offerId',
|
||||
'projectId',
|
||||
'runId',
|
||||
'secretRefs',
|
||||
'taskId',
|
||||
'taskRevision',
|
||||
'workerGeneration',
|
||||
'workerId',
|
||||
'workerSessionId',
|
||||
],
|
||||
'command',
|
||||
);
|
||||
try {
|
||||
assertWorkerId(command.workerId as string);
|
||||
assertWorkerSessionId(command.workerSessionId as string);
|
||||
@@ -206,7 +232,10 @@ export function normalizeRemoteWorkerSecretDeliveryCommand(
|
||||
const normalized = Object.freeze({
|
||||
workerId: command.workerId as string,
|
||||
workerSessionId: command.workerSessionId as string,
|
||||
workerGeneration: positiveInteger('workerGeneration', command.workerGeneration),
|
||||
workerGeneration: positiveInteger(
|
||||
'workerGeneration',
|
||||
command.workerGeneration,
|
||||
),
|
||||
runId: command.runId as string,
|
||||
attemptId: command.attemptId as string,
|
||||
projectId,
|
||||
@@ -214,13 +243,38 @@ export function normalizeRemoteWorkerSecretDeliveryCommand(
|
||||
taskRevision: identifier('taskRevision', command.taskRevision),
|
||||
executionDigest: identifier('executionDigest', command.executionDigest, 64),
|
||||
offerId: command.offerId as string,
|
||||
leaseGeneration: positiveInteger('leaseGeneration', command.leaseGeneration),
|
||||
leaseGeneration: positiveInteger(
|
||||
'leaseGeneration',
|
||||
command.leaseGeneration,
|
||||
),
|
||||
leaseToken: identifier('leaseToken', command.leaseToken, 128),
|
||||
expectedLeaseVersion: positiveInteger(
|
||||
'expectedLeaseVersion', command.expectedLeaseVersion, 0,
|
||||
'expectedLeaseVersion',
|
||||
command.expectedLeaseVersion,
|
||||
0,
|
||||
),
|
||||
secretRefs: normalizeSecretRefs(
|
||||
command.secretRefs,
|
||||
projectId,
|
||||
MAX_REMOTE_SECRET_DELIVERY_REFS,
|
||||
),
|
||||
environmentBundleRefs: normalizeSecretRefs(
|
||||
command.environmentBundleRefs,
|
||||
projectId,
|
||||
MAX_REMOTE_ENVIRONMENT_BUNDLE_REFS,
|
||||
),
|
||||
secretRefs: normalizeSecretRefs(command.secretRefs, projectId),
|
||||
});
|
||||
if (
|
||||
normalized.secretRefs.length + normalized.environmentBundleRefs.length <
|
||||
1
|
||||
)
|
||||
return invalid('Secret reference set is empty');
|
||||
if (
|
||||
normalized.secretRefs.some((reference) =>
|
||||
normalized.environmentBundleRefs.includes(reference),
|
||||
)
|
||||
)
|
||||
return invalid('Secret reference roles overlap');
|
||||
if (!/^[0-9a-f]{64}$/.test(normalized.executionDigest)) {
|
||||
return invalid('executionDigest is invalid');
|
||||
}
|
||||
@@ -236,11 +290,26 @@ export function normalizeRemoteWorkerSecretDeliveryAuthority(
|
||||
value: RemoteWorkerSecretDeliveryAuthority,
|
||||
): Readonly<RemoteWorkerSecretDeliveryAuthority> {
|
||||
const authority = object(value, 'authority');
|
||||
exactKeys(authority, [
|
||||
'attemptId', 'executionDigest', 'leaseGeneration', 'leaseVersion',
|
||||
'offerId', 'projectId', 'runId', 'secretRefs', 'taskId', 'taskRevision',
|
||||
'workerGeneration', 'workerId', 'workerSessionId',
|
||||
], 'authority');
|
||||
exactKeys(
|
||||
authority,
|
||||
[
|
||||
'attemptId',
|
||||
'environmentBundleRefs',
|
||||
'executionDigest',
|
||||
'leaseGeneration',
|
||||
'leaseVersion',
|
||||
'offerId',
|
||||
'projectId',
|
||||
'runId',
|
||||
'secretRefs',
|
||||
'taskId',
|
||||
'taskRevision',
|
||||
'workerGeneration',
|
||||
'workerId',
|
||||
'workerSessionId',
|
||||
],
|
||||
'authority',
|
||||
);
|
||||
try {
|
||||
assertWorkerId(authority.workerId as string);
|
||||
assertWorkerSessionId(authority.workerSessionId as string);
|
||||
@@ -255,7 +324,8 @@ export function normalizeRemoteWorkerSecretDeliveryAuthority(
|
||||
workerId: authority.workerId as string,
|
||||
workerSessionId: authority.workerSessionId as string,
|
||||
workerGeneration: positiveInteger(
|
||||
'workerGeneration', authority.workerGeneration,
|
||||
'workerGeneration',
|
||||
authority.workerGeneration,
|
||||
),
|
||||
runId: authority.runId as string,
|
||||
attemptId: authority.attemptId as string,
|
||||
@@ -263,15 +333,38 @@ export function normalizeRemoteWorkerSecretDeliveryAuthority(
|
||||
taskId: identifier('taskId', authority.taskId),
|
||||
taskRevision: identifier('taskRevision', authority.taskRevision),
|
||||
executionDigest: identifier(
|
||||
'executionDigest', authority.executionDigest, 64,
|
||||
'executionDigest',
|
||||
authority.executionDigest,
|
||||
64,
|
||||
),
|
||||
offerId: authority.offerId as string,
|
||||
leaseGeneration: positiveInteger(
|
||||
'leaseGeneration', authority.leaseGeneration,
|
||||
'leaseGeneration',
|
||||
authority.leaseGeneration,
|
||||
),
|
||||
leaseVersion: positiveInteger('leaseVersion', authority.leaseVersion, 0),
|
||||
secretRefs: normalizeSecretRefs(authority.secretRefs, projectId),
|
||||
secretRefs: normalizeSecretRefs(
|
||||
authority.secretRefs,
|
||||
projectId,
|
||||
MAX_REMOTE_SECRET_DELIVERY_REFS,
|
||||
),
|
||||
environmentBundleRefs: normalizeSecretRefs(
|
||||
authority.environmentBundleRefs,
|
||||
projectId,
|
||||
MAX_REMOTE_ENVIRONMENT_BUNDLE_REFS,
|
||||
),
|
||||
});
|
||||
if (
|
||||
normalized.secretRefs.length + normalized.environmentBundleRefs.length <
|
||||
1
|
||||
)
|
||||
return invalid('Secret reference set is empty');
|
||||
if (
|
||||
normalized.secretRefs.some((reference) =>
|
||||
normalized.environmentBundleRefs.includes(reference),
|
||||
)
|
||||
)
|
||||
return invalid('Secret reference roles overlap');
|
||||
if (!/^[0-9a-f]{64}$/.test(normalized.executionDigest)) {
|
||||
return invalid('executionDigest is invalid');
|
||||
}
|
||||
@@ -282,59 +375,98 @@ export function createRemoteWorkerSecretDeliveryRequestBody(
|
||||
command: RemoteWorkerSecretDeliveryCommand,
|
||||
): RemoteWorkerSecretDeliveryRequestBody {
|
||||
const normalized = normalizeRemoteWorkerSecretDeliveryCommand(command);
|
||||
const { workerId: _workerId, workerSessionId: _sessionId, ...request } = normalized;
|
||||
const {
|
||||
workerId: _workerId,
|
||||
workerSessionId: _sessionId,
|
||||
...request
|
||||
} = normalized;
|
||||
return Object.freeze({ schema: REMOTE_SECRET_DELIVERY_SCHEMA, ...request });
|
||||
}
|
||||
|
||||
function normalizeValues(
|
||||
value: unknown,
|
||||
expectedRefs: readonly string[],
|
||||
maximumValueBytes: number,
|
||||
maximumTotalBytes: number,
|
||||
label: string,
|
||||
): readonly RemoteWorkerSecretValue[] {
|
||||
if (!Array.isArray(value) || value.length !== expectedRefs.length) {
|
||||
return invalid('Secret values are invalid');
|
||||
return invalid(`${label} are invalid`);
|
||||
}
|
||||
let totalValueBytes = 0;
|
||||
return Object.freeze(value.map((entry, index) => {
|
||||
const item = object(entry, `values[${index}]`);
|
||||
exactKeys(item, ['secretRef', 'value'], `values[${index}]`);
|
||||
if (
|
||||
item.secretRef !== expectedRefs[index] ||
|
||||
typeof item.value !== 'string' ||
|
||||
item.value.includes('\0') ||
|
||||
Buffer.byteLength(item.value, 'utf8') > MAX_REMOTE_SECRET_VALUE_BYTES
|
||||
) return invalid(`values[${index}] is invalid`);
|
||||
totalValueBytes += Buffer.byteLength(item.value, 'utf8');
|
||||
if (totalValueBytes > MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES) {
|
||||
return invalid('Secret value byte budget exceeded');
|
||||
}
|
||||
return Object.freeze({
|
||||
secretRef: item.secretRef as string,
|
||||
value: item.value,
|
||||
});
|
||||
}));
|
||||
return Object.freeze(
|
||||
value.map((entry, index) => {
|
||||
const item = object(entry, `values[${index}]`);
|
||||
exactKeys(item, ['secretRef', 'value'], `values[${index}]`);
|
||||
if (
|
||||
item.secretRef !== expectedRefs[index] ||
|
||||
typeof item.value !== 'string' ||
|
||||
item.value.includes('\0') ||
|
||||
Buffer.byteLength(item.value, 'utf8') > maximumValueBytes
|
||||
)
|
||||
return invalid(`values[${index}] is invalid`);
|
||||
totalValueBytes += Buffer.byteLength(item.value, 'utf8');
|
||||
if (totalValueBytes > maximumTotalBytes) {
|
||||
return invalid('Secret value byte budget exceeded');
|
||||
}
|
||||
return Object.freeze({
|
||||
secretRef: item.secretRef as string,
|
||||
value: item.value,
|
||||
});
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export function createRemoteWorkerSecretDeliveryResponseBody(
|
||||
result: Readonly<RemoteWorkerSecretDeliveryResult>,
|
||||
expectedRefs: readonly string[],
|
||||
expected: Readonly<{
|
||||
secretRefs: readonly string[];
|
||||
environmentBundleRefs: readonly string[];
|
||||
}>,
|
||||
): RemoteWorkerSecretDeliveryResponseBody {
|
||||
const value = object(result, 'result');
|
||||
const allowed = ['attemptId', 'dispose', 'executionDigest', 'offerId', 'runId', 'values'];
|
||||
const allowed = [
|
||||
'attemptId',
|
||||
'dispose',
|
||||
'environmentBundles',
|
||||
'executionDigest',
|
||||
'offerId',
|
||||
'runId',
|
||||
'values',
|
||||
];
|
||||
if (Object.keys(value).some((key) => !allowed.includes(key))) {
|
||||
return invalid('result shape is invalid');
|
||||
}
|
||||
const runId = identifier('runId', value.runId, 36);
|
||||
const attemptId = identifier('attemptId', value.attemptId, 36);
|
||||
const offerId = identifier('offerId', value.offerId, 128);
|
||||
const executionDigest = identifier('executionDigest', value.executionDigest, 64);
|
||||
if (!/^[0-9a-f]{64}$/.test(executionDigest)) invalid('executionDigest is invalid');
|
||||
const executionDigest = identifier(
|
||||
'executionDigest',
|
||||
value.executionDigest,
|
||||
64,
|
||||
);
|
||||
if (!/^[0-9a-f]{64}$/.test(executionDigest))
|
||||
invalid('executionDigest is invalid');
|
||||
return Object.freeze({
|
||||
schema: REMOTE_SECRET_DELIVERY_SCHEMA,
|
||||
runId,
|
||||
attemptId,
|
||||
offerId,
|
||||
executionDigest,
|
||||
values: normalizeValues(value.values, expectedRefs),
|
||||
values: normalizeValues(
|
||||
value.values,
|
||||
expected.secretRefs,
|
||||
MAX_REMOTE_SECRET_VALUE_BYTES,
|
||||
MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES,
|
||||
'Secret values',
|
||||
),
|
||||
environmentBundles: normalizeValues(
|
||||
value.environmentBundles,
|
||||
expected.environmentBundleRefs,
|
||||
MAX_REMOTE_ENVIRONMENT_BUNDLE_VALUE_BYTES,
|
||||
MAX_REMOTE_ENVIRONMENT_BUNDLE_VALUE_BYTES,
|
||||
'environment bundles',
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -346,15 +478,18 @@ export function parseRemoteWorkerSecretDeliveryResponse(
|
||||
offerId: string;
|
||||
executionDigest: string;
|
||||
secretRefs: readonly string[];
|
||||
environmentBundleRefs: readonly string[];
|
||||
}>,
|
||||
): Readonly<RemoteWorkerSecretDeliveryResult> {
|
||||
const bytes = typeof serialized === 'string'
|
||||
? Buffer.from(serialized, 'utf8')
|
||||
: Buffer.from(serialized);
|
||||
const bytes =
|
||||
typeof serialized === 'string'
|
||||
? Buffer.from(serialized, 'utf8')
|
||||
: Buffer.from(serialized);
|
||||
if (
|
||||
bytes.byteLength < 2 ||
|
||||
bytes.byteLength > MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES
|
||||
) return invalid('response byte size is outside the allowed range');
|
||||
)
|
||||
return invalid('response byte size is outside the allowed range');
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(bytes.toString('utf8')) as unknown;
|
||||
@@ -364,30 +499,47 @@ export function parseRemoteWorkerSecretDeliveryResponse(
|
||||
bytes.fill(0);
|
||||
}
|
||||
const response = object(parsed, 'response');
|
||||
exactKeys(response, [
|
||||
'attemptId', 'executionDigest', 'offerId', 'runId', 'schema', 'values',
|
||||
], 'response');
|
||||
exactKeys(
|
||||
response,
|
||||
[
|
||||
'attemptId',
|
||||
'environmentBundles',
|
||||
'executionDigest',
|
||||
'offerId',
|
||||
'runId',
|
||||
'schema',
|
||||
'values',
|
||||
],
|
||||
'response',
|
||||
);
|
||||
if (response.schema !== REMOTE_SECRET_DELIVERY_SCHEMA) {
|
||||
return invalid('response schema is invalid');
|
||||
}
|
||||
const result = createRemoteWorkerSecretDeliveryResponseBody({
|
||||
runId: response.runId as string,
|
||||
attemptId: response.attemptId as string,
|
||||
offerId: response.offerId as string,
|
||||
executionDigest: response.executionDigest as string,
|
||||
values: response.values as readonly RemoteWorkerSecretValue[],
|
||||
}, expected.secretRefs);
|
||||
const result = createRemoteWorkerSecretDeliveryResponseBody(
|
||||
{
|
||||
runId: response.runId as string,
|
||||
attemptId: response.attemptId as string,
|
||||
offerId: response.offerId as string,
|
||||
executionDigest: response.executionDigest as string,
|
||||
values: response.values as readonly RemoteWorkerSecretValue[],
|
||||
environmentBundles:
|
||||
response.environmentBundles as readonly RemoteWorkerSecretValue[],
|
||||
},
|
||||
expected,
|
||||
);
|
||||
if (
|
||||
result.runId !== expected.runId ||
|
||||
result.attemptId !== expected.attemptId ||
|
||||
result.offerId !== expected.offerId ||
|
||||
result.executionDigest !== expected.executionDigest
|
||||
) return invalid('response authority does not match request');
|
||||
)
|
||||
return invalid('response authority does not match request');
|
||||
return Object.freeze({
|
||||
runId: result.runId,
|
||||
attemptId: result.attemptId,
|
||||
offerId: result.offerId,
|
||||
executionDigest: result.executionDigest,
|
||||
values: result.values,
|
||||
environmentBundles: result.environmentBundles,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
export const ENVIRONMENT_BUNDLE_SCHEMA =
|
||||
'qinglong/environment-bundle@v1' as const;
|
||||
export const MAX_ENVIRONMENT_BUNDLE_ENTRIES = 256;
|
||||
export const MAX_ENVIRONMENT_BUNDLE_VALUE_BYTES = 16 * 1024;
|
||||
export const MAX_ENVIRONMENT_BUNDLE_TOTAL_BYTES = 64 * 1024;
|
||||
export const MAX_ENVIRONMENT_BUNDLE_ENCODED_BYTES = 96 * 1024;
|
||||
|
||||
const ENVIRONMENT_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/;
|
||||
|
||||
export interface EnvironmentBundleEntry {
|
||||
readonly name: string;
|
||||
readonly value: string;
|
||||
}
|
||||
|
||||
export interface EnvironmentBundle {
|
||||
readonly schema: typeof ENVIRONMENT_BUNDLE_SCHEMA;
|
||||
readonly entries: readonly EnvironmentBundleEntry[];
|
||||
}
|
||||
|
||||
export class InvalidEnvironmentBundleError extends TypeError {
|
||||
readonly code = 'ENVIRONMENT_BUNDLE_INVALID';
|
||||
|
||||
constructor(message: string) {
|
||||
super(`Environment bundle is invalid: ${message}`);
|
||||
this.name = 'InvalidEnvironmentBundleError';
|
||||
}
|
||||
}
|
||||
|
||||
function invalid(message: string): never {
|
||||
throw new InvalidEnvironmentBundleError(message);
|
||||
}
|
||||
|
||||
function dataObject(value: unknown, label: string): Record<string, unknown> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
(Object.getPrototypeOf(value) !== Object.prototype &&
|
||||
Object.getPrototypeOf(value) !== null)
|
||||
) {
|
||||
return invalid(`${label} must be an object`);
|
||||
}
|
||||
const descriptors = Object.getOwnPropertyDescriptors(value);
|
||||
if (
|
||||
Object.values(descriptors).some(
|
||||
(descriptor) =>
|
||||
descriptor.get !== undefined ||
|
||||
descriptor.set !== undefined ||
|
||||
descriptor.enumerable !== true,
|
||||
)
|
||||
) {
|
||||
return invalid(`${label} must contain enumerable data properties`);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function exactKeys(
|
||||
value: object,
|
||||
expected: readonly string[],
|
||||
label: string,
|
||||
): void {
|
||||
const actual = Reflect.ownKeys(value);
|
||||
const canonical = [...expected].sort();
|
||||
if (
|
||||
actual.some((key) => typeof key !== 'string') ||
|
||||
actual.length !== canonical.length ||
|
||||
actual
|
||||
.map(String)
|
||||
.sort()
|
||||
.some((key, index) => key !== canonical[index])
|
||||
) {
|
||||
return invalid(`${label} shape is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeEnvironmentBundle(
|
||||
value: EnvironmentBundle,
|
||||
): Readonly<EnvironmentBundle> {
|
||||
const bundle = dataObject(value, 'bundle');
|
||||
exactKeys(bundle, ['entries', 'schema'], 'bundle');
|
||||
if (bundle.schema !== ENVIRONMENT_BUNDLE_SCHEMA) {
|
||||
return invalid('schema is invalid');
|
||||
}
|
||||
if (
|
||||
!Array.isArray(bundle.entries) ||
|
||||
bundle.entries.length < 1 ||
|
||||
bundle.entries.length > MAX_ENVIRONMENT_BUNDLE_ENTRIES
|
||||
) {
|
||||
return invalid('entry count is invalid');
|
||||
}
|
||||
const names = new Set<string>();
|
||||
let totalBytes = 0;
|
||||
const entries = bundle.entries.map((value, index) => {
|
||||
const entry = dataObject(value, `entries[${index}]`);
|
||||
exactKeys(entry, ['name', 'value'], `entries[${index}]`);
|
||||
if (
|
||||
typeof entry.name !== 'string' ||
|
||||
!ENVIRONMENT_NAME_PATTERN.test(entry.name) ||
|
||||
entry.name.startsWith('QL3_') ||
|
||||
names.has(entry.name)
|
||||
) {
|
||||
return invalid(`entries[${index}].name is invalid or duplicated`);
|
||||
}
|
||||
if (
|
||||
typeof entry.value !== 'string' ||
|
||||
entry.value.includes('\0') ||
|
||||
Buffer.byteLength(entry.value, 'utf8') >
|
||||
MAX_ENVIRONMENT_BUNDLE_VALUE_BYTES
|
||||
) {
|
||||
return invalid(`entries[${index}].value is invalid`);
|
||||
}
|
||||
names.add(entry.name);
|
||||
totalBytes +=
|
||||
Buffer.byteLength(entry.name, 'utf8') +
|
||||
Buffer.byteLength(entry.value, 'utf8');
|
||||
if (totalBytes > MAX_ENVIRONMENT_BUNDLE_TOTAL_BYTES) {
|
||||
return invalid('environment byte budget exceeded');
|
||||
}
|
||||
return Object.freeze({ name: entry.name, value: entry.value });
|
||||
});
|
||||
entries.sort((left, right) =>
|
||||
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
|
||||
);
|
||||
const normalized = Object.freeze({
|
||||
schema: ENVIRONMENT_BUNDLE_SCHEMA,
|
||||
entries: Object.freeze(entries),
|
||||
});
|
||||
if (
|
||||
Buffer.byteLength(JSON.stringify(normalized), 'utf8') >
|
||||
MAX_ENVIRONMENT_BUNDLE_ENCODED_BYTES
|
||||
) {
|
||||
return invalid('encoded byte budget exceeded');
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function serializeEnvironmentBundle(value: EnvironmentBundle): string {
|
||||
return JSON.stringify(normalizeEnvironmentBundle(value));
|
||||
}
|
||||
|
||||
export function parseEnvironmentBundle(
|
||||
serialized: string | Uint8Array,
|
||||
): Readonly<EnvironmentBundle> {
|
||||
const bytes =
|
||||
typeof serialized === 'string'
|
||||
? Buffer.from(serialized, 'utf8')
|
||||
: Buffer.from(serialized);
|
||||
if (
|
||||
bytes.byteLength < 2 ||
|
||||
bytes.byteLength > MAX_ENVIRONMENT_BUNDLE_ENCODED_BYTES
|
||||
) {
|
||||
bytes.fill(0);
|
||||
return invalid('encoded byte size is outside the allowed range');
|
||||
}
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(bytes.toString('utf8')) as unknown;
|
||||
} catch {
|
||||
return invalid('payload is not valid JSON');
|
||||
} finally {
|
||||
bytes.fill(0);
|
||||
}
|
||||
return normalizeEnvironmentBundle(parsed as EnvironmentBundle);
|
||||
}
|
||||
@@ -32,6 +32,7 @@ export interface ClusterTaskExecutionRevisionContent {
|
||||
readonly planSchema: typeof CLUSTER_EXECUTION_PLAN_SCHEMA;
|
||||
readonly command: LocalDispatchCommand;
|
||||
readonly environment: readonly LocalExecutionEnvironmentBinding[];
|
||||
readonly environmentBundleRef?: string;
|
||||
readonly workingDirectory?: string;
|
||||
readonly timeoutMs?: number;
|
||||
readonly placement?: RemoteWorkerPlacementSpec;
|
||||
@@ -79,9 +80,7 @@ function revision(value: unknown): number {
|
||||
(value as number) < 1 ||
|
||||
(value as number) > 2_147_483_647
|
||||
) {
|
||||
throw new InvalidClusterExecutionRevisionError(
|
||||
'sourceRevision is invalid',
|
||||
);
|
||||
throw new InvalidClusterExecutionRevisionError('sourceRevision is invalid');
|
||||
}
|
||||
return value as number;
|
||||
}
|
||||
@@ -103,6 +102,7 @@ function normalizeContent(
|
||||
'command',
|
||||
'createdAtMs',
|
||||
'environment',
|
||||
'environmentBundleRef',
|
||||
'executorType',
|
||||
'planSchema',
|
||||
'placement',
|
||||
@@ -155,6 +155,7 @@ function normalizeContent(
|
||||
}
|
||||
let command: LocalDispatchCommand;
|
||||
let environment: readonly LocalExecutionEnvironmentBinding[];
|
||||
let environmentBundleRef: string | undefined;
|
||||
try {
|
||||
command = normalizeLocalDispatchCommand(value.command);
|
||||
environment = createLocalExecutionContextRecipe({
|
||||
@@ -169,6 +170,16 @@ function normalizeContent(
|
||||
throw new Error('cross-project Secret reference');
|
||||
}
|
||||
}
|
||||
if (value.environmentBundleRef !== undefined) {
|
||||
const reference = parseSecretRef(value.environmentBundleRef);
|
||||
if (
|
||||
reference.projectId !== projectId ||
|
||||
reference.version === undefined
|
||||
) {
|
||||
throw new Error('invalid environment bundle Secret reference');
|
||||
}
|
||||
environmentBundleRef = value.environmentBundleRef;
|
||||
}
|
||||
} catch {
|
||||
throw new InvalidClusterExecutionRevisionError(
|
||||
'command or environment is invalid',
|
||||
@@ -200,9 +211,10 @@ function normalizeContent(
|
||||
timeoutMs = value.timeoutMs;
|
||||
}
|
||||
const createdAtMs = timestamp(value.createdAtMs);
|
||||
const placement = value.placement === undefined
|
||||
? undefined
|
||||
: effectiveRemoteWorkerPlacement(value.placement);
|
||||
const placement =
|
||||
value.placement === undefined
|
||||
? undefined
|
||||
: effectiveRemoteWorkerPlacement(value.placement);
|
||||
const normalized = Object.freeze({
|
||||
projectId,
|
||||
taskId,
|
||||
@@ -213,20 +225,22 @@ function normalizeContent(
|
||||
planSchema: CLUSTER_EXECUTION_PLAN_SCHEMA,
|
||||
command,
|
||||
environment,
|
||||
...(environmentBundleRef === undefined ? {} : { environmentBundleRef }),
|
||||
...(workingDirectory === undefined ? {} : { workingDirectory }),
|
||||
...(timeoutMs === undefined ? {} : { timeoutMs }),
|
||||
...(placement === undefined ? {} : { placement }),
|
||||
createdAtMs,
|
||||
});
|
||||
if (Buffer.byteLength(JSON.stringify(normalized), 'utf8') > MAX_CLUSTER_EXECUTION_PLAN_BYTES) {
|
||||
if (
|
||||
Buffer.byteLength(JSON.stringify(normalized), 'utf8') >
|
||||
MAX_CLUSTER_EXECUTION_PLAN_BYTES
|
||||
) {
|
||||
throw new InvalidClusterExecutionRevisionError('plan byte budget exceeded');
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function digest(
|
||||
content: ClusterTaskExecutionRevisionContent,
|
||||
): string {
|
||||
function digest(content: ClusterTaskExecutionRevisionContent): string {
|
||||
const { createdAtMs: _createdAtMs, ...immutable } = content;
|
||||
return createHash('sha256')
|
||||
.update('qinglong.cluster-task-execution-revision.v1\0', 'utf8')
|
||||
@@ -273,6 +287,9 @@ export function compileClusterCommandTaskDefinition(
|
||||
planSchema: CLUSTER_EXECUTION_PLAN_SCHEMA,
|
||||
command: plan.command,
|
||||
environment: plan.environment,
|
||||
...(plan.environmentBundleRef === undefined
|
||||
? {}
|
||||
: { environmentBundleRef: plan.environmentBundleRef }),
|
||||
...(plan.workingDirectory === undefined
|
||||
? {}
|
||||
: { workingDirectory: plan.workingDirectory }),
|
||||
|
||||
@@ -36,6 +36,7 @@ export interface CommandTaskExecutionPlan {
|
||||
readonly sourceContentDigest: string;
|
||||
readonly command: LocalDispatchCommand;
|
||||
readonly environment: readonly LocalExecutionEnvironmentBinding[];
|
||||
readonly environmentBundleRef?: string;
|
||||
readonly workingDirectory?: string;
|
||||
readonly timeoutMs?: number;
|
||||
readonly placement?: RemoteWorkerPlacementSpec;
|
||||
@@ -112,13 +113,13 @@ export function parseTaskDefinitionRevisionRef(
|
||||
return Object.freeze({ revision, contentDigest });
|
||||
}
|
||||
|
||||
function canonicalRecord(definition: TaskDefinitionRecord): TaskDefinitionRecord {
|
||||
function canonicalRecord(
|
||||
definition: TaskDefinitionRecord,
|
||||
): TaskDefinitionRecord {
|
||||
try {
|
||||
return normalizeTaskDefinitionRecord(definition);
|
||||
} catch {
|
||||
throw new InvalidTaskDefinitionCompilationError(
|
||||
'source record is invalid',
|
||||
);
|
||||
throw new InvalidTaskDefinitionCompilationError('source record is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,6 +173,7 @@ export function compileCommandTaskDefinition(
|
||||
const config = semanticSpec.config as unknown as Readonly<{
|
||||
command: LocalDispatchCommand;
|
||||
environment: readonly LocalExecutionEnvironmentBinding[];
|
||||
environmentBundleRef?: string;
|
||||
workingDirectory?: string;
|
||||
timeoutMs?: number;
|
||||
placement?: RemoteWorkerPlacementSpec;
|
||||
@@ -188,6 +190,9 @@ export function compileCommandTaskDefinition(
|
||||
sourceContentDigest: source.contentDigest,
|
||||
command: config.command,
|
||||
environment: config.environment,
|
||||
...(config.environmentBundleRef === undefined
|
||||
? {}
|
||||
: { environmentBundleRef: config.environmentBundleRef }),
|
||||
...(config.workingDirectory === undefined
|
||||
? {}
|
||||
: { workingDirectory: config.workingDirectory }),
|
||||
@@ -202,6 +207,9 @@ export function compileLocalCommandTaskDefinition(
|
||||
semanticRegistry: TaskSpecSemanticRegistry,
|
||||
): LocalCommandTaskExecutionPlan {
|
||||
const source = compileCommandTaskDefinition(definition, semanticRegistry);
|
||||
if (source.environmentBundleRef !== undefined) {
|
||||
throw new UnsupportedTaskDefinitionCompilationError();
|
||||
}
|
||||
const contextRecipe = createLocalExecutionContextRecipe({
|
||||
environment: source.environment,
|
||||
createdAtMs: source.createdAtMs,
|
||||
|
||||
@@ -196,9 +196,7 @@ function normalizeEnvironment(
|
||||
);
|
||||
});
|
||||
if (bytes > MAX_COMMAND_TASK_ENVIRONMENT_BYTES) {
|
||||
throw new InvalidTaskSpecSemanticError(
|
||||
'environment byte budget exceeded',
|
||||
);
|
||||
throw new InvalidTaskSpecSemanticError('environment byte budget exceeded');
|
||||
}
|
||||
environment.sort((left, right) =>
|
||||
(left as { name: string }).name.localeCompare(
|
||||
@@ -215,14 +213,46 @@ function normalizeCommandConfig(
|
||||
exactKeys(
|
||||
config,
|
||||
['command'],
|
||||
['environment', 'placement', 'timeoutMs', 'workingDirectory'],
|
||||
[
|
||||
'environment',
|
||||
'environmentBundleRef',
|
||||
'placement',
|
||||
'timeoutMs',
|
||||
'workingDirectory',
|
||||
],
|
||||
'command config',
|
||||
);
|
||||
const command = normalizeCommand(config.command);
|
||||
const environment = normalizeEnvironment(config.environment ?? [], context.projectId);
|
||||
const placement = config.placement === undefined
|
||||
? undefined
|
||||
: normalizeRemoteWorkerPlacement(config.placement);
|
||||
const environment = normalizeEnvironment(
|
||||
config.environment ?? [],
|
||||
context.projectId,
|
||||
);
|
||||
let environmentBundleRef: string | undefined;
|
||||
if (config.environmentBundleRef !== undefined) {
|
||||
environmentBundleRef = boundedText(
|
||||
config.environmentBundleRef,
|
||||
'environmentBundleRef',
|
||||
512,
|
||||
);
|
||||
let reference;
|
||||
try {
|
||||
reference = parseSecretRef(environmentBundleRef);
|
||||
} catch {
|
||||
throw new InvalidTaskSpecSemanticError('environmentBundleRef is invalid');
|
||||
}
|
||||
if (
|
||||
reference.projectId !== context.projectId ||
|
||||
reference.version === undefined
|
||||
) {
|
||||
throw new InvalidTaskSpecSemanticError(
|
||||
'environmentBundleRef must pin a version in the same Project',
|
||||
);
|
||||
}
|
||||
}
|
||||
const placement =
|
||||
config.placement === undefined
|
||||
? undefined
|
||||
: normalizeRemoteWorkerPlacement(config.placement);
|
||||
let workingDirectory: string | undefined;
|
||||
if (config.workingDirectory !== undefined) {
|
||||
workingDirectory = boundedText(
|
||||
@@ -250,6 +280,7 @@ function normalizeCommandConfig(
|
||||
return Object.freeze({
|
||||
command,
|
||||
environment,
|
||||
...(environmentBundleRef === undefined ? {} : { environmentBundleRef }),
|
||||
...(placement === undefined
|
||||
? {}
|
||||
: { placement: placement as unknown as TaskDefinitionJson }),
|
||||
@@ -268,10 +299,7 @@ const BUILT_IN_DESCRIPTORS: readonly TaskSpecSemanticDescriptor[] =
|
||||
]);
|
||||
|
||||
export class TaskSpecSemanticRegistry {
|
||||
readonly #descriptors: ReadonlyMap<
|
||||
string,
|
||||
TaskSpecSemanticDescriptor
|
||||
>;
|
||||
readonly #descriptors: ReadonlyMap<string, TaskSpecSemanticDescriptor>;
|
||||
readonly #metadata: readonly TaskSpecSemanticMetadata[];
|
||||
|
||||
constructor(descriptors: readonly TaskSpecSemanticDescriptor[]) {
|
||||
@@ -393,8 +421,5 @@ export function createTaskSpecSemanticRegistry(
|
||||
'extension descriptor uses the reserved qinglong namespace',
|
||||
);
|
||||
}
|
||||
return new TaskSpecSemanticRegistry([
|
||||
...BUILT_IN_DESCRIPTORS,
|
||||
...extensions,
|
||||
]);
|
||||
return new TaskSpecSemanticRegistry([...BUILT_IN_DESCRIPTORS, ...extensions]);
|
||||
}
|
||||
|
||||
@@ -44,15 +44,55 @@ function definition() {
|
||||
});
|
||||
return {
|
||||
registry,
|
||||
record: createTaskDefinitionRecord({
|
||||
...command,
|
||||
spec: registry.normalize({
|
||||
projectId: command.projectId,
|
||||
taskId: command.taskId,
|
||||
kind: command.kind,
|
||||
spec: command.spec,
|
||||
}),
|
||||
}, 90),
|
||||
record: createTaskDefinitionRecord(
|
||||
{
|
||||
...command,
|
||||
spec: registry.normalize({
|
||||
projectId: command.projectId,
|
||||
taskId: command.taskId,
|
||||
kind: command.kind,
|
||||
spec: command.spec,
|
||||
}),
|
||||
},
|
||||
90,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function definitionWithBundle() {
|
||||
const input = definition();
|
||||
const environmentBundleRef = createSecretRef({
|
||||
projectId: 'default',
|
||||
name: 'legacy-env-bundle',
|
||||
version: 4,
|
||||
});
|
||||
const spec = input.registry.normalize({
|
||||
projectId: input.record.projectId,
|
||||
taskId: input.record.taskId,
|
||||
kind: input.record.kind,
|
||||
spec: {
|
||||
...input.record.spec,
|
||||
config: { ...input.record.spec.config, environmentBundleRef },
|
||||
},
|
||||
});
|
||||
return {
|
||||
registry: input.registry,
|
||||
environmentBundleRef,
|
||||
record: createTaskDefinitionRecord(
|
||||
{
|
||||
projectId: input.record.projectId,
|
||||
taskId: input.record.taskId,
|
||||
expectedRevision: null,
|
||||
mutationId: input.record.mutationId,
|
||||
name: input.record.name,
|
||||
kind: input.record.kind,
|
||||
spec,
|
||||
labels: input.record.labels,
|
||||
enabled: input.record.enabled,
|
||||
occurredAtMs: input.record.updatedAtMs,
|
||||
},
|
||||
input.record.createdAtMs,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -70,6 +110,17 @@ test('compiles one digest-bound remote Worker execution revision', () => {
|
||||
assert.deepEqual(normalizeClusterTaskExecutionRevision(revision), revision);
|
||||
});
|
||||
|
||||
test('carries only the pinned environment bundle reference into Cluster plans', () => {
|
||||
const input = definitionWithBundle();
|
||||
const revision = compileClusterCommandTaskDefinition(
|
||||
input.record,
|
||||
input.registry,
|
||||
);
|
||||
assert.equal(revision.environmentBundleRef, input.environmentBundleRef);
|
||||
assert.equal(JSON.stringify(revision).includes('legacy env value'), false);
|
||||
assert.deepEqual(normalizeClusterTaskExecutionRevision(revision), revision);
|
||||
});
|
||||
|
||||
test('rejects digest drift and cross-Project Secret references', () => {
|
||||
const input = definition();
|
||||
const revision = compileClusterCommandTaskDefinition(
|
||||
@@ -77,21 +128,25 @@ test('rejects digest drift and cross-Project Secret references', () => {
|
||||
input.registry,
|
||||
);
|
||||
assert.throws(
|
||||
() => normalizeClusterTaskExecutionRevision({
|
||||
...revision,
|
||||
contentDigest: '0'.repeat(64),
|
||||
}),
|
||||
() =>
|
||||
normalizeClusterTaskExecutionRevision({
|
||||
...revision,
|
||||
contentDigest: '0'.repeat(64),
|
||||
}),
|
||||
InvalidClusterExecutionRevisionError,
|
||||
);
|
||||
assert.throws(
|
||||
() => normalizeClusterTaskExecutionRevision({
|
||||
...revision,
|
||||
environment: [{
|
||||
kind: 'secret',
|
||||
name: 'TOKEN',
|
||||
secretRef: createSecretRef({ projectId: 'another', name: 'TOKEN' }),
|
||||
}],
|
||||
}),
|
||||
() =>
|
||||
normalizeClusterTaskExecutionRevision({
|
||||
...revision,
|
||||
environment: [
|
||||
{
|
||||
kind: 'secret',
|
||||
name: 'TOKEN',
|
||||
secretRef: createSecretRef({ projectId: 'another', name: 'TOKEN' }),
|
||||
},
|
||||
],
|
||||
}),
|
||||
InvalidClusterExecutionRevisionError,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -4,6 +4,7 @@ const { test } = require('node:test');
|
||||
const {
|
||||
CLUSTER_LEGACY_ENV_MIGRATION_PLAN_SCHEMA,
|
||||
MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BYTES,
|
||||
MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BINDINGS,
|
||||
MAX_CLUSTER_LEGACY_ENV_SOURCE_ROWS,
|
||||
MAX_CLUSTER_LEGACY_ENV_TASKS,
|
||||
MAX_CLUSTER_LEGACY_ENV_TRIGGERS,
|
||||
@@ -111,6 +112,15 @@ test('enforces source consistency and router-safe bounded targets', () => {
|
||||
sourceRowCount: MAX_CLUSTER_LEGACY_ENV_SOURCE_ROWS + 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
source: {
|
||||
...intent().source,
|
||||
sourceRowCount: MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BINDINGS + 1,
|
||||
activeRowCount: MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BINDINGS + 1,
|
||||
disabledRowCount: 0,
|
||||
effectiveBindingCount: MAX_CLUSTER_LEGACY_ENV_EFFECTIVE_BINDINGS + 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
target: {
|
||||
...intent().target,
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
ENVIRONMENT_BUNDLE_SCHEMA,
|
||||
InvalidEnvironmentBundleError,
|
||||
parseEnvironmentBundle,
|
||||
serializeEnvironmentBundle,
|
||||
} = require('../dist/secret/environmentBundle');
|
||||
|
||||
test('canonicalizes one opaque environment bundle without external authority', () => {
|
||||
const serialized = serializeEnvironmentBundle({
|
||||
schema: ENVIRONMENT_BUNDLE_SCHEMA,
|
||||
entries: [
|
||||
{ name: 'TOKEN', value: 'secret' },
|
||||
{ name: 'EMPTY', value: '' },
|
||||
],
|
||||
});
|
||||
assert.deepEqual(parseEnvironmentBundle(serialized), {
|
||||
schema: ENVIRONMENT_BUNDLE_SCHEMA,
|
||||
entries: [
|
||||
{ name: 'EMPTY', value: '' },
|
||||
{ name: 'TOKEN', value: 'secret' },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
test('rejects duplicate, reserved, widened and over-budget bundle entries', () => {
|
||||
const values = [
|
||||
{ schema: ENVIRONMENT_BUNDLE_SCHEMA, entries: [] },
|
||||
{
|
||||
schema: ENVIRONMENT_BUNDLE_SCHEMA,
|
||||
entries: [
|
||||
{ name: 'TOKEN', value: 'a' },
|
||||
{ name: 'TOKEN', value: 'b' },
|
||||
],
|
||||
},
|
||||
{
|
||||
schema: ENVIRONMENT_BUNDLE_SCHEMA,
|
||||
entries: [{ name: 'QL3_TOKEN', value: 'a' }],
|
||||
},
|
||||
{
|
||||
schema: ENVIRONMENT_BUNDLE_SCHEMA,
|
||||
entries: [{ name: 'TOKEN', value: 'x'.repeat(16 * 1024 + 1) }],
|
||||
},
|
||||
{
|
||||
schema: ENVIRONMENT_BUNDLE_SCHEMA,
|
||||
entries: [{ name: 'TOKEN', value: 'a', secretRef: 'forbidden' }],
|
||||
},
|
||||
];
|
||||
for (const value of values) {
|
||||
assert.throws(
|
||||
() => serializeEnvironmentBundle(value),
|
||||
InvalidEnvironmentBundleError,
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -14,6 +14,11 @@ const { createSecretRef } = require('../dist/secret/secretReference');
|
||||
const SESSION_ID = '018f0000-0000-7000-8000-000000000001';
|
||||
const DIGEST = 'a'.repeat(64);
|
||||
const SECRET_REF = createSecretRef({ projectId: 'project-1', name: 'token' });
|
||||
const BUNDLE_REF = createSecretRef({
|
||||
projectId: 'project-1',
|
||||
name: 'legacy-env-bundle',
|
||||
version: 7,
|
||||
});
|
||||
|
||||
function command(overrides = {}) {
|
||||
return {
|
||||
@@ -31,13 +36,14 @@ function command(overrides = {}) {
|
||||
leaseToken: 'worker_generated_lease_capability_0000000000000001',
|
||||
expectedLeaseVersion: 4,
|
||||
secretRefs: [SECRET_REF],
|
||||
environmentBundleRefs: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('creates a versioned request without duplicating path-bound identity', () => {
|
||||
const body = createRemoteWorkerSecretDeliveryRequestBody(command());
|
||||
assert.equal(body.schema, 'qinglong/remote-secret-delivery@v1');
|
||||
assert.equal(body.schema, 'qinglong/remote-secret-delivery@v2');
|
||||
assert.equal('workerId' in body, false);
|
||||
assert.equal('workerSessionId' in body, false);
|
||||
assert.deepEqual(body.secretRefs, [SECRET_REF]);
|
||||
@@ -45,67 +51,129 @@ test('creates a versioned request without duplicating path-bound identity', () =
|
||||
});
|
||||
|
||||
test('parses only an exact authority and ordered Secret set', () => {
|
||||
const response = createRemoteWorkerSecretDeliveryResponseBody({
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
offerId: 'offer-1',
|
||||
executionDigest: DIGEST,
|
||||
values: [{ secretRef: SECRET_REF, value: 'private-value' }],
|
||||
}, [SECRET_REF]);
|
||||
const response = createRemoteWorkerSecretDeliveryResponseBody(
|
||||
{
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
offerId: 'offer-1',
|
||||
executionDigest: DIGEST,
|
||||
values: [{ secretRef: SECRET_REF, value: 'private-value' }],
|
||||
environmentBundles: [],
|
||||
},
|
||||
{ secretRefs: [SECRET_REF], environmentBundleRefs: [] },
|
||||
);
|
||||
const parsed = parseRemoteWorkerSecretDeliveryResponse(
|
||||
JSON.stringify(response),
|
||||
{
|
||||
runId: 'run-1', attemptId: 'attempt-1', offerId: 'offer-1',
|
||||
executionDigest: DIGEST, secretRefs: [SECRET_REF],
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
offerId: 'offer-1',
|
||||
executionDigest: DIGEST,
|
||||
secretRefs: [SECRET_REF],
|
||||
environmentBundleRefs: [],
|
||||
},
|
||||
);
|
||||
assert.deepEqual(parsed.values, [
|
||||
{ secretRef: SECRET_REF, value: 'private-value' },
|
||||
]);
|
||||
assert.throws(
|
||||
() => parseRemoteWorkerSecretDeliveryResponse(JSON.stringify(response), {
|
||||
runId: 'run-other', attemptId: 'attempt-1', offerId: 'offer-1',
|
||||
executionDigest: DIGEST, secretRefs: [SECRET_REF],
|
||||
}),
|
||||
() =>
|
||||
parseRemoteWorkerSecretDeliveryResponse(JSON.stringify(response), {
|
||||
runId: 'run-other',
|
||||
attemptId: 'attempt-1',
|
||||
offerId: 'offer-1',
|
||||
executionDigest: DIGEST,
|
||||
secretRefs: [SECRET_REF],
|
||||
environmentBundleRefs: [],
|
||||
}),
|
||||
/authority does not match/,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects duplicate, cross-project and oversized delivery input', () => {
|
||||
assert.throws(
|
||||
() => normalizeRemoteWorkerSecretDeliveryCommand(command({
|
||||
secretRefs: [SECRET_REF, SECRET_REF],
|
||||
})),
|
||||
() =>
|
||||
normalizeRemoteWorkerSecretDeliveryCommand(
|
||||
command({
|
||||
secretRefs: [SECRET_REF, SECRET_REF],
|
||||
}),
|
||||
),
|
||||
/secretRefs are invalid/,
|
||||
);
|
||||
const foreign = createSecretRef({ projectId: 'project-2', name: 'token' });
|
||||
assert.throws(
|
||||
() => normalizeRemoteWorkerSecretDeliveryCommand(command({
|
||||
secretRefs: [foreign],
|
||||
})),
|
||||
() =>
|
||||
normalizeRemoteWorkerSecretDeliveryCommand(
|
||||
command({
|
||||
secretRefs: [foreign],
|
||||
}),
|
||||
),
|
||||
/project is invalid/,
|
||||
);
|
||||
assert.throws(
|
||||
() => parseRemoteWorkerSecretDeliveryResponse(
|
||||
Buffer.alloc(MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES + 1),
|
||||
{
|
||||
runId: 'run-1', attemptId: 'attempt-1', offerId: 'offer-1',
|
||||
executionDigest: DIGEST, secretRefs: [SECRET_REF],
|
||||
},
|
||||
),
|
||||
() =>
|
||||
parseRemoteWorkerSecretDeliveryResponse(
|
||||
Buffer.alloc(MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES + 1),
|
||||
{
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
offerId: 'offer-1',
|
||||
executionDigest: DIGEST,
|
||||
secretRefs: [SECRET_REF],
|
||||
environmentBundleRefs: [],
|
||||
},
|
||||
),
|
||||
/byte size/,
|
||||
);
|
||||
const refs = Array.from({ length: 5 }, (_, index) =>
|
||||
createSecretRef({ projectId: 'project-1', name: `item-${index}` }));
|
||||
createSecretRef({ projectId: 'project-1', name: `item-${index}` }),
|
||||
);
|
||||
assert.throws(
|
||||
() => createRemoteWorkerSecretDeliveryResponseBody({
|
||||
runId: 'run-1', attemptId: 'attempt-1', offerId: 'offer-1',
|
||||
executionDigest: DIGEST,
|
||||
values: refs.map((secretRef) => ({
|
||||
secretRef,
|
||||
value: 'x'.repeat(16 * 1024),
|
||||
})),
|
||||
}, refs),
|
||||
() =>
|
||||
createRemoteWorkerSecretDeliveryResponseBody(
|
||||
{
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
offerId: 'offer-1',
|
||||
executionDigest: DIGEST,
|
||||
values: refs.map((secretRef) => ({
|
||||
secretRef,
|
||||
value: 'x'.repeat(16 * 1024),
|
||||
})),
|
||||
environmentBundles: [],
|
||||
},
|
||||
{ secretRefs: refs, environmentBundleRefs: [] },
|
||||
),
|
||||
/byte budget/,
|
||||
);
|
||||
});
|
||||
|
||||
test('keeps one environment bundle in a distinct bounded authority role', () => {
|
||||
const normalized = normalizeRemoteWorkerSecretDeliveryCommand(
|
||||
command({
|
||||
secretRefs: [],
|
||||
environmentBundleRefs: [BUNDLE_REF],
|
||||
}),
|
||||
);
|
||||
assert.deepEqual(normalized.environmentBundleRefs, [BUNDLE_REF]);
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeRemoteWorkerSecretDeliveryCommand(
|
||||
command({
|
||||
secretRefs: [BUNDLE_REF],
|
||||
environmentBundleRefs: [BUNDLE_REF],
|
||||
}),
|
||||
),
|
||||
/roles overlap/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeRemoteWorkerSecretDeliveryCommand(
|
||||
command({
|
||||
secretRefs: [],
|
||||
environmentBundleRefs: [],
|
||||
}),
|
||||
),
|
||||
/set is empty/,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const { createLocalSecretRef } = require('../dist/secret/localSecret');
|
||||
const { createSecretRef } = require('../dist/secret/secretReference');
|
||||
const {
|
||||
BUILT_IN_COMMAND_TASK_SPEC_SCHEMA,
|
||||
InvalidTaskSpecSemanticError,
|
||||
@@ -231,17 +232,67 @@ test('canonicalizes an optional bounded Remote Worker PlacementSpec in command s
|
||||
preferred: [{ labels: { tier: 'edge' }, weight: 5 }],
|
||||
});
|
||||
assert.throws(
|
||||
() => registry.normalize(context({
|
||||
() =>
|
||||
registry.normalize(
|
||||
context({
|
||||
spec: {
|
||||
schema: BUILT_IN_COMMAND_TASK_SPEC_SCHEMA,
|
||||
config: {
|
||||
command: { kind: 'argv', file: '/bin/echo', args: [] },
|
||||
placement: {
|
||||
required: {
|
||||
runtimes: [{ name: 'node', versionRange: 'not-semver' }],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
),
|
||||
InvalidTaskSpecSemanticError,
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts only a same-Project version-pinned environment bundle reference', () => {
|
||||
const registry = createBuiltInTaskSpecSemanticRegistry();
|
||||
const environmentBundleRef = createSecretRef({
|
||||
projectId: 'default',
|
||||
name: 'legacy-env-bundle',
|
||||
version: 3,
|
||||
});
|
||||
const normalized = registry.normalize(
|
||||
context({
|
||||
spec: {
|
||||
schema: BUILT_IN_COMMAND_TASK_SPEC_SCHEMA,
|
||||
config: {
|
||||
command: { kind: 'argv', file: '/bin/echo', args: [] },
|
||||
placement: {
|
||||
required: { runtimes: [{ name: 'node', versionRange: 'not-semver' }] },
|
||||
},
|
||||
environmentBundleRef,
|
||||
},
|
||||
},
|
||||
})),
|
||||
InvalidTaskSpecSemanticError,
|
||||
}),
|
||||
);
|
||||
assert.equal(normalized.config.environmentBundleRef, environmentBundleRef);
|
||||
for (const invalidRef of [
|
||||
createSecretRef({ projectId: 'default', name: 'legacy-env-bundle' }),
|
||||
createSecretRef({
|
||||
projectId: 'other',
|
||||
name: 'legacy-env-bundle',
|
||||
version: 3,
|
||||
}),
|
||||
]) {
|
||||
assert.throws(
|
||||
() =>
|
||||
registry.normalize(
|
||||
context({
|
||||
spec: {
|
||||
schema: BUILT_IN_COMMAND_TASK_SPEC_SCHEMA,
|
||||
config: {
|
||||
command: { kind: 'argv', file: '/bin/echo', args: [] },
|
||||
environmentBundleRef: invalidRef,
|
||||
},
|
||||
},
|
||||
}),
|
||||
),
|
||||
/environmentBundleRef/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user