open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-21 09:58:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): establish 3.0 incubation baseline

Browse Source
This commit is contained in:
whyour
2026-08-12 00:25:26 +08:00
parent 4bf92dcfeb
commit c699c32461
2817 changed files with 779642 additions and 653 deletions
Download Patch File Download Diff File Expand all files Collapse all files
back/runtime/domain/approvalRequest.ts
+524
View File
@@ -0,0 +1,524 @@
import {
MAX_PROJECT_ROLE_BINDING_VERSION,
assertProjectPolicyProjectId,
normalizePolicySubject,
normalizeProjectPermission,
type PolicySubject,
type ProjectPermission,
type ProjectPolicyFence,
} from './projectPolicy';
export const APPROVAL_RISKS = ['low', 'medium', 'high', 'critical'] as const;
export const APPROVAL_REQUEST_STATES = [
'pending',
'approved',
'rejected',
'consumed',
] as const;
export const APPROVAL_DECISIONS = ['approved', 'rejected'] as const;
export const APPROVED_ACTION_DISPATCH_STATES = ['pending'] as const;
export type ApprovalRisk = (typeof APPROVAL_RISKS)[number];
export type ApprovalRequestState = (typeof APPROVAL_REQUEST_STATES)[number];
export type ApprovalDecision = (typeof APPROVAL_DECISIONS)[number];
export type ApprovalRequestEffectiveStatus = ApprovalRequestState | 'expired';
export type ApprovedActionDispatchState =
(typeof APPROVED_ACTION_DISPATCH_STATES)[number];
export interface ApprovalActionBinding {
permission: ProjectPermission;
actionType: string;
actionRef: string;
actionDigest: string;
previewDigest: string;
}
export interface ApprovalRequestRecord {
id: string;
projectId: string;
version: number;
state: ApprovalRequestState;
action: ApprovalActionBinding;
risk: ApprovalRisk;
requestedBy: PolicySubject;
requestedAtMs: number;
expiresAtMs: number;
decisionId: string | null;
decision: ApprovalDecision | null;
decisionReasonCode: string | null;
decidedBy: PolicySubject | null;
decidedAtMs: number | null;
consumptionId: string | null;
dispatchId: string | null;
consumedBy: PolicySubject | null;
consumedAtMs: number | null;
}
export interface ApprovedActionDispatchRecord {
id: string;
approvalRequestId: string;
approvalRequestVersion: number;
projectId: string;
state: ApprovedActionDispatchState;
action: ApprovalActionBinding;
requestedBy: PolicySubject;
consumedBy: PolicySubject;
createdAtMs: number;
}
export const MAX_APPROVAL_REQUEST_ID_LENGTH = 64;
export const MAX_APPROVAL_MUTATION_ID_LENGTH = 64;
export const MAX_APPROVAL_ACTION_TYPE_LENGTH = 64;
export const MAX_APPROVAL_ACTION_REF_LENGTH = 255;
export const MAX_APPROVAL_REASON_CODE_LENGTH = 64;
export const MAX_APPROVAL_LIFETIME_MS = 24 * 60 * 60 * 1000;
export const MAX_APPROVAL_REQUEST_VERSION = 2_147_483_647;
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/;
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
export class InvalidApprovalValueError extends TypeError {
constructor(message: string) {
super(`Approval value is invalid: ${message}`);
this.name = 'InvalidApprovalValueError';
}
}
export class ApprovalRequestNotFoundError extends Error {
readonly code = 'APPROVAL_REQUEST_NOT_FOUND';
constructor() {
super('Approval request does not exist');
this.name = 'ApprovalRequestNotFoundError';
}
}
export class ApprovalPolicyDeniedError extends Error {
readonly code = 'APPROVAL_POLICY_DENIED';
constructor() {
super('Approval operation is denied by project policy');
this.name = 'ApprovalPolicyDeniedError';
}
}
export class ApprovalPolicyFenceConflictError extends Error {
readonly code = 'APPROVAL_POLICY_FENCE_CONFLICT';
constructor() {
super('Approval policy snapshot changed before the mutation committed');
this.name = 'ApprovalPolicyFenceConflictError';
}
}
export class ApprovalRequestVersionConflictError extends Error {
readonly code = 'APPROVAL_REQUEST_VERSION_CONFLICT';
constructor() {
super('Approval request version changed');
this.name = 'ApprovalRequestVersionConflictError';
}
}
export class ApprovalMutationConflictError extends Error {
readonly code = 'APPROVAL_MUTATION_CONFLICT';
constructor() {
super('Approval mutation does not match its previous request');
this.name = 'ApprovalMutationConflictError';
}
}
export class ApprovalRequestStateConflictError extends Error {
readonly code = 'APPROVAL_REQUEST_STATE_CONFLICT';
constructor() {
super('Approval request is not in the required state');
this.name = 'ApprovalRequestStateConflictError';
}
}
export class ApprovalRequestExpiredError extends Error {
readonly code = 'APPROVAL_REQUEST_EXPIRED';
constructor() {
super('Approval request expired');
this.name = 'ApprovalRequestExpiredError';
}
}
export class ApprovalHumanDecisionRequiredError extends Error {
readonly code = 'APPROVAL_HUMAN_DECISION_REQUIRED';
constructor() {
super('Approval decisions require an authenticated user subject');
this.name = 'ApprovalHumanDecisionRequiredError';
}
}
export class ApprovalSelfDecisionError extends Error {
readonly code = 'APPROVAL_SELF_DECISION_REJECTED';
constructor() {
super('An approval requester cannot decide its own request');
this.name = 'ApprovalSelfDecisionError';
}
}
export class ApprovalUnavailableError extends Error {
readonly code = 'APPROVAL_UNAVAILABLE';
constructor() {
super('Approval storage is unavailable');
this.name = 'ApprovalUnavailableError';
}
}
function assertExactKeys(
name: string,
value: object,
expected: readonly string[],
): void {
const keys = Object.keys(value).sort();
const canonical = [...expected].sort();
if (
keys.length !== canonical.length ||
keys.some((key, index) => key !== canonical[index])
) {
throw new InvalidApprovalValueError(`${name} shape is invalid`);
}
}
function assertIdentifier(name: string, value: string, maximum: number): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > maximum ||
!IDENTIFIER_PATTERN.test(value)
) {
throw new InvalidApprovalValueError(`${name} is invalid`);
}
}
function assertTimestamp(name: string, value: number): void {
if (!Number.isSafeInteger(value) || value < 0) {
throw new InvalidApprovalValueError(`${name} is invalid`);
}
}
function sameSubject(
left: Readonly<PolicySubject>,
right: Readonly<PolicySubject>,
): boolean {
return left.type === right.type && left.id === right.id;
}
export function assertApprovalRequestId(value: string): void {
assertIdentifier('request id', value, MAX_APPROVAL_REQUEST_ID_LENGTH);
}
export function assertApprovalMutationId(value: string): void {
assertIdentifier('mutation id', value, MAX_APPROVAL_MUTATION_ID_LENGTH);
}
export function assertApprovalReasonCode(value: string): void {
assertIdentifier(
'decision reason code',
value,
MAX_APPROVAL_REASON_CODE_LENGTH,
);
}
export function assertApprovalTimestamp(name: string, value: number): void {
assertTimestamp(name, value);
}
export function assertApprovalRequestVersion(value: number): void {
if (
!Number.isSafeInteger(value) ||
value < 1 ||
value > MAX_APPROVAL_REQUEST_VERSION
) {
throw new InvalidApprovalValueError('request version is invalid');
}
}
export function normalizeApprovalActionBinding(
value: ApprovalActionBinding,
): Readonly<ApprovalActionBinding> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidApprovalValueError('action must be an object');
}
assertExactKeys('action', value, [
'permission',
'actionType',
'actionRef',
'actionDigest',
'previewDigest',
]);
const permission = normalizeProjectPermission(value.permission);
assertIdentifier(
'action type',
value.actionType,
MAX_APPROVAL_ACTION_TYPE_LENGTH,
);
assertIdentifier(
'action ref',
value.actionRef,
MAX_APPROVAL_ACTION_REF_LENGTH,
);
if (!DIGEST_PATTERN.test(value.actionDigest)) {
throw new InvalidApprovalValueError('action digest is invalid');
}
if (!DIGEST_PATTERN.test(value.previewDigest)) {
throw new InvalidApprovalValueError('preview digest is invalid');
}
return Object.freeze({
permission,
actionType: value.actionType,
actionRef: value.actionRef,
actionDigest: value.actionDigest,
previewDigest: value.previewDigest,
});
}
export function normalizeApprovalPolicyFence(
value: ProjectPolicyFence,
): Readonly<ProjectPolicyFence> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidApprovalValueError('policy fence must be an object');
}
assertExactKeys('policy fence', value, ['projectVersion', 'bindingVersion']);
if (
!Number.isSafeInteger(value.projectVersion) ||
value.projectVersion < 1 ||
value.projectVersion > MAX_PROJECT_ROLE_BINDING_VERSION
) {
throw new InvalidApprovalValueError('project policy version is invalid');
}
if (
value.bindingVersion !== null &&
(!Number.isSafeInteger(value.bindingVersion) ||
value.bindingVersion < 1 ||
value.bindingVersion > MAX_PROJECT_ROLE_BINDING_VERSION)
) {
throw new InvalidApprovalValueError('binding policy version is invalid');
}
return Object.freeze({ ...value });
}
export function normalizeApprovalRequestRecord(
value: ApprovalRequestRecord,
): Readonly<ApprovalRequestRecord> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidApprovalValueError('request must be an object');
}
assertExactKeys('request', value, [
'id',
'projectId',
'version',
'state',
'action',
'risk',
'requestedBy',
'requestedAtMs',
'expiresAtMs',
'decisionId',
'decision',
'decisionReasonCode',
'decidedBy',
'decidedAtMs',
'consumptionId',
'dispatchId',
'consumedBy',
'consumedAtMs',
]);
assertApprovalRequestId(value.id);
assertProjectPolicyProjectId(value.projectId);
assertApprovalRequestVersion(value.version);
if (!APPROVAL_REQUEST_STATES.includes(value.state)) {
throw new InvalidApprovalValueError('request state is invalid');
}
const action = normalizeApprovalActionBinding(value.action);
if (!APPROVAL_RISKS.includes(value.risk)) {
throw new InvalidApprovalValueError('risk is invalid');
}
const requestedBy = normalizePolicySubject(value.requestedBy);
assertTimestamp('requestedAtMs', value.requestedAtMs);
assertTimestamp('expiresAtMs', value.expiresAtMs);
if (
value.expiresAtMs <= value.requestedAtMs ||
value.expiresAtMs - value.requestedAtMs > MAX_APPROVAL_LIFETIME_MS
) {
throw new InvalidApprovalValueError('request lifetime is invalid');
}
const decisionValues = [
value.decisionId,
value.decision,
value.decisionReasonCode,
value.decidedBy,
value.decidedAtMs,
];
const hasDecision = decisionValues.every((candidate) => candidate !== null);
if (!hasDecision && decisionValues.some((candidate) => candidate !== null)) {
throw new InvalidApprovalValueError('decision tuple is incomplete');
}
let decidedBy: Readonly<PolicySubject> | null = null;
if (hasDecision) {
assertApprovalMutationId(value.decisionId!);
if (!APPROVAL_DECISIONS.includes(value.decision!)) {
throw new InvalidApprovalValueError('decision is invalid');
}
assertApprovalReasonCode(value.decisionReasonCode!);
decidedBy = normalizePolicySubject(value.decidedBy!);
assertTimestamp('decidedAtMs', value.decidedAtMs!);
if (
value.decidedAtMs! < value.requestedAtMs ||
value.decidedAtMs! >= value.expiresAtMs
) {
throw new InvalidApprovalValueError('decision timestamp is invalid');
}
}
const consumptionValues = [
value.consumptionId,
value.dispatchId,
value.consumedBy,
value.consumedAtMs,
];
const hasConsumption = consumptionValues.every(
(candidate) => candidate !== null,
);
if (
!hasConsumption &&
consumptionValues.some((candidate) => candidate !== null)
) {
throw new InvalidApprovalValueError('consumption tuple is incomplete');
}
let consumedBy: Readonly<PolicySubject> | null = null;
if (hasConsumption) {
assertApprovalMutationId(value.consumptionId!);
assertApprovalMutationId(value.dispatchId!);
consumedBy = normalizePolicySubject(value.consumedBy!);
assertTimestamp('consumedAtMs', value.consumedAtMs!);
if (
!hasDecision ||
value.decision !== 'approved' ||
value.consumedAtMs! < value.decidedAtMs! ||
value.consumedAtMs! >= value.expiresAtMs
) {
throw new InvalidApprovalValueError('consumption tuple is invalid');
}
}
if (
(value.state === 'pending' && (hasDecision || hasConsumption)) ||
(value.state === 'approved' &&
(!hasDecision || value.decision !== 'approved' || hasConsumption)) ||
(value.state === 'rejected' &&
(!hasDecision || value.decision !== 'rejected' || hasConsumption)) ||
(value.state === 'consumed' && !hasConsumption)
) {
throw new InvalidApprovalValueError('state tuple is inconsistent');
}
if (value.state === 'pending' && value.version !== 1) {
throw new InvalidApprovalValueError('pending request version is invalid');
}
if (
(value.state === 'approved' || value.state === 'rejected') &&
value.version !== 2
) {
throw new InvalidApprovalValueError('decided request version is invalid');
}
if (value.state === 'consumed' && value.version !== 3) {
throw new InvalidApprovalValueError('consumed request version is invalid');
}
return Object.freeze({
...value,
action,
requestedBy,
decidedBy,
consumedBy,
});
}
export function normalizeApprovedActionDispatchRecord(
value: ApprovedActionDispatchRecord,
): Readonly<ApprovedActionDispatchRecord> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidApprovalValueError('dispatch must be an object');
}
assertExactKeys('dispatch', value, [
'id',
'approvalRequestId',
'approvalRequestVersion',
'projectId',
'state',
'action',
'requestedBy',
'consumedBy',
'createdAtMs',
]);
assertApprovalMutationId(value.id);
assertApprovalRequestId(value.approvalRequestId);
assertApprovalRequestVersion(value.approvalRequestVersion);
if (value.approvalRequestVersion !== 3) {
throw new InvalidApprovalValueError('dispatch approval version is invalid');
}
assertProjectPolicyProjectId(value.projectId);
if (!APPROVED_ACTION_DISPATCH_STATES.includes(value.state)) {
throw new InvalidApprovalValueError('dispatch state is invalid');
}
const action = normalizeApprovalActionBinding(value.action);
const requestedBy = normalizePolicySubject(value.requestedBy);
const consumedBy = normalizePolicySubject(value.consumedBy);
assertTimestamp('dispatch createdAtMs', value.createdAtMs);
return Object.freeze({
...value,
action,
requestedBy,
consumedBy,
});
}
export function approvalRequestEffectiveStatus(
request: Readonly<ApprovalRequestRecord>,
nowMs: number,
): ApprovalRequestEffectiveStatus {
const normalized = normalizeApprovalRequestRecord(request);
assertTimestamp('nowMs', nowMs);
if (
nowMs >= normalized.expiresAtMs &&
(normalized.state === 'pending' || normalized.state === 'approved')
) {
return 'expired';
}
return normalized.state;
}
export function sameApprovalAction(
left: Readonly<ApprovalActionBinding>,
right: Readonly<ApprovalActionBinding>,
): boolean {
const normalizedLeft = normalizeApprovalActionBinding(left);
const normalizedRight = normalizeApprovalActionBinding(right);
return (
normalizedLeft.permission === normalizedRight.permission &&
normalizedLeft.actionType === normalizedRight.actionType &&
normalizedLeft.actionRef === normalizedRight.actionRef &&
normalizedLeft.actionDigest === normalizedRight.actionDigest &&
normalizedLeft.previewDigest === normalizedRight.previewDigest
);
}
export function sameApprovalSubject(
left: Readonly<PolicySubject>,
right: Readonly<PolicySubject>,
): boolean {
return sameSubject(
normalizePolicySubject(left),
normalizePolicySubject(right),
);
}
back/runtime/domain/approvedActionDispatchExecution.ts
+430
View File
@@ -0,0 +1,430 @@
import {
assertApprovalMutationId,
assertApprovalTimestamp,
type ApprovedActionDispatchRecord,
} from './approvalRequest';
import { assertProjectPolicyProjectId } from './projectPolicy';
export const APPROVED_ACTION_EXECUTION_STATUSES = [
'pending',
'leased',
'executing',
'retry_wait',
'succeeded',
'failed',
'blocked',
] as const;
export type ApprovedActionExecutionStatus =
(typeof APPROVED_ACTION_EXECUTION_STATUSES)[number];
export type ApprovedActionExecutionEffectiveStatus =
| ApprovedActionExecutionStatus
| 'recovery_required';
export interface ApprovedActionDispatchExecutionRecord {
dispatchId: string;
projectId: string;
status: ApprovedActionExecutionStatus;
version: number;
attemptCount: number;
maxAttempts: number;
eligibleAtMs: number | null;
nextAttemptAtMs: number | null;
leaseOwner: string | null;
leaseToken: string | null;
leaseExpiresAtMs: number | null;
startedAtMs: number | null;
resultMutationId: string | null;
lastResultCode: string | null;
completedAtMs: number | null;
createdAtMs: number;
updatedAtMs: number;
}
export interface ApprovedActionDispatchExecutionSnapshot {
dispatch: Readonly<ApprovedActionDispatchRecord>;
execution: Readonly<ApprovedActionDispatchExecutionRecord>;
}
export interface ApprovedActionDispatchCursor {
eligibleAtMs: number;
dispatchId: string;
}
export const DEFAULT_APPROVED_ACTION_MAX_ATTEMPTS = 5;
export const MAX_APPROVED_ACTION_ATTEMPTS = 16;
export const MAX_APPROVED_ACTION_EXECUTION_VERSION = 2_147_483_647;
export const MAX_APPROVED_ACTION_LEASE_ID_LENGTH = 128;
export const MAX_APPROVED_ACTION_RESULT_CODE_LENGTH = 64;
export const MAX_APPROVED_ACTION_DISPATCH_PAGE_SIZE = 64;
export const MAX_APPROVED_ACTION_LEASE_DURATION_MS = 10 * 60 * 1000;
const EXECUTION_IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/;
export class InvalidApprovedActionExecutionError extends TypeError {
constructor(message: string) {
super(`Approved action execution is invalid: ${message}`);
this.name = 'InvalidApprovedActionExecutionError';
}
}
export class ApprovedActionDispatchNotFoundError extends Error {
readonly code = 'APPROVED_ACTION_DISPATCH_NOT_FOUND';
constructor() {
super('Approved action dispatch does not exist');
this.name = 'ApprovedActionDispatchNotFoundError';
}
}
export class ApprovedActionDispatchFenceRejectedError extends Error {
readonly code = 'APPROVED_ACTION_DISPATCH_FENCE_REJECTED';
constructor() {
super('Approved action dispatch execution fence was rejected');
this.name = 'ApprovedActionDispatchFenceRejectedError';
}
}
export class ApprovedActionDispatchStateConflictError extends Error {
readonly code = 'APPROVED_ACTION_DISPATCH_STATE_CONFLICT';
constructor() {
super('Approved action dispatch is not in the required execution state');
this.name = 'ApprovedActionDispatchStateConflictError';
}
}
export class ApprovedActionDispatchBindingConflictError extends Error {
readonly code = 'APPROVED_ACTION_DISPATCH_BINDING_CONFLICT';
constructor() {
super('Approved action dispatch identity does not match its execution');
this.name = 'ApprovedActionDispatchBindingConflictError';
}
}
export class ApprovedActionDispatchRepositoryError extends Error {
readonly code = 'APPROVED_ACTION_DISPATCH_REPOSITORY_ERROR';
constructor() {
super('Approved action dispatch repository is unavailable');
this.name = 'ApprovedActionDispatchRepositoryError';
}
}
function assertExactKeys(
name: string,
value: object,
expected: readonly string[],
): void {
const keys = Object.keys(value).sort();
const canonical = [...expected].sort();
if (
keys.length !== canonical.length ||
keys.some((key, index) => key !== canonical[index])
) {
throw new InvalidApprovedActionExecutionError(`${name} shape is invalid`);
}
}
function assertInteger(
name: string,
value: number,
minimum: number,
maximum: number,
): void {
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
throw new InvalidApprovedActionExecutionError(`${name} is invalid`);
}
}
function assertNullableTimestamp(name: string, value: number | null): void {
if (value !== null) assertApprovalTimestamp(name, value);
}
function assertNullableIdentifier(
name: string,
value: string | null,
maximum: number,
): void {
if (
value !== null &&
(typeof value !== 'string' ||
value.length < 1 ||
value.length > maximum ||
!EXECUTION_IDENTIFIER_PATTERN.test(value))
) {
throw new InvalidApprovedActionExecutionError(`${name} is invalid`);
}
}
function allNull(values: readonly unknown[]): boolean {
return values.every((value) => value === null);
}
function allPresent(values: readonly unknown[]): boolean {
return values.every((value) => value !== null);
}
export function assertApprovedActionLeaseIdentity(value: string): void {
if (typeof value !== 'string') {
throw new InvalidApprovedActionExecutionError('lease identity is invalid');
}
assertNullableIdentifier(
'lease identity',
value,
MAX_APPROVED_ACTION_LEASE_ID_LENGTH,
);
}
export function assertApprovedActionResultCode(value: string): void {
if (typeof value !== 'string') {
throw new InvalidApprovedActionExecutionError('result code is invalid');
}
assertNullableIdentifier(
'result code',
value,
MAX_APPROVED_ACTION_RESULT_CODE_LENGTH,
);
}
export function assertApprovedActionLeaseDuration(value: number): void {
assertInteger(
'lease duration',
value,
1,
MAX_APPROVED_ACTION_LEASE_DURATION_MS,
);
}
export function assertApprovedActionPageSize(value: number): void {
assertInteger('page size', value, 1, MAX_APPROVED_ACTION_DISPATCH_PAGE_SIZE);
}
export function normalizeApprovedActionDispatchCursor(
value: ApprovedActionDispatchCursor,
): Readonly<ApprovedActionDispatchCursor> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidApprovedActionExecutionError('cursor must be an object');
}
assertExactKeys('cursor', value, ['eligibleAtMs', 'dispatchId']);
assertApprovalTimestamp('cursor eligibleAtMs', value.eligibleAtMs);
assertApprovalMutationId(value.dispatchId);
return Object.freeze({ ...value });
}
export function normalizeApprovedActionDispatchExecutionRecord(
value: ApprovedActionDispatchExecutionRecord,
): Readonly<ApprovedActionDispatchExecutionRecord> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidApprovedActionExecutionError('record must be an object');
}
assertExactKeys('record', value, [
'dispatchId',
'projectId',
'status',
'version',
'attemptCount',
'maxAttempts',
'eligibleAtMs',
'nextAttemptAtMs',
'leaseOwner',
'leaseToken',
'leaseExpiresAtMs',
'startedAtMs',
'resultMutationId',
'lastResultCode',
'completedAtMs',
'createdAtMs',
'updatedAtMs',
]);
assertApprovalMutationId(value.dispatchId);
assertProjectPolicyProjectId(value.projectId);
if (!APPROVED_ACTION_EXECUTION_STATUSES.includes(value.status)) {
throw new InvalidApprovedActionExecutionError('status is invalid');
}
assertInteger(
'version',
value.version,
0,
MAX_APPROVED_ACTION_EXECUTION_VERSION,
);
assertInteger(
'attempt count',
value.attemptCount,
0,
MAX_APPROVED_ACTION_ATTEMPTS,
);
assertInteger(
'max attempts',
value.maxAttempts,
1,
MAX_APPROVED_ACTION_ATTEMPTS,
);
if (value.attemptCount > value.maxAttempts) {
throw new InvalidApprovedActionExecutionError(
'attempt count exceeds its maximum',
);
}
for (const [name, timestamp] of [
['eligibleAtMs', value.eligibleAtMs],
['nextAttemptAtMs', value.nextAttemptAtMs],
['leaseExpiresAtMs', value.leaseExpiresAtMs],
['startedAtMs', value.startedAtMs],
['completedAtMs', value.completedAtMs],
] as const) {
assertNullableTimestamp(name, timestamp);
}
assertApprovalTimestamp('createdAtMs', value.createdAtMs);
assertApprovalTimestamp('updatedAtMs', value.updatedAtMs);
if (value.updatedAtMs < value.createdAtMs) {
throw new InvalidApprovedActionExecutionError('timestamps are invalid');
}
assertNullableIdentifier(
'lease owner',
value.leaseOwner,
MAX_APPROVED_ACTION_LEASE_ID_LENGTH,
);
assertNullableIdentifier(
'lease token',
value.leaseToken,
MAX_APPROVED_ACTION_LEASE_ID_LENGTH,
);
assertNullableIdentifier('result mutation id', value.resultMutationId, 64);
assertNullableIdentifier(
'last result code',
value.lastResultCode,
MAX_APPROVED_ACTION_RESULT_CODE_LENGTH,
);
const leaseTuple = [
value.leaseOwner,
value.leaseToken,
value.leaseExpiresAtMs,
];
if (!allNull(leaseTuple) && !allPresent(leaseTuple)) {
throw new InvalidApprovedActionExecutionError('lease tuple is incomplete');
}
const hasLease = allPresent(leaseTuple);
if (
hasLease &&
(value.leaseExpiresAtMs! <= value.updatedAtMs ||
value.leaseExpiresAtMs! - value.updatedAtMs >
MAX_APPROVED_ACTION_LEASE_DURATION_MS)
) {
throw new InvalidApprovedActionExecutionError('lease lifetime is invalid');
}
const terminal = ['succeeded', 'failed', 'blocked'].includes(value.status);
if (
value.status === 'pending' &&
(value.version !== 0 ||
value.attemptCount !== 0 ||
value.eligibleAtMs === null ||
value.nextAttemptAtMs !== null ||
hasLease ||
value.startedAtMs !== null ||
value.resultMutationId !== null ||
value.lastResultCode !== null ||
value.completedAtMs !== null)
) {
throw new InvalidApprovedActionExecutionError('pending tuple is invalid');
}
if (
value.status === 'leased' &&
(!hasLease ||
value.attemptCount < 1 ||
value.eligibleAtMs !== value.leaseExpiresAtMs ||
value.nextAttemptAtMs !== null ||
value.startedAtMs !== null ||
value.resultMutationId !== null ||
value.lastResultCode !== null ||
value.completedAtMs !== null)
) {
throw new InvalidApprovedActionExecutionError('leased tuple is invalid');
}
if (
value.status === 'executing' &&
(!hasLease ||
value.attemptCount < 1 ||
value.eligibleAtMs !== null ||
value.nextAttemptAtMs !== null ||
value.startedAtMs === null ||
value.resultMutationId !== null ||
value.lastResultCode !== null ||
value.completedAtMs !== null)
) {
throw new InvalidApprovedActionExecutionError('executing tuple is invalid');
}
if (
value.status === 'retry_wait' &&
(hasLease ||
value.attemptCount < 1 ||
value.attemptCount >= value.maxAttempts ||
value.eligibleAtMs === null ||
value.eligibleAtMs !== value.nextAttemptAtMs ||
value.startedAtMs !== null ||
value.resultMutationId === null ||
value.lastResultCode === null ||
value.completedAtMs !== null)
) {
throw new InvalidApprovedActionExecutionError(
'retry wait tuple is invalid',
);
}
if (
terminal &&
(hasLease ||
value.eligibleAtMs !== null ||
value.nextAttemptAtMs !== null ||
value.resultMutationId === null ||
value.lastResultCode === null ||
value.completedAtMs === null)
) {
throw new InvalidApprovedActionExecutionError('terminal tuple is invalid');
}
if (
(value.status === 'succeeded' || value.status === 'failed') &&
value.startedAtMs === null
) {
throw new InvalidApprovedActionExecutionError(
'completed execution has no start barrier',
);
}
if (
value.startedAtMs !== null &&
(value.startedAtMs < value.createdAtMs ||
value.startedAtMs > value.updatedAtMs)
) {
throw new InvalidApprovedActionExecutionError(
'execution start timestamp is invalid',
);
}
if (
value.completedAtMs !== null &&
(value.completedAtMs < (value.startedAtMs ?? value.createdAtMs) ||
value.completedAtMs !== value.updatedAtMs)
) {
throw new InvalidApprovedActionExecutionError(
'completion timestamp is invalid',
);
}
return Object.freeze({ ...value });
}
export function approvedActionExecutionEffectiveStatus(
record: Readonly<ApprovedActionDispatchExecutionRecord>,
nowMs: number,
): ApprovedActionExecutionEffectiveStatus {
const normalized = normalizeApprovedActionDispatchExecutionRecord(record);
assertApprovalTimestamp('nowMs', nowMs);
if (
normalized.status === 'executing' &&
normalized.leaseExpiresAtMs !== null &&
nowMs >= normalized.leaseExpiresAtMs
) {
return 'recovery_required';
}
return normalized.status;
}
back/runtime/domain/approvedActionRecovery.ts
+402
View File
@@ -0,0 +1,402 @@
import {
assertApprovedActionLeaseDuration,
assertApprovedActionLeaseIdentity,
assertApprovedActionPageSize,
assertApprovedActionResultCode,
type ApprovedActionDispatchExecutionSnapshot,
} from './approvedActionDispatchExecution';
import {
assertApprovalMutationId,
assertApprovalTimestamp,
} from './approvalRequest';
import {
assertProjectPolicyProjectId,
normalizePolicySubject,
type PolicySubject,
} from './projectPolicy';
export const APPROVED_ACTION_RECOVERY_CONTROL_STATUSES = [
'armed',
'leased',
'manual_required',
'resolved',
] as const;
export const APPROVED_ACTION_RECOVERY_FINDINGS = [
'verified_succeeded',
'verified_failed',
'still_running',
'missing',
'conflict',
'unsupported',
'unavailable',
] as const;
export const APPROVED_ACTION_RECOVERY_SOURCES = [
'automatic_evidence',
'human',
] as const;
export const APPROVED_ACTION_RECOVERY_DECISIONS = [
'confirm_succeeded',
'confirm_failed',
'abandon_unknown',
] as const;
export type ApprovedActionRecoveryControlStatus =
(typeof APPROVED_ACTION_RECOVERY_CONTROL_STATUSES)[number];
export type ApprovedActionRecoveryFinding =
(typeof APPROVED_ACTION_RECOVERY_FINDINGS)[number];
export type ApprovedActionRecoverySource =
(typeof APPROVED_ACTION_RECOVERY_SOURCES)[number];
export type ApprovedActionRecoveryDecision =
(typeof APPROVED_ACTION_RECOVERY_DECISIONS)[number];
export interface ApprovedActionRecoveryControlRecord {
dispatchId: string;
projectId: string;
executionVersion: number;
status: ApprovedActionRecoveryControlStatus;
version: number;
nextScanAtMs: number | null;
leaseOwner: string | null;
leaseToken: string | null;
leaseExpiresAtMs: number | null;
findingCount: number;
lastFindingMutationId: string | null;
lastFinding: ApprovedActionRecoveryFinding | null;
lastResultCode: string | null;
lastEvidenceDigest: string | null;
resolutionMutationId: string | null;
createdAtMs: number;
updatedAtMs: number;
}
export interface ApprovedActionRecoveryResolutionRecord {
dispatchId: string;
projectId: string;
executionVersion: number;
mutationId: string;
source: ApprovedActionRecoverySource;
decision: ApprovedActionRecoveryDecision;
evidenceDigest: string | null;
reasonCode: string;
resolvedBy: Readonly<PolicySubject> | null;
resolvedAtMs: number;
}
export interface ApprovedActionRecoverySnapshot {
action: Readonly<ApprovedActionDispatchExecutionSnapshot>;
recovery: Readonly<ApprovedActionRecoveryControlRecord>;
resolution: Readonly<ApprovedActionRecoveryResolutionRecord> | null;
}
export interface ApprovedActionRecoveryCursor {
nextScanAtMs: number;
dispatchId: string;
}
export const MAX_APPROVED_ACTION_RECOVERY_VERSION = 2_147_483_647;
export const MAX_APPROVED_ACTION_RECOVERY_FINDINGS = 2_147_483_647;
const EVIDENCE_DIGEST_PATTERN = /^[0-9a-f]{64}$/;
export class InvalidApprovedActionRecoveryError extends TypeError {
constructor(message: string) {
super(`Approved action recovery is invalid: ${message}`);
this.name = 'InvalidApprovedActionRecoveryError';
}
}
export class ApprovedActionRecoveryFenceRejectedError extends Error {
readonly code = 'APPROVED_ACTION_RECOVERY_FENCE_REJECTED';
constructor() {
super('Approved action recovery fence was rejected');
this.name = 'ApprovedActionRecoveryFenceRejectedError';
}
}
export class ApprovedActionRecoveryBindingConflictError extends Error {
readonly code = 'APPROVED_ACTION_RECOVERY_BINDING_CONFLICT';
constructor() {
super('Approved action recovery identity does not match its execution');
this.name = 'ApprovedActionRecoveryBindingConflictError';
}
}
export class ApprovedActionRecoveryRepositoryError extends Error {
readonly code = 'APPROVED_ACTION_RECOVERY_REPOSITORY_ERROR';
constructor() {
super('Approved action recovery repository is unavailable');
this.name = 'ApprovedActionRecoveryRepositoryError';
}
}
function assertExactKeys(
name: string,
value: object,
expected: readonly string[],
): void {
const keys = Object.keys(value).sort();
const canonical = [...expected].sort();
if (
keys.length !== canonical.length ||
keys.some((key, index) => key !== canonical[index])
) {
throw new InvalidApprovedActionRecoveryError(`${name} shape is invalid`);
}
}
function assertInteger(
name: string,
value: number,
minimum: number,
maximum: number,
): void {
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
throw new InvalidApprovedActionRecoveryError(`${name} is invalid`);
}
}
function allNull(values: readonly unknown[]): boolean {
return values.every((value) => value === null);
}
function allPresent(values: readonly unknown[]): boolean {
return values.every((value) => value !== null);
}
export function assertApprovedActionEvidenceDigest(value: string): void {
if (typeof value !== 'string' || !EVIDENCE_DIGEST_PATTERN.test(value)) {
throw new InvalidApprovedActionRecoveryError(
'evidence digest must be a lowercase SHA-256 digest',
);
}
}
export function normalizeApprovedActionRecoveryCursor(
value: ApprovedActionRecoveryCursor,
): Readonly<ApprovedActionRecoveryCursor> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidApprovedActionRecoveryError('cursor must be an object');
}
assertExactKeys('cursor', value, ['nextScanAtMs', 'dispatchId']);
assertApprovalTimestamp('nextScanAtMs', value.nextScanAtMs);
assertApprovalMutationId(value.dispatchId);
return Object.freeze({ ...value });
}
export function normalizeApprovedActionRecoveryControlRecord(
value: ApprovedActionRecoveryControlRecord,
): Readonly<ApprovedActionRecoveryControlRecord> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidApprovedActionRecoveryError(
'recovery control must be an object',
);
}
assertExactKeys('recovery control', value, [
'dispatchId',
'projectId',
'executionVersion',
'status',
'version',
'nextScanAtMs',
'leaseOwner',
'leaseToken',
'leaseExpiresAtMs',
'findingCount',
'lastFindingMutationId',
'lastFinding',
'lastResultCode',
'lastEvidenceDigest',
'resolutionMutationId',
'createdAtMs',
'updatedAtMs',
]);
assertApprovalMutationId(value.dispatchId);
assertProjectPolicyProjectId(value.projectId);
if (!APPROVED_ACTION_RECOVERY_CONTROL_STATUSES.includes(value.status)) {
throw new InvalidApprovedActionRecoveryError('status is invalid');
}
assertInteger(
'execution version',
value.executionVersion,
1,
MAX_APPROVED_ACTION_RECOVERY_VERSION,
);
assertInteger(
'version',
value.version,
0,
MAX_APPROVED_ACTION_RECOVERY_VERSION,
);
assertInteger(
'finding count',
value.findingCount,
0,
MAX_APPROVED_ACTION_RECOVERY_FINDINGS,
);
assertApprovalTimestamp('createdAtMs', value.createdAtMs);
assertApprovalTimestamp('updatedAtMs', value.updatedAtMs);
if (value.updatedAtMs < value.createdAtMs) {
throw new InvalidApprovedActionRecoveryError('timestamps are invalid');
}
if (value.nextScanAtMs !== null) {
assertApprovalTimestamp('nextScanAtMs', value.nextScanAtMs);
if (value.nextScanAtMs <= value.updatedAtMs) {
throw new InvalidApprovedActionRecoveryError(
'next scan must be after the latest update',
);
}
}
const lease = [value.leaseOwner, value.leaseToken, value.leaseExpiresAtMs];
if (!allNull(lease) && !allPresent(lease)) {
throw new InvalidApprovedActionRecoveryError('lease tuple is incomplete');
}
const hasLease = allPresent(lease);
if (hasLease) {
assertApprovedActionLeaseIdentity(value.leaseOwner!);
assertApprovedActionLeaseIdentity(value.leaseToken!);
assertApprovalTimestamp('leaseExpiresAtMs', value.leaseExpiresAtMs!);
assertApprovedActionLeaseDuration(
value.leaseExpiresAtMs! - value.updatedAtMs,
);
}
const finding = [
value.lastFindingMutationId,
value.lastFinding,
value.lastResultCode,
];
if (!allNull(finding) && !allPresent(finding)) {
throw new InvalidApprovedActionRecoveryError('finding tuple is incomplete');
}
if (value.lastFinding !== null) {
assertApprovalMutationId(value.lastFindingMutationId!);
if (!APPROVED_ACTION_RECOVERY_FINDINGS.includes(value.lastFinding)) {
throw new InvalidApprovedActionRecoveryError('finding is invalid');
}
assertApprovedActionResultCode(value.lastResultCode!);
}
if (
(value.findingCount === 0) !==
(value.lastFindingMutationId === null &&
value.lastFinding === null &&
value.lastResultCode === null)
) {
throw new InvalidApprovedActionRecoveryError('finding count is invalid');
}
if (value.lastEvidenceDigest !== null) {
assertApprovedActionEvidenceDigest(value.lastEvidenceDigest);
}
if (value.resolutionMutationId !== null) {
assertApprovalMutationId(value.resolutionMutationId);
}
if (
value.status === 'armed' &&
(value.nextScanAtMs === null ||
hasLease ||
value.resolutionMutationId !== null)
) {
throw new InvalidApprovedActionRecoveryError('armed tuple is invalid');
}
if (
value.status === 'leased' &&
(!hasLease ||
value.nextScanAtMs !== value.leaseExpiresAtMs ||
value.resolutionMutationId !== null)
) {
throw new InvalidApprovedActionRecoveryError('leased tuple is invalid');
}
if (
value.status === 'manual_required' &&
(value.nextScanAtMs !== null ||
hasLease ||
value.findingCount < 1 ||
value.resolutionMutationId !== null)
) {
throw new InvalidApprovedActionRecoveryError(
'manual-required tuple is invalid',
);
}
if (
value.status === 'resolved' &&
(value.nextScanAtMs !== null ||
hasLease ||
value.resolutionMutationId === null)
) {
throw new InvalidApprovedActionRecoveryError('resolved tuple is invalid');
}
return Object.freeze({ ...value });
}
export function normalizeApprovedActionRecoveryResolutionRecord(
value: ApprovedActionRecoveryResolutionRecord,
): Readonly<ApprovedActionRecoveryResolutionRecord> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidApprovedActionRecoveryError(
'recovery resolution must be an object',
);
}
assertExactKeys('recovery resolution', value, [
'dispatchId',
'projectId',
'executionVersion',
'mutationId',
'source',
'decision',
'evidenceDigest',
'reasonCode',
'resolvedBy',
'resolvedAtMs',
]);
assertApprovalMutationId(value.dispatchId);
assertProjectPolicyProjectId(value.projectId);
assertInteger(
'execution version',
value.executionVersion,
1,
MAX_APPROVED_ACTION_RECOVERY_VERSION,
);
assertApprovalMutationId(value.mutationId);
if (!APPROVED_ACTION_RECOVERY_SOURCES.includes(value.source)) {
throw new InvalidApprovedActionRecoveryError('source is invalid');
}
if (!APPROVED_ACTION_RECOVERY_DECISIONS.includes(value.decision)) {
throw new InvalidApprovedActionRecoveryError('decision is invalid');
}
assertApprovedActionResultCode(value.reasonCode);
assertApprovalTimestamp('resolvedAtMs', value.resolvedAtMs);
if (value.evidenceDigest !== null) {
assertApprovedActionEvidenceDigest(value.evidenceDigest);
}
const resolvedBy = value.resolvedBy
? normalizePolicySubject(value.resolvedBy)
: null;
if (
value.source === 'automatic_evidence' &&
(value.evidenceDigest === null ||
resolvedBy !== null ||
value.decision === 'abandon_unknown')
) {
throw new InvalidApprovedActionRecoveryError(
'automatic resolution tuple is invalid',
);
}
if (value.source === 'human' && (!resolvedBy || resolvedBy.type !== 'user')) {
throw new InvalidApprovedActionRecoveryError(
'human resolution requires a User',
);
}
return Object.freeze({ ...value, resolvedBy });
}
export function assertApprovedActionRecoveryPageSize(value: number): void {
assertApprovedActionPageSize(value);
}
export function assertApprovedActionRecoveryLeaseDuration(value: number): void {
assertApprovedActionLeaseDuration(value);
}
back/runtime/domain/approvedActionRecoveryAuthorization.ts
+214
View File
@@ -0,0 +1,214 @@
import { createHash } from 'crypto';
import type { AuthenticationAssurance } from './authenticatedPrincipal';
import {
assertApprovalMutationId,
assertApprovalTimestamp,
} from './approvalRequest';
import {
assertProjectPolicyProjectId,
normalizePolicySubject,
type PolicySubject,
} from './projectPolicy';
export const APPROVED_ACTION_RECOVERY_STRONG_ASSURANCES = [
'multi_factor',
'hardware',
'local_console',
] as const satisfies readonly AuthenticationAssurance[];
export type ApprovedActionRecoveryStrongAssurance =
(typeof APPROVED_ACTION_RECOVERY_STRONG_ASSURANCES)[number];
export const MAX_APPROVED_ACTION_RECOVERY_AUTH_AGE_MS = 5 * 60 * 1000;
export const MAX_APPROVED_ACTION_RECOVERY_AUTHENTICATION_ID_LENGTH = 128;
export interface ApprovedActionRecoveryAuthorizationFact {
dispatchId: string;
projectId: string;
mutationId: string;
resolvedBy: Readonly<PolicySubject>;
authenticationId: string;
assurance: ApprovedActionRecoveryStrongAssurance;
authenticatedAtMs: number;
projectVersion: number;
bindingVersion: number;
authorizedAtMs: number;
factDigest: string;
}
const AUTHENTICATION_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/;
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
const MAX_VERSION = 2_147_483_647;
export class InvalidApprovedActionRecoveryAuthorizationError extends TypeError {
constructor(message: string) {
super(`Approved action recovery authorization is invalid: ${message}`);
this.name = 'InvalidApprovedActionRecoveryAuthorizationError';
}
}
export class ApprovedActionRecoveryHumanRequiredError extends Error {
readonly code = 'APPROVED_ACTION_RECOVERY_HUMAN_REQUIRED';
constructor() {
super('Manual recovery requires a stable authenticated User');
this.name = 'ApprovedActionRecoveryHumanRequiredError';
}
}
export class ApprovedActionRecoveryStrongAuthenticationRequiredError extends Error {
readonly code = 'APPROVED_ACTION_RECOVERY_STRONG_AUTHENTICATION_REQUIRED';
constructor() {
super('Manual recovery requires recent strong authentication');
this.name = 'ApprovedActionRecoveryStrongAuthenticationRequiredError';
}
}
export class ApprovedActionRecoveryAuthorizationDeniedError extends Error {
readonly code = 'APPROVED_ACTION_RECOVERY_AUTHORIZATION_DENIED';
constructor() {
super('Manual recovery is denied by Project policy');
this.name = 'ApprovedActionRecoveryAuthorizationDeniedError';
}
}
export class ApprovedActionRecoveryNotFoundError extends Error {
readonly code = 'APPROVED_ACTION_RECOVERY_NOT_FOUND';
constructor() {
super('Approved action recovery does not exist');
this.name = 'ApprovedActionRecoveryNotFoundError';
}
}
function exactKeys(value: object, expected: readonly string[]): void {
const keys = Object.keys(value).sort();
const canonical = [...expected].sort();
if (
keys.length !== canonical.length ||
keys.some((key, index) => key !== canonical[index])
) {
throw new InvalidApprovedActionRecoveryAuthorizationError(
'fact shape is invalid',
);
}
}
function assertVersion(name: string, value: number): void {
if (!Number.isSafeInteger(value) || value < 1 || value > MAX_VERSION) {
throw new InvalidApprovedActionRecoveryAuthorizationError(
`${name} is invalid`,
);
}
}
function canonicalFact(
fact: Omit<ApprovedActionRecoveryAuthorizationFact, 'factDigest'>,
): string {
return JSON.stringify({
dispatchId: fact.dispatchId,
projectId: fact.projectId,
mutationId: fact.mutationId,
resolvedByType: fact.resolvedBy.type,
resolvedById: fact.resolvedBy.id,
authenticationId: fact.authenticationId,
assurance: fact.assurance,
authenticatedAtMs: fact.authenticatedAtMs,
projectVersion: fact.projectVersion,
bindingVersion: fact.bindingVersion,
authorizedAtMs: fact.authorizedAtMs,
});
}
export function digestApprovedActionRecoveryAuthorizationFact(
fact: Omit<ApprovedActionRecoveryAuthorizationFact, 'factDigest'>,
): string {
return createHash('sha256').update(canonicalFact(fact), 'utf8').digest('hex');
}
export function normalizeApprovedActionRecoveryAuthorizationFact(
value: ApprovedActionRecoveryAuthorizationFact,
): Readonly<ApprovedActionRecoveryAuthorizationFact> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidApprovedActionRecoveryAuthorizationError(
'fact must be an object',
);
}
exactKeys(value, [
'dispatchId',
'projectId',
'mutationId',
'resolvedBy',
'authenticationId',
'assurance',
'authenticatedAtMs',
'projectVersion',
'bindingVersion',
'authorizedAtMs',
'factDigest',
]);
assertApprovalMutationId(value.dispatchId);
assertProjectPolicyProjectId(value.projectId);
assertApprovalMutationId(value.mutationId);
const resolvedBy = normalizePolicySubject(value.resolvedBy);
if (resolvedBy.type !== 'user') {
throw new InvalidApprovedActionRecoveryAuthorizationError(
'resolvedBy must be a User',
);
}
if (
typeof value.authenticationId !== 'string' ||
value.authenticationId.length < 1 ||
value.authenticationId.length >
MAX_APPROVED_ACTION_RECOVERY_AUTHENTICATION_ID_LENGTH ||
!AUTHENTICATION_ID_PATTERN.test(value.authenticationId)
) {
throw new InvalidApprovedActionRecoveryAuthorizationError(
'authenticationId is invalid',
);
}
if (!APPROVED_ACTION_RECOVERY_STRONG_ASSURANCES.includes(value.assurance)) {
throw new InvalidApprovedActionRecoveryAuthorizationError(
'assurance is not strong',
);
}
assertApprovalTimestamp('authenticatedAtMs', value.authenticatedAtMs);
assertApprovalTimestamp('authorizedAtMs', value.authorizedAtMs);
if (
value.authenticatedAtMs > value.authorizedAtMs ||
value.authorizedAtMs - value.authenticatedAtMs >
MAX_APPROVED_ACTION_RECOVERY_AUTH_AGE_MS
) {
throw new InvalidApprovedActionRecoveryAuthorizationError(
'authentication is not recent',
);
}
assertVersion('projectVersion', value.projectVersion);
assertVersion('bindingVersion', value.bindingVersion);
if (
typeof value.factDigest !== 'string' ||
!DIGEST_PATTERN.test(value.factDigest)
) {
throw new InvalidApprovedActionRecoveryAuthorizationError(
'factDigest is invalid',
);
}
const { factDigest, ...unsigned } = value;
if (digestApprovedActionRecoveryAuthorizationFact(unsigned) !== factDigest) {
throw new InvalidApprovedActionRecoveryAuthorizationError(
'factDigest does not match',
);
}
return Object.freeze({ ...value, resolvedBy });
}
export function createApprovedActionRecoveryAuthorizationFact(
value: Omit<ApprovedActionRecoveryAuthorizationFact, 'factDigest'>,
): Readonly<ApprovedActionRecoveryAuthorizationFact> {
return normalizeApprovedActionRecoveryAuthorizationFact({
...value,
factDigest: digestApprovedActionRecoveryAuthorizationFact(value),
});
}
back/runtime/domain/approvedRunAction.ts
+286
View File
@@ -0,0 +1,286 @@
import { createHash } from 'crypto';
import { EXECUTOR_TYPES, type ExecutorType } from './execution';
import {
assertApprovalMutationId,
assertApprovalRequestId,
assertApprovalTimestamp,
} from './approvalRequest';
import { assertProjectPolicyProjectId } from './projectPolicy';
export const APPROVED_RUN_ACTION_TYPE = 'run.create';
export const APPROVED_RUN_RECEIPT_SCHEMA_VERSION = 1;
export const APPROVED_RUN_RECEIPT_RESULT_CODE = 'approved_run_created';
export interface ApprovedRunCreationPlan {
schemaVersion: 1;
actionRef: string;
projectId: string;
taskId: string;
taskRevision: string;
executorType: ExecutorType;
priority: number;
taskName?: string;
taskSnapshotRef?: string;
inputRef?: string;
}
export interface ApprovedRunCreationReceipt {
schemaVersion: 1;
dispatchId: string;
approvalRequestId: string;
projectId: string;
actionType: typeof APPROVED_RUN_ACTION_TYPE;
actionDigest: string;
executionAttempt: number;
executionVersion: number;
startedAtMs: number;
idempotencyKey: string;
outcome: 'succeeded';
resultCode: typeof APPROVED_RUN_RECEIPT_RESULT_CODE;
resourceType: 'run';
resourceId: string;
finishedAtMs: number;
evidenceDigest: string;
createdAtMs: number;
}
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/;
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
const MAX_EXECUTION_VERSION = 2_147_483_647;
export class InvalidApprovedRunActionError extends TypeError {
constructor(message: string) {
super(`Approved Run action is invalid: ${message}`);
this.name = 'InvalidApprovedRunActionError';
}
}
export class ApprovedRunActionBindingConflictError extends Error {
readonly code = 'APPROVED_RUN_ACTION_BINDING_CONFLICT';
constructor() {
super('Approved Run action identity does not match its durable receipt');
this.name = 'ApprovedRunActionBindingConflictError';
}
}
export class ApprovedRunActionRepositoryError extends Error {
readonly code = 'APPROVED_RUN_ACTION_REPOSITORY_ERROR';
constructor() {
super('Approved Run action repository is unavailable');
this.name = 'ApprovedRunActionRepositoryError';
}
}
function exactKeys(value: object, expected: readonly string[]): void {
const keys = Object.keys(value).sort();
const canonical = [...expected].sort();
if (
keys.length !== canonical.length ||
keys.some((key, index) => key !== canonical[index])
) {
throw new InvalidApprovedRunActionError('object shape is invalid');
}
}
function assertBoundedText(name: string, value: string, maximum: number): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > maximum ||
/[\u0000-\u001f\u007f]/.test(value)
) {
throw new InvalidApprovedRunActionError(`${name} is invalid`);
}
}
function assertIdentifier(name: string, value: string, maximum: number): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > maximum ||
!IDENTIFIER_PATTERN.test(value)
) {
throw new InvalidApprovedRunActionError(`${name} is invalid`);
}
}
function assertPositiveInteger(
name: string,
value: number,
maximum = MAX_EXECUTION_VERSION,
): void {
if (!Number.isSafeInteger(value) || value < 1 || value > maximum) {
throw new InvalidApprovedRunActionError(`${name} is invalid`);
}
}
export function normalizeApprovedRunCreationPlan(
value: ApprovedRunCreationPlan,
): Readonly<ApprovedRunCreationPlan> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidApprovedRunActionError('plan must be an object');
}
exactKeys(value, [
'schemaVersion',
'actionRef',
'projectId',
'taskId',
'taskRevision',
'executorType',
'priority',
...(value.taskName === undefined ? [] : ['taskName']),
...(value.taskSnapshotRef === undefined ? [] : ['taskSnapshotRef']),
...(value.inputRef === undefined ? [] : ['inputRef']),
]);
if (value.schemaVersion !== 1) {
throw new InvalidApprovedRunActionError('schema version is unsupported');
}
assertIdentifier('actionRef', value.actionRef, 255);
assertProjectPolicyProjectId(value.projectId);
assertBoundedText('taskId', value.taskId, 255);
assertBoundedText('taskRevision', value.taskRevision, 128);
if (!EXECUTOR_TYPES.includes(value.executorType)) {
throw new InvalidApprovedRunActionError('executorType is unsupported');
}
if (
!Number.isSafeInteger(value.priority) ||
value.priority < -2_147_483_648 ||
value.priority > 2_147_483_647
) {
throw new InvalidApprovedRunActionError('priority is invalid');
}
if (value.taskName !== undefined) {
assertBoundedText('taskName', value.taskName, 255);
}
if (value.taskSnapshotRef !== undefined) {
assertBoundedText('taskSnapshotRef', value.taskSnapshotRef, 512);
}
if (value.inputRef !== undefined) {
assertBoundedText('inputRef', value.inputRef, 512);
}
return Object.freeze({ ...value });
}
export function digestApprovedRunCreationPlan(
value: ApprovedRunCreationPlan,
): string {
const plan = normalizeApprovedRunCreationPlan(value);
const canonical = JSON.stringify({
schemaVersion: plan.schemaVersion,
actionType: APPROVED_RUN_ACTION_TYPE,
actionRef: plan.actionRef,
projectId: plan.projectId,
taskId: plan.taskId,
taskRevision: plan.taskRevision,
executorType: plan.executorType,
priority: plan.priority,
taskName: plan.taskName ?? null,
taskSnapshotRef: plan.taskSnapshotRef ?? null,
inputRef: plan.inputRef ?? null,
});
return createHash('sha256').update(canonical, 'utf8').digest('hex');
}
function canonicalApprovedRunReceipt(
receipt: Omit<ApprovedRunCreationReceipt, 'evidenceDigest'>,
): string {
return JSON.stringify({
schemaVersion: receipt.schemaVersion,
dispatchId: receipt.dispatchId,
approvalRequestId: receipt.approvalRequestId,
projectId: receipt.projectId,
actionType: receipt.actionType,
actionDigest: receipt.actionDigest,
executionAttempt: receipt.executionAttempt,
executionVersion: receipt.executionVersion,
startedAtMs: receipt.startedAtMs,
idempotencyKey: receipt.idempotencyKey,
outcome: receipt.outcome,
resultCode: receipt.resultCode,
resourceType: receipt.resourceType,
resourceId: receipt.resourceId,
finishedAtMs: receipt.finishedAtMs,
createdAtMs: receipt.createdAtMs,
});
}
export function digestApprovedRunCreationReceipt(
receipt: Omit<ApprovedRunCreationReceipt, 'evidenceDigest'>,
): string {
return createHash('sha256')
.update(canonicalApprovedRunReceipt(receipt), 'utf8')
.digest('hex');
}
export function normalizeApprovedRunCreationReceipt(
value: ApprovedRunCreationReceipt,
): Readonly<ApprovedRunCreationReceipt> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidApprovedRunActionError('receipt must be an object');
}
exactKeys(value, [
'schemaVersion',
'dispatchId',
'approvalRequestId',
'projectId',
'actionType',
'actionDigest',
'executionAttempt',
'executionVersion',
'startedAtMs',
'idempotencyKey',
'outcome',
'resultCode',
'resourceType',
'resourceId',
'finishedAtMs',
'evidenceDigest',
'createdAtMs',
]);
if (value.schemaVersion !== APPROVED_RUN_RECEIPT_SCHEMA_VERSION) {
throw new InvalidApprovedRunActionError(
'receipt schema version is unsupported',
);
}
assertApprovalMutationId(value.dispatchId);
assertApprovalRequestId(value.approvalRequestId);
assertProjectPolicyProjectId(value.projectId);
if (value.actionType !== APPROVED_RUN_ACTION_TYPE) {
throw new InvalidApprovedRunActionError('actionType is invalid');
}
if (!DIGEST_PATTERN.test(value.actionDigest)) {
throw new InvalidApprovedRunActionError('actionDigest is invalid');
}
assertPositiveInteger('executionAttempt', value.executionAttempt, 16);
assertPositiveInteger('executionVersion', value.executionVersion);
assertApprovalTimestamp('startedAtMs', value.startedAtMs);
if (value.idempotencyKey !== value.dispatchId) {
throw new InvalidApprovedRunActionError('idempotency binding is invalid');
}
if (
value.outcome !== 'succeeded' ||
value.resultCode !== APPROVED_RUN_RECEIPT_RESULT_CODE ||
value.resourceType !== 'run'
) {
throw new InvalidApprovedRunActionError('result tuple is invalid');
}
assertIdentifier('resourceId', value.resourceId, 64);
assertApprovalTimestamp('finishedAtMs', value.finishedAtMs);
assertApprovalTimestamp('createdAtMs', value.createdAtMs);
if (
value.finishedAtMs < value.startedAtMs ||
value.createdAtMs !== value.finishedAtMs
) {
throw new InvalidApprovedRunActionError('receipt timestamps are invalid');
}
if (!DIGEST_PATTERN.test(value.evidenceDigest)) {
throw new InvalidApprovedRunActionError('evidenceDigest is invalid');
}
const { evidenceDigest, ...unsigned } = value;
if (digestApprovedRunCreationReceipt(unsigned) !== evidenceDigest) {
throw new InvalidApprovedRunActionError('evidenceDigest does not match');
}
return Object.freeze({ ...value });
}
back/runtime/domain/artifactRead.ts
+138
View File
@@ -0,0 +1,138 @@
import { assertCompletionReceiptId } from './completionReceipt';
import {
LOCAL_ARTIFACT_RETENTION_DISPOSITIONS,
assertLocalArtifactRetentionTimestamp,
type LocalArtifactRetentionDisposition,
} from './localArtifactRetention';
import { assertLocalExecutionArtifactId } from './localExecutionArtifact';
import {
MAX_POLICY_SUBJECT_ID_LENGTH,
POLICY_SUBJECT_TYPES,
normalizePolicySubject,
type PolicySubject,
type PolicySubjectType,
} from './projectPolicy';
export const MAX_LOCAL_ARTIFACT_READ_BYTES = 256 * 1024;
export const MAX_ARTIFACT_READ_SUBJECT_ID_LENGTH = MAX_POLICY_SUBJECT_ID_LENGTH;
export const ARTIFACT_READ_SUBJECT_TYPES = POLICY_SUBJECT_TYPES;
export type ArtifactReadSubjectType = PolicySubjectType;
export type ArtifactReadSubject = PolicySubject;
export interface LocalArtifactReadRange {
offset: number;
length: number;
}
export interface LocalArtifactReadRetentionEvidence {
disposition: LocalArtifactRetentionDisposition;
finishedAtMs: number;
eligibleAtMs: number;
bytesReclaimed: number;
recordedAtMs: number;
}
export interface LocalArtifactReadMetadata {
projectId: string;
runId: string;
attemptId: string;
logArtifactId: string;
retention?: LocalArtifactReadRetentionEvidence;
}
export function assertArtifactReadProjectId(value: string): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > 128 ||
/[\u0000-\u001f\u007f]/.test(value)
) {
throw new TypeError('Artifact read projectId is invalid');
}
}
export function normalizeArtifactReadSubject(
subject: ArtifactReadSubject,
): Readonly<ArtifactReadSubject> {
return normalizePolicySubject(subject);
}
export function normalizeLocalArtifactReadRange(
range: LocalArtifactReadRange,
): Readonly<LocalArtifactReadRange> {
if (!range || typeof range !== 'object' || Array.isArray(range)) {
throw new TypeError('Local Artifact read range must be an object');
}
if (!Number.isSafeInteger(range.offset) || range.offset < 0) {
throw new RangeError('Local Artifact read offset is invalid');
}
if (
!Number.isSafeInteger(range.length) ||
range.length < 1 ||
range.length > MAX_LOCAL_ARTIFACT_READ_BYTES
) {
throw new RangeError('Local Artifact read length is invalid');
}
return Object.freeze({ offset: range.offset, length: range.length });
}
export function normalizeLocalArtifactReadMetadata(
value: LocalArtifactReadMetadata,
): Readonly<LocalArtifactReadMetadata> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new TypeError('Local Artifact read metadata must be an object');
}
assertArtifactReadProjectId(value.projectId);
assertCompletionReceiptId(value.runId, 'runId');
assertCompletionReceiptId(value.attemptId, 'attemptId');
assertLocalExecutionArtifactId(value.logArtifactId);
let retention: Readonly<LocalArtifactReadRetentionEvidence> | undefined;
if (value.retention) {
if (
!LOCAL_ARTIFACT_RETENTION_DISPOSITIONS.includes(
value.retention.disposition,
)
) {
throw new TypeError('Local Artifact retention disposition is invalid');
}
assertLocalArtifactRetentionTimestamp(
'finishedAtMs',
value.retention.finishedAtMs,
);
assertLocalArtifactRetentionTimestamp(
'eligibleAtMs',
value.retention.eligibleAtMs,
);
assertLocalArtifactRetentionTimestamp(
'bytesReclaimed',
value.retention.bytesReclaimed,
);
assertLocalArtifactRetentionTimestamp(
'recordedAtMs',
value.retention.recordedAtMs,
);
if (value.retention.eligibleAtMs < value.retention.finishedAtMs) {
throw new TypeError('Artifact retention eligibility is invalid');
}
if (value.retention.recordedAtMs < value.retention.eligibleAtMs) {
throw new TypeError('Artifact retention recording time is invalid');
}
if (
value.retention.disposition === 'already_absent' &&
value.retention.bytesReclaimed !== 0
) {
throw new TypeError(
'Absent Artifact retention reclaimed bytes is invalid',
);
}
retention = Object.freeze({ ...value.retention });
}
return Object.freeze({
projectId: value.projectId,
runId: value.runId,
attemptId: value.attemptId,
logArtifactId: value.logArtifactId,
...(retention ? { retention } : {}),
});
}
back/runtime/domain/authenticatedPrincipal.ts
+109
View File
@@ -0,0 +1,109 @@
import {
InvalidProjectPolicyValueError,
normalizePolicySubject,
type PolicySubject,
} from './projectPolicy';
export const AUTHENTICATION_ASSURANCE_LEVELS = [
'single_factor',
'multi_factor',
'service',
'hardware',
'local_console',
] as const;
export type AuthenticationAssurance =
(typeof AUTHENTICATION_ASSURANCE_LEVELS)[number];
export interface AuthenticatedPrincipal {
subject: PolicySubject;
authenticationId: string;
authenticatedAtMs: number;
expiresAtMs: number;
assurance: AuthenticationAssurance;
}
export const MAX_AUTHENTICATION_ID_LENGTH = 128;
const AUTHENTICATION_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/;
function assertTimestamp(name: string, value: number): void {
if (!Number.isSafeInteger(value) || value < 0) {
throw new InvalidProjectPolicyValueError(`${name} is invalid`);
}
}
export function normalizeAuthenticatedPrincipal(
value: AuthenticatedPrincipal,
): Readonly<AuthenticatedPrincipal> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidProjectPolicyValueError(
'authenticated principal must be an object',
);
}
const keys = Object.keys(value).sort();
const expected = [
'assurance',
'authenticatedAtMs',
'authenticationId',
'expiresAtMs',
'subject',
];
if (
keys.length !== expected.length ||
keys.some((key, index) => key !== expected[index])
) {
throw new InvalidProjectPolicyValueError(
'authenticated principal shape is invalid',
);
}
const subject = normalizePolicySubject(value.subject);
if (
typeof value.authenticationId !== 'string' ||
value.authenticationId.length < 1 ||
value.authenticationId.length > MAX_AUTHENTICATION_ID_LENGTH ||
!AUTHENTICATION_ID_PATTERN.test(value.authenticationId)
) {
throw new InvalidProjectPolicyValueError('authenticationId is invalid');
}
assertTimestamp('authenticatedAtMs', value.authenticatedAtMs);
assertTimestamp('expiresAtMs', value.expiresAtMs);
if (value.expiresAtMs <= value.authenticatedAtMs) {
throw new InvalidProjectPolicyValueError(
'authenticated principal lifetime is invalid',
);
}
if (!AUTHENTICATION_ASSURANCE_LEVELS.includes(value.assurance)) {
throw new InvalidProjectPolicyValueError(
'authentication assurance is invalid',
);
}
return Object.freeze({
subject,
authenticationId: value.authenticationId,
authenticatedAtMs: value.authenticatedAtMs,
expiresAtMs: value.expiresAtMs,
assurance: value.assurance,
});
}
export function assertAuthenticatedPrincipalActive(
principal: Readonly<AuthenticatedPrincipal>,
nowMs: number,
): void {
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
throw new InvalidProjectPolicyValueError('current time is invalid');
}
if (principal.authenticatedAtMs > nowMs || principal.expiresAtMs <= nowMs) {
throw new AuthenticatedPrincipalExpiredError();
}
}
export class AuthenticatedPrincipalExpiredError extends Error {
readonly code = 'AUTHENTICATED_PRINCIPAL_EXPIRED';
constructor() {
super('Authenticated principal is not active');
this.name = 'AuthenticatedPrincipalExpiredError';
}
}
back/runtime/domain/cancellationDispatch.ts
+41
View File
@@ -0,0 +1,41 @@
export const CANCELLATION_DISPATCH_STATUSES = [
'pending',
'leased',
'retry_wait',
'dispatched',
'blocked',
] as const;
export type CancellationDispatchStatus =
(typeof CANCELLATION_DISPATCH_STATUSES)[number];
export const CANCELLATION_DISPATCH_RESULTS = [
'termination_requested',
'already_exited',
'identity_mismatch',
'pid_mismatch',
'unsupported',
'invalid',
'controller_missing',
'handle_missing',
'dispatch_error',
] as const;
export type CancellationDispatchResult =
(typeof CANCELLATION_DISPATCH_RESULTS)[number];
export interface CancellationDispatchRecord {
runId: string;
attemptId: string;
status: CancellationDispatchStatus;
version: number;
dispatchCount: number;
nextAttemptAtMs?: number;
leaseOwner?: string;
leaseToken?: string;
leaseExpiresAtMs?: number;
lastResult?: CancellationDispatchResult;
lastDispatchedAtMs?: number;
createdAtMs: number;
updatedAtMs: number;
}
back/runtime/domain/cancellationDispatchErrors.ts
+46
View File
@@ -0,0 +1,46 @@
export class CancellationDispatchError extends Error {
constructor(
message: string,
public readonly code: string,
public readonly retryable = false,
public readonly cause?: unknown,
) {
super(message);
this.name = new.target.name;
}
}
export class InvalidCancellationDispatchCommandError extends CancellationDispatchError {
constructor(message: string) {
super(message, 'INVALID_CANCELLATION_DISPATCH_COMMAND');
}
}
export class CancellationDispatchBindingConflictError extends CancellationDispatchError {
constructor(runId: string, attemptId: string) {
super(
`Cancellation dispatch for Run ${runId} is already bound to another Attempt than ${attemptId}`,
'CANCELLATION_DISPATCH_BINDING_CONFLICT',
);
}
}
export class CancellationDispatchFenceRejectedError extends CancellationDispatchError {
constructor(runId: string) {
super(
`Cancellation dispatch lease for Run ${runId} is stale or no longer owned by this worker`,
'CANCELLATION_DISPATCH_FENCE_REJECTED',
);
}
}
export class CancellationDispatchRepositoryError extends CancellationDispatchError {
constructor(cause?: unknown) {
super(
'Cancellation dispatch repository operation failed',
'CANCELLATION_DISPATCH_REPOSITORY_FAILED',
false,
cause,
);
}
}
back/runtime/domain/completionReceipt.ts
+218
View File
@@ -0,0 +1,218 @@
export const COMPLETION_RECEIPT_SCHEMA_VERSION = 1;
export const MAX_COMPLETION_RECEIPT_BYTES = 4 * 1024;
const RECEIPT_KEYS = [
'schemaVersion',
'runId',
'attemptId',
'callbackSequence',
'token',
'startedAtMs',
'finishedAtMs',
'exitCode',
] as const;
const RECEIPT_KEY_SET = new Set<string>(RECEIPT_KEYS);
const UUID_V7_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
const TOKEN_PATTERN = /^[A-Za-z0-9_-]{32,128}$/;
export interface CompletionReceipt {
schemaVersion: typeof COMPLETION_RECEIPT_SCHEMA_VERSION;
runId: string;
attemptId: string;
callbackSequence: number;
token: string;
startedAtMs: number;
finishedAtMs: number;
exitCode: number;
}
export class InvalidCompletionReceiptError extends Error {
constructor(message: string) {
super(message);
this.name = 'InvalidCompletionReceiptError';
}
}
function invalid(message: string): never {
throw new InvalidCompletionReceiptError(message);
}
function skipWhitespace(value: string, from: number): number {
let index = from;
while (/\s/.test(value[index] ?? '')) index += 1;
return index;
}
function stringEnd(value: string, from: number): number {
if (value[from] !== '"') invalid('Completion receipt key is not a string');
for (let index = from + 1; index < value.length; index += 1) {
if (value[index] === '\\') {
index += 1;
continue;
}
if (value[index] === '"') return index + 1;
}
return invalid('Completion receipt contains an unterminated string');
}
/** JSON.parse silently accepts duplicate keys, so reject them before parsing. */
function assertUniqueFlatObjectKeys(value: string): void {
let index = skipWhitespace(value, 0);
if (value[index] !== '{') invalid('Completion receipt must be a JSON object');
index = skipWhitespace(value, index + 1);
const keys = new Set<string>();
if (value[index] === '}') return;
while (index < value.length) {
const keyStart = index;
const keyEnd = stringEnd(value, keyStart);
let key: string;
try {
key = JSON.parse(value.slice(keyStart, keyEnd));
} catch {
return invalid('Completion receipt contains an invalid key');
}
if (keys.has(key)) invalid('Completion receipt contains a duplicate key');
keys.add(key);
index = skipWhitespace(value, keyEnd);
if (value[index] !== ':') invalid('Completion receipt key has no value');
index = skipWhitespace(value, index + 1);
if (value[index] === '"') {
index = stringEnd(value, index);
} else {
if (value[index] === '{' || value[index] === '[') {
invalid('Completion receipt values must be scalar');
}
const valueStart = index;
while (
index < value.length &&
value[index] !== ',' &&
value[index] !== '}'
) {
index += 1;
}
if (value.slice(valueStart, index).trim().length === 0) {
invalid('Completion receipt contains an empty value');
}
}
index = skipWhitespace(value, index);
if (value[index] === '}') return;
if (value[index] !== ',') invalid('Completion receipt is not a flat object');
index = skipWhitespace(value, index + 1);
}
invalid('Completion receipt object is incomplete');
}
export function assertCompletionReceiptId(
value: string,
field: 'runId' | 'attemptId',
): void {
if (!UUID_V7_PATTERN.test(value)) {
invalid(`${field} must be a lowercase UUIDv7`);
}
}
function assertSafeTimestamp(value: unknown, field: string): asserts value is number {
if (!Number.isSafeInteger(value) || (value as number) < 0) {
invalid(`${field} must be a non-negative safe integer`);
}
}
export function validateCompletionReceipt(
candidate: unknown,
): CompletionReceipt {
if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) {
invalid('Completion receipt must be an object');
}
const record = candidate as Record<string, unknown>;
const keys = Object.keys(record);
if (
keys.length !== RECEIPT_KEYS.length ||
keys.some((key) => !RECEIPT_KEY_SET.has(key)) ||
RECEIPT_KEYS.some((key) => !Object.hasOwn(record, key))
) {
invalid('Completion receipt fields do not match schema version 1');
}
if (record.schemaVersion !== COMPLETION_RECEIPT_SCHEMA_VERSION) {
invalid('Completion receipt schema version is unsupported');
}
if (typeof record.runId !== 'string') invalid('runId must be a string');
if (typeof record.attemptId !== 'string') {
invalid('attemptId must be a string');
}
assertCompletionReceiptId(record.runId, 'runId');
assertCompletionReceiptId(record.attemptId, 'attemptId');
if (
!Number.isSafeInteger(record.callbackSequence) ||
(record.callbackSequence as number) < 1
) {
invalid('callbackSequence must be a positive safe integer');
}
if (typeof record.token !== 'string' || !TOKEN_PATTERN.test(record.token)) {
invalid('token must be a bounded base64url value');
}
assertSafeTimestamp(record.startedAtMs, 'startedAtMs');
assertSafeTimestamp(record.finishedAtMs, 'finishedAtMs');
if (record.finishedAtMs < record.startedAtMs) {
invalid('finishedAtMs must not be before startedAtMs');
}
if (
!Number.isInteger(record.exitCode) ||
(record.exitCode as number) < 0 ||
(record.exitCode as number) > 255
) {
invalid('exitCode must be an integer between 0 and 255');
}
return {
schemaVersion: COMPLETION_RECEIPT_SCHEMA_VERSION,
runId: record.runId,
attemptId: record.attemptId,
callbackSequence: record.callbackSequence as number,
token: record.token,
startedAtMs: record.startedAtMs,
finishedAtMs: record.finishedAtMs,
exitCode: record.exitCode as number,
};
}
export function serializeCompletionReceipt(receipt: CompletionReceipt): string {
const value = validateCompletionReceipt(receipt);
const serialized = JSON.stringify({
schemaVersion: value.schemaVersion,
runId: value.runId,
attemptId: value.attemptId,
callbackSequence: value.callbackSequence,
token: value.token,
startedAtMs: value.startedAtMs,
finishedAtMs: value.finishedAtMs,
exitCode: value.exitCode,
});
if (Buffer.byteLength(serialized, 'utf8') > MAX_COMPLETION_RECEIPT_BYTES) {
invalid('Completion receipt exceeds the byte limit');
}
return serialized;
}
export function parseCompletionReceipt(
input: string | Uint8Array,
): CompletionReceipt {
const bytes = typeof input === 'string' ? Buffer.from(input) : Buffer.from(input);
if (bytes.length === 0 || bytes.length > MAX_COMPLETION_RECEIPT_BYTES) {
invalid('Completion receipt size is outside the allowed range');
}
const value = bytes.toString('utf8');
if (!Buffer.from(value, 'utf8').equals(bytes)) {
invalid('Completion receipt must be valid UTF-8');
}
assertUniqueFlatObjectKeys(value);
let parsed: unknown;
try {
parsed = JSON.parse(value);
} catch {
return invalid('Completion receipt is not valid JSON');
}
return validateCompletionReceipt(parsed);
}
back/runtime/domain/completionReceiptJournal.ts
+37
View File
@@ -0,0 +1,37 @@
import type { RunAttemptStatus } from './run';
export const COMPLETION_RECEIPT_JOURNAL_STATES = [
'pending',
'quarantined',
] as const;
export type CompletionReceiptJournalState =
(typeof COMPLETION_RECEIPT_JOURNAL_STATES)[number];
export interface CompletionReceiptJournalRecord {
attemptId: string;
runId: string;
state: CompletionReceiptJournalState;
quarantineRef?: string;
purgeAfterMs?: number;
registeredAtMs: number;
updatedAtMs: number;
}
export interface CompletionReceiptJournalCandidate
extends CompletionReceiptJournalRecord {
attemptStatus: RunAttemptStatus;
executorType: string;
finishedAtMs?: number;
}
export interface CompletionReceiptJournalCursor {
updatedAtMs: number;
attemptId: string;
}
export interface CompletionReceiptJournalPage {
candidates: readonly CompletionReceiptJournalCandidate[];
truncated: boolean;
nextCursor?: CompletionReceiptJournalCursor;
}
back/runtime/domain/deploymentProfile.ts
+224
View File
@@ -0,0 +1,224 @@
export const DEPLOYMENT_PROFILES = [
'edge',
'standalone',
'cluster-control',
'worker',
] as const;
export type DeploymentProfile = (typeof DEPLOYMENT_PROFILES)[number];
export interface BoundedRecoveryResourcePolicy {
intervalMs: number;
initialDelayMs: number;
stopTimeoutMs: number;
pageSize: number;
maxPages: number;
}
export interface LocalArtifactRetentionResourcePolicy {
intervalMs: number;
initialDelayMs: number;
stopTimeoutMs: number;
pageSize: number;
maximumDeletions: number;
normalRetentionMs: number;
pressureRetentionMs: number;
}
export interface ApprovedActionResourcePolicy {
intervalMs: number;
initialDelayMs: number;
stopTimeoutMs: number;
dispatch: {
pageSize: number;
maxPages: number;
};
recovery: {
pageSize: number;
maxPages: number;
};
}
export interface LocalPrimaryResourcePolicy {
profile: 'edge' | 'standalone';
receiptPublishGraceMs: number;
receiptTerminalMissingRetentionMs: number;
receiptQuarantineRetentionMs: number;
completion: BoundedRecoveryResourcePolicy;
cancellation: BoundedRecoveryResourcePolicy;
timeout: BoundedRecoveryResourcePolicy;
retry: BoundedRecoveryResourcePolicy;
approvedAction: ApprovedActionResourcePolicy;
artifactRetention: LocalArtifactRetentionResourcePolicy;
}
const VALID_PROFILES = new Set<DeploymentProfile>(DEPLOYMENT_PROFILES);
const LOCAL_PRIMARY_POLICIES: Record<
LocalPrimaryResourcePolicy['profile'],
LocalPrimaryResourcePolicy
> = {
edge: {
profile: 'edge',
receiptPublishGraceMs: 50,
receiptTerminalMissingRetentionMs: 2 * 60_000,
receiptQuarantineRetentionMs: 5 * 60_000,
completion: {
intervalMs: 30_000,
initialDelayMs: 0,
stopTimeoutMs: 5_000,
pageSize: 8,
maxPages: 2,
},
cancellation: {
intervalMs: 5_000,
initialDelayMs: 0,
stopTimeoutMs: 5_000,
pageSize: 8,
maxPages: 2,
},
timeout: {
intervalMs: 30_000,
initialDelayMs: 0,
stopTimeoutMs: 5_000,
pageSize: 8,
maxPages: 2,
},
retry: {
intervalMs: 30_000,
initialDelayMs: 0,
stopTimeoutMs: 5_000,
pageSize: 8,
maxPages: 1,
},
approvedAction: {
intervalMs: 30_000,
initialDelayMs: 0,
stopTimeoutMs: 5_000,
dispatch: { pageSize: 8, maxPages: 1 },
recovery: { pageSize: 8, maxPages: 1 },
},
artifactRetention: {
intervalMs: 5 * 60_000,
initialDelayMs: 0,
stopTimeoutMs: 5_000,
pageSize: 8,
maximumDeletions: 4,
normalRetentionMs: 7 * 24 * 60 * 60_000,
pressureRetentionMs: 24 * 60 * 60_000,
},
},
standalone: {
profile: 'standalone',
receiptPublishGraceMs: 100,
receiptTerminalMissingRetentionMs: 60_000,
receiptQuarantineRetentionMs: 60 * 60_000,
completion: {
intervalMs: 2_000,
initialDelayMs: 0,
stopTimeoutMs: 5_000,
pageSize: 32,
maxPages: 4,
},
cancellation: {
intervalMs: 1_000,
initialDelayMs: 0,
stopTimeoutMs: 5_000,
pageSize: 32,
maxPages: 4,
},
timeout: {
intervalMs: 5_000,
initialDelayMs: 0,
stopTimeoutMs: 5_000,
pageSize: 32,
maxPages: 4,
},
retry: {
intervalMs: 5_000,
initialDelayMs: 0,
stopTimeoutMs: 5_000,
pageSize: 32,
maxPages: 1,
},
approvedAction: {
intervalMs: 2_000,
initialDelayMs: 0,
stopTimeoutMs: 5_000,
dispatch: { pageSize: 32, maxPages: 4 },
recovery: { pageSize: 16, maxPages: 2 },
},
artifactRetention: {
intervalMs: 60_000,
initialDelayMs: 0,
stopTimeoutMs: 5_000,
pageSize: 32,
maximumDeletions: 16,
normalRetentionMs: 30 * 24 * 60 * 60_000,
pressureRetentionMs: 7 * 24 * 60 * 60_000,
},
},
};
export function parseDeploymentProfile(
value: string | undefined,
): DeploymentProfile {
if (value === undefined || value === '') return 'standalone';
if (
value.trim() !== value ||
!VALID_PROFILES.has(value as DeploymentProfile)
) {
throw new TypeError('QL_DEPLOYMENT_PROFILE is invalid');
}
return value as DeploymentProfile;
}
function cloneRecoveryPolicy(
policy: BoundedRecoveryResourcePolicy,
): BoundedRecoveryResourcePolicy {
return { ...policy };
}
function cloneArtifactRetentionPolicy(
policy: LocalArtifactRetentionResourcePolicy,
): LocalArtifactRetentionResourcePolicy {
return { ...policy };
}
function cloneApprovedActionPolicy(
policy: ApprovedActionResourcePolicy,
): ApprovedActionResourcePolicy {
return {
...policy,
dispatch: { ...policy.dispatch },
recovery: { ...policy.recovery },
};
}
/**
* The incubating SQLite + LocalProcess Primary stack is intentionally local.
* Cluster control needs PostgreSQL/shared artifacts; worker has a separate boot
* topology. Refusing those profiles prevents accidental shared-SQLite clusters.
*/
export function localPrimaryResourcePolicy(
profile: DeploymentProfile,
): LocalPrimaryResourcePolicy {
if (profile !== 'edge' && profile !== 'standalone') {
throw new TypeError(
`Deployment profile ${profile} cannot host the local SQLite Primary stack`,
);
}
const policy = LOCAL_PRIMARY_POLICIES[profile];
return {
profile,
receiptPublishGraceMs: policy.receiptPublishGraceMs,
receiptTerminalMissingRetentionMs: policy.receiptTerminalMissingRetentionMs,
receiptQuarantineRetentionMs: policy.receiptQuarantineRetentionMs,
completion: cloneRecoveryPolicy(policy.completion),
cancellation: cloneRecoveryPolicy(policy.cancellation),
timeout: cloneRecoveryPolicy(policy.timeout),
retry: cloneRecoveryPolicy(policy.retry),
approvedAction: cloneApprovedActionPolicy(policy.approvedAction),
artifactRetention: cloneArtifactRetentionPolicy(policy.artifactRetention),
};
}
back/runtime/domain/execution.ts
+154
View File
@@ -0,0 +1,154 @@
export const EXECUTOR_TYPES = [
'local_process',
'docker',
'kubernetes',
'remote_worker',
] as const;
export type ExecutorType = (typeof EXECUTOR_TYPES)[number];
export type ExecutionCommand =
| {
kind: 'argv';
file: string;
args: readonly string[];
}
| {
kind: 'shell';
command: string;
shell?: string;
};
export type ExecutionLimitEnforcement = 'required' | 'best_effort';
export interface ExecutionNumericLimit {
value: number;
enforcement: ExecutionLimitEnforcement;
}
export interface ExecutionResourcePolicy {
memoryBytes?: ExecutionNumericLimit;
cpuMillisPerSecond?: ExecutionNumericLimit;
filesystemIsolation?: ExecutionLimitEnforcement;
networkIsolation?: ExecutionLimitEnforcement;
}
export interface ExecutionSpec {
runId: string;
attemptId: string;
projectId: string;
taskId: string;
taskRevision: string;
command: ExecutionCommand;
workingDirectory?: string;
environmentPolicy: 'inherit' | 'isolated';
timeoutMs?: number;
terminationGraceMs: number;
resourcePolicy?: ExecutionResourcePolicy;
}
export type ExecutionOutputStream = 'stdout' | 'stderr';
export interface ExecutionOutputChunk {
stream: ExecutionOutputStream;
chunk: Uint8Array;
observedAtMs: number;
}
export interface ExecutionOutputSink {
write(output: ExecutionOutputChunk): Promise<void>;
}
export interface ExecutionAbortSignal {
readonly aborted: boolean;
addEventListener?: (
type: 'abort',
listener: () => void,
options?: { once?: boolean },
) => void;
removeEventListener?: (type: 'abort', listener: () => void) => void;
}
export interface ExecutionContext {
environment: Readonly<Record<string, string>>;
signal?: ExecutionAbortSignal;
output: ExecutionOutputSink;
/** Ephemeral capability for the durable completion launcher; never persist raw. */
completionCallback?: {
token: string;
callbackSequence: number;
};
}
export type ExecutionOutcome =
| 'succeeded'
| 'failed'
| 'cancelled'
| 'timed_out'
| 'lost';
export interface ExecutionDiagnostic {
code: string;
summary: string;
}
export interface ExecutionResult {
outcome: ExecutionOutcome;
startedAtMs: number;
finishedAtMs: number;
exitCode?: number;
signal?: NodeJS.Signals;
errorCode?: string;
errorSummary?: string;
diagnostics?: readonly ExecutionDiagnostic[];
}
export interface ExecutionHandle {
id: string;
/** Opaque, bounded identity used to verify ownership after a process restart. */
durableHandle?: string;
executorType: ExecutorType;
runId: string;
attemptId: string;
startedAtMs: number;
pid?: number;
completion: Promise<ExecutionResult>;
}
export type ExecutionStopKind =
| 'user'
| 'policy'
| 'shutdown'
| 'reconcile'
| 'timeout';
export interface ExecutionStopReason {
kind: ExecutionStopKind;
requestedAtMs: number;
}
export interface ExecutionStopResult {
status: 'already_exited' | 'termination_requested';
termSignalSent: boolean;
killSignalSent: boolean;
}
export type ExecutionInspectionStatus = 'running' | 'stopping' | 'exited';
export interface ExecutionInspection {
status: ExecutionInspectionStatus;
result?: ExecutionResult;
}
export type ExecutorCapabilityLevel = 'none' | 'best_effort' | 'enforced';
export interface ExecutorCapabilities {
timeout: boolean;
processGroupTermination: boolean;
workingDirectory: boolean;
isolatedEnvironment: boolean;
memoryLimit: ExecutorCapabilityLevel;
cpuLimit: ExecutorCapabilityLevel;
filesystemIsolation: ExecutorCapabilityLevel;
networkIsolation: ExecutorCapabilityLevel;
}
back/runtime/domain/executionContext.ts
+72
View File
@@ -0,0 +1,72 @@
import type { ExecutionContext } from './execution';
export const MAX_EXECUTION_ENVIRONMENT_ENTRIES = 256;
export const MAX_EXECUTION_ENVIRONMENT_BYTES = 256 * 1024;
export const MAX_EXECUTION_ENVIRONMENT_VALUE_BYTES = 64 * 1024;
export function normalizeExecutionContext(
context: ExecutionContext,
): ExecutionContext {
if (!context || typeof context !== 'object' || Array.isArray(context)) {
throw new TypeError('ExecutionContext must be an object');
}
const environment = context.environment;
if (
!environment ||
typeof environment !== 'object' ||
Array.isArray(environment)
) {
throw new TypeError('ExecutionContext environment must be an object');
}
const entries = Object.entries(environment);
if (entries.length > MAX_EXECUTION_ENVIRONMENT_ENTRIES) {
throw new RangeError('ExecutionContext environment has too many entries');
}
const cloned: Record<string, string> = Object.create(null);
let totalBytes = 0;
for (const [name, value] of entries) {
if (
name.length < 1 ||
name.length > 255 ||
name.includes('=') ||
name.includes('\0') ||
typeof value !== 'string' ||
value.includes('\0')
) {
throw new TypeError('ExecutionContext environment entry is invalid');
}
const valueBytes = Buffer.byteLength(value, 'utf8');
if (valueBytes > MAX_EXECUTION_ENVIRONMENT_VALUE_BYTES) {
throw new RangeError('ExecutionContext environment value is too large');
}
totalBytes += Buffer.byteLength(name, 'utf8') + valueBytes;
if (totalBytes > MAX_EXECUTION_ENVIRONMENT_BYTES) {
throw new RangeError('ExecutionContext environment is too large');
}
cloned[name] = value;
}
if (
!context.output ||
typeof context.output !== 'object' ||
typeof context.output.write !== 'function'
) {
throw new TypeError('ExecutionContext output sink is invalid');
}
if (
context.signal !== undefined &&
(!context.signal ||
typeof context.signal !== 'object' ||
typeof context.signal.aborted !== 'boolean' ||
(context.signal.addEventListener !== undefined &&
typeof context.signal.addEventListener !== 'function') ||
(context.signal.removeEventListener !== undefined &&
typeof context.signal.removeEventListener !== 'function'))
) {
throw new TypeError('ExecutionContext signal is invalid');
}
return {
environment: Object.freeze(cloned),
...(context.signal === undefined ? {} : { signal: context.signal }),
output: context.output,
};
}
back/runtime/domain/executionSpec.ts
+242
View File
@@ -0,0 +1,242 @@
import path from 'path';
import type {
ExecutionNumericLimit,
ExecutionResourcePolicy,
ExecutionSpec,
} from './execution';
import { InvalidExecutionSpecError } from './executorErrors';
export const MAX_EXECUTION_TIMEOUT_MS = 7 * 24 * 60 * 60 * 1000;
export const MAX_TERMINATION_GRACE_MS = 5 * 60 * 1000;
export const MAX_EXECUTION_ARGUMENTS = 4096;
export const MAX_EXECUTION_COMMAND_BYTES = 128 * 1024;
function invalid(message: string): never {
throw new InvalidExecutionSpecError(message);
}
function assertNonEmptyIdentifier(value: string, name: string): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > 255 ||
/[\u0000-\u001f\u007f]/.test(value)
) {
invalid(
`${name} must be between 1 and 255 characters and contain no control characters`,
);
}
}
function assertSafeDuration(
value: number,
name: string,
maximum: number,
allowZero: boolean,
): void {
if (
!Number.isSafeInteger(value) ||
value < (allowZero ? 0 : 1) ||
value > maximum
) {
invalid(
`${name} must be a safe integer between ${
allowZero ? 0 : 1
} and ${maximum}`,
);
}
}
function assertEnforcement(
value: unknown,
name: string,
): asserts value is 'required' | 'best_effort' {
if (value !== 'required' && value !== 'best_effort') {
invalid(`${name} enforcement is invalid`);
}
}
function assertNumericLimit(
value: ExecutionNumericLimit | undefined,
name: string,
): void {
if (value === undefined) return;
if (!value || typeof value !== 'object' || Array.isArray(value)) {
invalid(`${name} must be an object`);
}
if (!Number.isSafeInteger(value.value) || value.value < 1) {
invalid(`${name} must be a positive safe integer`);
}
assertEnforcement(value.enforcement, name);
}
function assertResourcePolicy(
policy: ExecutionResourcePolicy | undefined,
): void {
if (policy === undefined) return;
if (!policy || typeof policy !== 'object' || Array.isArray(policy)) {
invalid('resourcePolicy must be an object');
}
assertNumericLimit(policy.memoryBytes, 'memoryBytes');
assertNumericLimit(policy.cpuMillisPerSecond, 'cpuMillisPerSecond');
if (policy.filesystemIsolation !== undefined) {
assertEnforcement(policy.filesystemIsolation, 'filesystemIsolation');
}
if (policy.networkIsolation !== undefined) {
assertEnforcement(policy.networkIsolation, 'networkIsolation');
}
}
function commandSize(spec: ExecutionSpec): number {
if (spec.command.kind === 'shell') {
return Buffer.byteLength(spec.command.command, 'utf8');
}
return (
Buffer.byteLength(spec.command.file, 'utf8') +
spec.command.args.reduce(
(total, argument) => total + Buffer.byteLength(argument, 'utf8'),
0,
)
);
}
export function assertExecutionSpec(spec: ExecutionSpec): void {
if (!spec || typeof spec !== 'object' || Array.isArray(spec)) {
invalid('ExecutionSpec must be an object');
}
assertNonEmptyIdentifier(spec.runId, 'runId');
assertNonEmptyIdentifier(spec.attemptId, 'attemptId');
assertNonEmptyIdentifier(spec.projectId, 'projectId');
assertNonEmptyIdentifier(spec.taskId, 'taskId');
assertNonEmptyIdentifier(spec.taskRevision, 'taskRevision');
assertSafeDuration(
spec.terminationGraceMs,
'terminationGraceMs',
MAX_TERMINATION_GRACE_MS,
true,
);
if (spec.timeoutMs !== undefined) {
assertSafeDuration(
spec.timeoutMs,
'timeoutMs',
MAX_EXECUTION_TIMEOUT_MS,
false,
);
}
if (
spec.environmentPolicy !== 'inherit' &&
spec.environmentPolicy !== 'isolated'
) {
invalid('environmentPolicy is invalid');
}
if (
spec.workingDirectory !== undefined &&
(typeof spec.workingDirectory !== 'string' ||
spec.workingDirectory.includes('\0') ||
!path.isAbsolute(spec.workingDirectory))
) {
invalid('workingDirectory must be an absolute path containing no NUL');
}
if (!spec.command || typeof spec.command !== 'object') {
invalid('command must be an object');
}
if (spec.command.kind === 'argv') {
if (
typeof spec.command.file !== 'string' ||
!spec.command.file ||
spec.command.file.includes('\0')
) {
invalid('argv command file must be non-empty and contain no NUL');
}
if (!Array.isArray(spec.command.args)) {
invalid('argv command args must be an array');
}
if (spec.command.args.length > MAX_EXECUTION_ARGUMENTS) {
invalid('argv command has too many arguments');
}
if (
spec.command.args.some(
(argument) => typeof argument !== 'string' || argument.includes('\0'),
)
) {
invalid('argv command arguments must be strings containing no NUL');
}
} else if (spec.command.kind === 'shell') {
if (
typeof spec.command.command !== 'string' ||
!spec.command.command ||
spec.command.command.includes('\0')
) {
invalid('shell command must be non-empty and contain no NUL');
}
if (
spec.command.shell !== undefined &&
(typeof spec.command.shell !== 'string' ||
!path.isAbsolute(spec.command.shell) ||
spec.command.shell.includes('\0'))
) {
invalid('shell must be an absolute path containing no NUL');
}
} else {
invalid('command kind is invalid');
}
if (commandSize(spec) > MAX_EXECUTION_COMMAND_BYTES) {
invalid('execution command is too large');
}
assertResourcePolicy(spec.resourcePolicy);
}
export function cloneExecutionSpec(spec: ExecutionSpec): ExecutionSpec {
assertExecutionSpec(spec);
return {
runId: spec.runId,
attemptId: spec.attemptId,
projectId: spec.projectId,
taskId: spec.taskId,
taskRevision: spec.taskRevision,
command:
spec.command.kind === 'argv'
? {
kind: 'argv',
file: spec.command.file,
args: [...spec.command.args],
}
: {
kind: 'shell',
command: spec.command.command,
...(spec.command.shell === undefined
? {}
: { shell: spec.command.shell }),
},
...(spec.workingDirectory === undefined
? {}
: { workingDirectory: spec.workingDirectory }),
environmentPolicy: spec.environmentPolicy,
...(spec.timeoutMs === undefined ? {} : { timeoutMs: spec.timeoutMs }),
terminationGraceMs: spec.terminationGraceMs,
...(spec.resourcePolicy === undefined
? {}
: {
resourcePolicy: {
...(spec.resourcePolicy.memoryBytes === undefined
? {}
: { memoryBytes: { ...spec.resourcePolicy.memoryBytes } }),
...(spec.resourcePolicy.cpuMillisPerSecond === undefined
? {}
: {
cpuMillisPerSecond: {
...spec.resourcePolicy.cpuMillisPerSecond,
},
}),
...(spec.resourcePolicy.filesystemIsolation === undefined
? {}
: {
filesystemIsolation: spec.resourcePolicy.filesystemIsolation,
}),
...(spec.resourcePolicy.networkIsolation === undefined
? {}
: { networkIsolation: spec.resourcePolicy.networkIsolation }),
},
}),
};
}
back/runtime/domain/executorErrors.ts
+50
View File
@@ -0,0 +1,50 @@
export class ExecutorError extends Error {
constructor(
message: string,
public readonly code: string,
public readonly cause?: unknown,
) {
super(message);
this.name = new.target.name;
}
}
export class InvalidExecutionSpecError extends ExecutorError {
constructor(message: string) {
super(message, 'INVALID_EXECUTION_SPEC');
}
}
export class ExecutorCapabilityUnavailableError extends ExecutorError {
constructor(public readonly capability: string) {
super(
`Required executor capability is unavailable: ${capability}`,
'EXECUTOR_CAPABILITY_UNAVAILABLE',
);
}
}
export class ExecutorStartError extends ExecutorError {
constructor(cause?: unknown) {
super(
'Executor failed to start the process',
'EXECUTOR_START_FAILED',
cause,
);
}
}
export class ExecutorHandleNotFoundError extends ExecutorError {
constructor(public readonly handleId: string) {
super(
'Execution handle is not owned by this executor instance',
'EXECUTOR_HANDLE_NOT_FOUND',
);
}
}
export class ExecutorStopError extends ExecutorError {
constructor(cause?: unknown) {
super('Executor failed to stop the process', 'EXECUTOR_STOP_FAILED', cause);
}
}
back/runtime/domain/identityDirectory.ts
+192
View File
@@ -0,0 +1,192 @@
import {
InvalidProjectPolicyValueError,
normalizePolicySubject,
type PolicySubject,
} from './projectPolicy';
export const IDENTITY_SUBJECT_STATUSES = ['active', 'disabled'] as const;
export const IDENTITY_AUTHENTICATION_BINDING_STATES = [
'active',
'revoked',
] as const;
export const LEGACY_PANEL_IDENTITY_PROVIDER = 'legacy_panel';
export const LEGACY_PANEL_PROVIDER_SUBJECT = 'singleton';
export const LEGACY_PRIMARY_USER_SUBJECT_ID = 'usr_legacy_primary';
export type IdentitySubjectStatus = (typeof IDENTITY_SUBJECT_STATUSES)[number];
export type IdentityAuthenticationBindingState =
(typeof IDENTITY_AUTHENTICATION_BINDING_STATES)[number];
export interface IdentitySubjectRecord {
subject: PolicySubject;
status: IdentitySubjectStatus;
version: number;
createdAtMs: number;
updatedAtMs: number;
}
export interface IdentityAuthenticationBindingRecord {
provider: string;
providerSubject: string;
version: number;
state: IdentityAuthenticationBindingState;
subjectId: string;
createdAtMs: number;
}
export const MAX_IDENTITY_DIRECTORY_VERSION = 2_147_483_647;
export const MAX_IDENTITY_PROVIDER_LENGTH = 64;
export const MAX_IDENTITY_PROVIDER_SUBJECT_LENGTH = 128;
const IDENTITY_PROVIDER_PATTERN = /^[a-z][a-z0-9_:-]*$/;
const IDENTITY_PROVIDER_SUBJECT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/;
export class InvalidIdentityDirectoryValueError extends TypeError {
constructor(message: string) {
super(`Identity directory value is invalid: ${message}`);
this.name = 'InvalidIdentityDirectoryValueError';
}
}
export class IdentityDirectoryUnavailableError extends Error {
readonly code = 'IDENTITY_DIRECTORY_UNAVAILABLE';
constructor() {
super('Identity directory is unavailable');
this.name = 'IdentityDirectoryUnavailableError';
}
}
function assertExactKeys(
name: string,
value: object,
expected: readonly string[],
): void {
const keys = Object.keys(value).sort();
const canonical = [...expected].sort();
if (
keys.length !== canonical.length ||
keys.some((key, index) => key !== canonical[index])
) {
throw new InvalidIdentityDirectoryValueError(`${name} shape is invalid`);
}
}
function assertTimestamp(name: string, value: number): void {
if (!Number.isSafeInteger(value) || value < 0) {
throw new InvalidIdentityDirectoryValueError(`${name} is invalid`);
}
}
function assertVersion(name: string, value: number): void {
if (
!Number.isSafeInteger(value) ||
value < 1 ||
value > MAX_IDENTITY_DIRECTORY_VERSION
) {
throw new InvalidIdentityDirectoryValueError(`${name} is invalid`);
}
}
export function assertIdentityProvider(value: string): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > MAX_IDENTITY_PROVIDER_LENGTH ||
!IDENTITY_PROVIDER_PATTERN.test(value)
) {
throw new InvalidIdentityDirectoryValueError('provider is invalid');
}
}
export function assertIdentityProviderSubject(value: string): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > MAX_IDENTITY_PROVIDER_SUBJECT_LENGTH ||
!IDENTITY_PROVIDER_SUBJECT_PATTERN.test(value)
) {
throw new InvalidIdentityDirectoryValueError('providerSubject is invalid');
}
}
export function normalizeIdentitySubjectRecord(
value: IdentitySubjectRecord,
): Readonly<IdentitySubjectRecord> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidIdentityDirectoryValueError(
'identity subject must be an object',
);
}
assertExactKeys('identity subject', value, [
'subject',
'status',
'version',
'createdAtMs',
'updatedAtMs',
]);
let subject: Readonly<PolicySubject>;
try {
subject = normalizePolicySubject(value.subject);
} catch (error) {
if (error instanceof InvalidProjectPolicyValueError) {
throw new InvalidIdentityDirectoryValueError('subject is invalid');
}
throw error;
}
if (!IDENTITY_SUBJECT_STATUSES.includes(value.status)) {
throw new InvalidIdentityDirectoryValueError('subject status is invalid');
}
assertVersion('subject version', value.version);
assertTimestamp('subject createdAtMs', value.createdAtMs);
assertTimestamp('subject updatedAtMs', value.updatedAtMs);
if (value.updatedAtMs < value.createdAtMs) {
throw new InvalidIdentityDirectoryValueError(
'subject timestamps are invalid',
);
}
return Object.freeze({
subject,
status: value.status,
version: value.version,
createdAtMs: value.createdAtMs,
updatedAtMs: value.updatedAtMs,
});
}
export function normalizeIdentityAuthenticationBindingRecord(
value: IdentityAuthenticationBindingRecord,
): Readonly<IdentityAuthenticationBindingRecord> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidIdentityDirectoryValueError(
'authentication binding must be an object',
);
}
assertExactKeys('authentication binding', value, [
'provider',
'providerSubject',
'version',
'state',
'subjectId',
'createdAtMs',
]);
assertIdentityProvider(value.provider);
assertIdentityProviderSubject(value.providerSubject);
assertVersion('authentication binding version', value.version);
if (!IDENTITY_AUTHENTICATION_BINDING_STATES.includes(value.state)) {
throw new InvalidIdentityDirectoryValueError(
'authentication binding state is invalid',
);
}
try {
normalizePolicySubject({ type: 'user', id: value.subjectId });
} catch (error) {
if (error instanceof InvalidProjectPolicyValueError) {
throw new InvalidIdentityDirectoryValueError('subjectId is invalid');
}
throw error;
}
assertTimestamp('authentication binding createdAtMs', value.createdAtMs);
return Object.freeze({ ...value });
}
back/runtime/domain/legacyExecutionSelection.ts
+42
View File
@@ -0,0 +1,42 @@
export interface LegacyExecutionIdentity {
pid?: number;
logArtifactId?: string;
}
export function selectOneLegacyExecution<T extends LegacyExecutionIdentity>(
candidates: readonly T[],
selector: LegacyExecutionIdentity,
): T[] {
const byLog =
selector.logArtifactId === undefined
? []
: candidates.filter(
(candidate) => candidate.logArtifactId === selector.logArtifactId,
);
const byPid =
selector.pid === undefined
? []
: candidates.filter((candidate) => candidate.pid === selector.pid);
if (selector.logArtifactId !== undefined && selector.pid !== undefined) {
const byPidSet = new Set(byPid);
const intersection = byLog.filter((candidate) => byPidSet.has(candidate));
if (intersection.length === 1) return intersection;
if (intersection.length > 1) return [];
if (byLog.length === 0 && byPid.length === 1) return byPid;
if (byPid.length === 0 && byLog.length === 1) return byLog;
if (byLog.length === 0 && byPid.length === 0) {
return candidates.length === 1 ? [...candidates] : [];
}
return [];
}
if (selector.logArtifactId !== undefined) {
if (byLog.length === 1) return byLog;
return byLog.length === 0 && candidates.length === 1 ? [...candidates] : [];
}
if (selector.pid !== undefined) {
if (byPid.length === 1) return byPid;
return byPid.length === 0 && candidates.length === 1 ? [...candidates] : [];
}
return candidates.length === 1 ? [...candidates] : [];
}
back/runtime/domain/localArtifactCapacity.ts
+79
View File
@@ -0,0 +1,79 @@
export const MIN_LOCAL_ARTIFACT_MAXIMUM_BYTES = 64 * 1024;
export const MAX_LOCAL_ARTIFACT_MAXIMUM_BYTES = 1024 * 1024 * 1024;
export const MAX_LOCAL_ARTIFACT_MINIMUM_FREE_BYTES = 1024 * 1024 * 1024 * 1024;
export interface LocalArtifactCapacityPolicy {
maximumAttemptBytes: number;
minimumFreeBytes: number;
}
export class LocalArtifactCapacityUnavailableError extends Error {
readonly code = 'LOCAL_ARTIFACT_CAPACITY_UNAVAILABLE';
constructor() {
super('Local Artifact capacity is unavailable');
this.name = 'LocalArtifactCapacityUnavailableError';
}
}
export class LocalArtifactQuotaExceededError extends Error {
readonly code = 'LOCAL_ARTIFACT_QUOTA_EXCEEDED';
constructor() {
super('Local Artifact reached its byte quota');
this.name = 'LocalArtifactQuotaExceededError';
}
}
function assertIntegerBetween(
name: string,
value: number,
minimum: number,
maximum: number,
): void {
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
}
}
export function normalizeLocalArtifactCapacityPolicy(
policy: LocalArtifactCapacityPolicy,
): Readonly<LocalArtifactCapacityPolicy> {
if (!policy || typeof policy !== 'object' || Array.isArray(policy)) {
throw new TypeError('Local Artifact capacity policy must be an object');
}
assertIntegerBetween(
'maximumAttemptBytes',
policy.maximumAttemptBytes,
MIN_LOCAL_ARTIFACT_MAXIMUM_BYTES,
MAX_LOCAL_ARTIFACT_MAXIMUM_BYTES,
);
assertIntegerBetween(
'minimumFreeBytes',
policy.minimumFreeBytes,
0,
MAX_LOCAL_ARTIFACT_MINIMUM_FREE_BYTES,
);
return Object.freeze({
maximumAttemptBytes: policy.maximumAttemptBytes,
minimumFreeBytes: policy.minimumFreeBytes,
});
}
export function localArtifactCapacityPolicyForProfile(
profile: 'edge' | 'standalone',
): Readonly<LocalArtifactCapacityPolicy> {
if (profile === 'edge') {
return Object.freeze({
maximumAttemptBytes: 4 * 1024 * 1024,
minimumFreeBytes: 32 * 1024 * 1024,
});
}
if (profile === 'standalone') {
return Object.freeze({
maximumAttemptBytes: 64 * 1024 * 1024,
minimumFreeBytes: 256 * 1024 * 1024,
});
}
throw new TypeError('Local Artifact capacity profile is invalid');
}
back/runtime/domain/localArtifactRetention.ts
+93
View File
@@ -0,0 +1,93 @@
import { assertCompletionReceiptId } from './completionReceipt';
import { assertLocalExecutionArtifactId } from './localExecutionArtifact';
export const LOCAL_ARTIFACT_RETENTION_DISPOSITIONS = [
'deleted',
'already_absent',
] as const;
export type LocalArtifactRetentionDisposition =
(typeof LOCAL_ARTIFACT_RETENTION_DISPOSITIONS)[number];
export interface LocalArtifactRetentionCursor {
finishedAtMs: number;
attemptId: string;
}
export interface LocalArtifactRetentionCandidate
extends LocalArtifactRetentionCursor {
logArtifactId: string;
}
export interface LocalArtifactRetentionRecord
extends LocalArtifactRetentionCandidate {
eligibleAtMs: number;
disposition: LocalArtifactRetentionDisposition;
bytesReclaimed: number;
recordedAtMs: number;
}
export function assertLocalArtifactRetentionTimestamp(
name: string,
value: number,
): void {
if (!Number.isSafeInteger(value) || value < 0) {
throw new TypeError(`${name} must be a non-negative safe integer`);
}
}
export function normalizeLocalArtifactRetentionCursor(
cursor: LocalArtifactRetentionCursor,
): Readonly<LocalArtifactRetentionCursor> {
if (!cursor || typeof cursor !== 'object' || Array.isArray(cursor)) {
throw new TypeError('Local Artifact retention cursor must be an object');
}
assertLocalArtifactRetentionTimestamp('finishedAtMs', cursor.finishedAtMs);
assertCompletionReceiptId(cursor.attemptId, 'attemptId');
return Object.freeze({
finishedAtMs: cursor.finishedAtMs,
attemptId: cursor.attemptId,
});
}
export function normalizeLocalArtifactRetentionCandidate(
candidate: LocalArtifactRetentionCandidate,
): Readonly<LocalArtifactRetentionCandidate> {
const cursor = normalizeLocalArtifactRetentionCursor(candidate);
assertLocalExecutionArtifactId(candidate.logArtifactId);
return Object.freeze({
...cursor,
logArtifactId: candidate.logArtifactId,
});
}
export function normalizeLocalArtifactRetentionRecord(
record: LocalArtifactRetentionRecord,
): Readonly<LocalArtifactRetentionRecord> {
const candidate = normalizeLocalArtifactRetentionCandidate(record);
assertLocalArtifactRetentionTimestamp('eligibleAtMs', record.eligibleAtMs);
assertLocalArtifactRetentionTimestamp('recordedAtMs', record.recordedAtMs);
assertLocalArtifactRetentionTimestamp(
'bytesReclaimed',
record.bytesReclaimed,
);
if (record.eligibleAtMs < record.finishedAtMs) {
throw new TypeError('eligibleAtMs must not precede finishedAtMs');
}
if (record.recordedAtMs < record.eligibleAtMs) {
throw new TypeError('recordedAtMs must not precede eligibleAtMs');
}
if (!LOCAL_ARTIFACT_RETENTION_DISPOSITIONS.includes(record.disposition)) {
throw new TypeError('Local Artifact retention disposition is invalid');
}
if (record.disposition === 'already_absent' && record.bytesReclaimed !== 0) {
throw new TypeError('An absent Artifact cannot reclaim bytes');
}
return Object.freeze({
...candidate,
eligibleAtMs: record.eligibleAtMs,
disposition: record.disposition,
bytesReclaimed: record.bytesReclaimed,
recordedAtMs: record.recordedAtMs,
});
}
back/runtime/domain/localArtifactRetentionCheckpoint.ts
+35
View File
@@ -0,0 +1,35 @@
import {
normalizeLocalArtifactRetentionCursor,
type LocalArtifactRetentionCursor,
} from './localArtifactRetention';
export interface LocalArtifactRetentionCheckpoint {
version: number;
cursor?: LocalArtifactRetentionCursor;
}
export function normalizeLocalArtifactRetentionCheckpoint(
value: LocalArtifactRetentionCheckpoint,
): Readonly<LocalArtifactRetentionCheckpoint> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new TypeError(
'Local Artifact retention checkpoint must be an object',
);
}
if (
!Number.isSafeInteger(value.version) ||
value.version < 0 ||
value.version >= Number.MAX_SAFE_INTEGER
) {
throw new TypeError(
'Local Artifact retention checkpoint version is invalid',
);
}
const cursor = value.cursor
? normalizeLocalArtifactRetentionCursor(value.cursor)
: undefined;
return Object.freeze({
version: value.version,
...(cursor ? { cursor } : {}),
});
}
back/runtime/domain/localArtifactTruncation.ts
+112
View File
@@ -0,0 +1,112 @@
import { assertCompletionReceiptId } from './completionReceipt';
import {
MAX_LOCAL_ARTIFACT_MAXIMUM_BYTES,
MIN_LOCAL_ARTIFACT_MAXIMUM_BYTES,
} from './localArtifactCapacity';
import { assertLocalExecutionArtifactId } from './localExecutionArtifact';
export const LOCAL_ARTIFACT_TRUNCATION_SCHEMA_VERSION = 1;
export const MAX_LOCAL_ARTIFACT_TRUNCATION_FACT_BYTES = 1024;
export interface LocalArtifactTruncationFact {
schemaVersion: 1;
runId: string;
attemptId: string;
logArtifactId: string;
maximumBytes: number;
quotaReached: boolean;
observedAtMs: number;
}
const FACT_KEYS = [
'attemptId',
'logArtifactId',
'maximumBytes',
'observedAtMs',
'quotaReached',
'runId',
'schemaVersion',
] as const;
function assertExactKeys(value: Record<string, unknown>): void {
const keys = Object.keys(value).sort();
if (
keys.length !== FACT_KEYS.length ||
keys.some((key, index) => key !== FACT_KEYS[index])
) {
throw new TypeError('Local Artifact truncation fact shape is invalid');
}
}
export function normalizeLocalArtifactTruncationFact(
value: LocalArtifactTruncationFact,
): Readonly<LocalArtifactTruncationFact> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new TypeError('Local Artifact truncation fact must be an object');
}
assertExactKeys(value as unknown as Record<string, unknown>);
if (value.schemaVersion !== LOCAL_ARTIFACT_TRUNCATION_SCHEMA_VERSION) {
throw new TypeError('Local Artifact truncation schema version is invalid');
}
assertCompletionReceiptId(value.runId, 'runId');
assertCompletionReceiptId(value.attemptId, 'attemptId');
assertLocalExecutionArtifactId(value.logArtifactId);
if (
!Number.isSafeInteger(value.maximumBytes) ||
value.maximumBytes < MIN_LOCAL_ARTIFACT_MAXIMUM_BYTES ||
value.maximumBytes > MAX_LOCAL_ARTIFACT_MAXIMUM_BYTES
) {
throw new TypeError('Local Artifact truncation maximumBytes is invalid');
}
if (typeof value.quotaReached !== 'boolean') {
throw new TypeError('Local Artifact truncation quotaReached is invalid');
}
if (!Number.isSafeInteger(value.observedAtMs) || value.observedAtMs < 0) {
throw new TypeError('Local Artifact truncation observedAtMs is invalid');
}
return Object.freeze({
schemaVersion: LOCAL_ARTIFACT_TRUNCATION_SCHEMA_VERSION,
runId: value.runId,
attemptId: value.attemptId,
logArtifactId: value.logArtifactId,
maximumBytes: value.maximumBytes,
quotaReached: value.quotaReached,
observedAtMs: value.observedAtMs,
});
}
export function encodeLocalArtifactTruncationFact(
value: LocalArtifactTruncationFact,
): string {
const fact = normalizeLocalArtifactTruncationFact(value);
const encoded = JSON.stringify(fact);
if (Buffer.byteLength(encoded) > MAX_LOCAL_ARTIFACT_TRUNCATION_FACT_BYTES) {
throw new TypeError('Local Artifact truncation fact is too large');
}
return encoded;
}
export function decodeLocalArtifactTruncationFact(
value: Buffer | string,
): Readonly<LocalArtifactTruncationFact> {
const encoded = Buffer.isBuffer(value) ? value.toString('utf8') : value;
if (
Buffer.byteLength(encoded) < 1 ||
Buffer.byteLength(encoded) > MAX_LOCAL_ARTIFACT_TRUNCATION_FACT_BYTES
) {
throw new TypeError('Local Artifact truncation fact size is invalid');
}
let parsed: unknown;
try {
parsed = JSON.parse(encoded);
} catch {
throw new TypeError('Local Artifact truncation fact JSON is invalid');
}
const fact = normalizeLocalArtifactTruncationFact(
parsed as LocalArtifactTruncationFact,
);
if (encodeLocalArtifactTruncationFact(fact) !== encoded) {
throw new TypeError('Local Artifact truncation fact is not canonical');
}
return fact;
}
back/runtime/domain/localExecutionArtifact.ts
+28
View File
@@ -0,0 +1,28 @@
import { createHash } from 'crypto';
import { MAX_LOG_ARTIFACT_ID_LENGTH } from './runStateMachine';
import type { RunDispatchCandidate } from './runDispatchCandidate';
import { assertRunDispatchCandidate } from './runDispatchCandidate';
const LOCAL_ARTIFACT_ID_PATTERN = /^local-[0-9a-f]{30}$/;
export function localExecutionArtifactId(
candidate: Readonly<RunDispatchCandidate>,
): string {
assertRunDispatchCandidate(candidate);
return `local-${createHash('sha256')
.update(candidate.runId, 'utf8')
.update('\0', 'utf8')
.update(candidate.attemptId, 'utf8')
.digest('hex')
.slice(0, 30)}`;
}
export function assertLocalExecutionArtifactId(value: string): void {
if (
typeof value !== 'string' ||
value.length > MAX_LOG_ARTIFACT_ID_LENGTH ||
!LOCAL_ARTIFACT_ID_PATTERN.test(value)
) {
throw new TypeError('Local execution artifact id is invalid');
}
}
back/runtime/domain/localExecutionContextRecipe.ts
+174
View File
@@ -0,0 +1,174 @@
import { createHash } from 'crypto';
import { MAX_EXECUTION_ENVIRONMENT_ENTRIES } from './executionContext';
export const MAX_LOCAL_CONTEXT_REF_LENGTH = 512;
export const MAX_LOCAL_SECRET_REF_LENGTH = 512;
export type LocalExecutionEnvironmentBinding =
| {
name: string;
kind: 'public';
value: string;
}
| {
name: string;
kind: 'secret';
secretRef: string;
};
export interface LocalExecutionContextRecipe {
contextRef: string;
environment: readonly LocalExecutionEnvironmentBinding[];
}
export interface LocalExecutionContextRecipeRecord
extends LocalExecutionContextRecipe {
contentDigest: string;
createdAtMs: number;
}
function assertBoundedText(name: string, value: string, maximum: number): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > maximum ||
/[\u0000-\u001f\u007f]/.test(value)
) {
throw new TypeError(`${name} is invalid`);
}
}
function assertEnvironmentName(value: string): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > 255 ||
value.includes('=') ||
value.includes('\0')
) {
throw new TypeError('Local context environment name is invalid');
}
}
export function assertLocalExecutionContextRef(value: string): void {
assertBoundedText(
'Local execution contextRef',
value,
MAX_LOCAL_CONTEXT_REF_LENGTH,
);
}
export function normalizeLocalExecutionContextRecipe(
recipe: LocalExecutionContextRecipe,
): LocalExecutionContextRecipe {
if (!recipe || typeof recipe !== 'object' || Array.isArray(recipe)) {
throw new TypeError('Local execution context recipe must be an object');
}
assertLocalExecutionContextRef(recipe.contextRef);
if (!Array.isArray(recipe.environment)) {
throw new TypeError('Local context environment bindings must be an array');
}
if (recipe.environment.length > MAX_EXECUTION_ENVIRONMENT_ENTRIES) {
throw new RangeError('Local context has too many environment bindings');
}
const names = new Set<string>();
const environment = recipe.environment
.map((binding) => {
if (!binding || typeof binding !== 'object' || Array.isArray(binding)) {
throw new TypeError('Local context environment binding is invalid');
}
assertEnvironmentName(binding.name);
if (names.has(binding.name)) {
throw new TypeError('Local context environment binding is duplicated');
}
names.add(binding.name);
if (binding.kind === 'public') {
if (typeof binding.value !== 'string' || binding.value.includes('\0')) {
throw new TypeError('Local public environment value is invalid');
}
return Object.freeze({
name: binding.name,
kind: 'public' as const,
value: binding.value,
});
}
if (binding.kind === 'secret') {
assertBoundedText(
'Local environment secretRef',
binding.secretRef,
MAX_LOCAL_SECRET_REF_LENGTH,
);
return Object.freeze({
name: binding.name,
kind: 'secret' as const,
secretRef: binding.secretRef,
});
}
throw new TypeError('Local context environment binding kind is invalid');
})
.sort((left, right) =>
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
);
return Object.freeze({
contextRef: recipe.contextRef,
environment: Object.freeze(environment),
});
}
function environmentDigest(
environment: readonly LocalExecutionEnvironmentBinding[],
): string {
return createHash('sha256')
.update(JSON.stringify(environment), 'utf8')
.digest('hex');
}
export function createLocalExecutionContextRecipe(
environment: readonly LocalExecutionEnvironmentBinding[],
): LocalExecutionContextRecipe {
const placeholder = normalizeLocalExecutionContextRecipe({
contextRef: 'localctx:pending',
environment,
});
return normalizeLocalExecutionContextRecipe({
contextRef: `localctx:sha256:${environmentDigest(placeholder.environment)}`,
environment: placeholder.environment,
});
}
export function localExecutionContextRecipeDigest(
recipe: LocalExecutionContextRecipe,
): string {
const normalized = normalizeLocalExecutionContextRecipe(recipe);
return createHash('sha256')
.update(JSON.stringify(normalized), 'utf8')
.digest('hex');
}
export function assertContentAddressedLocalExecutionContextRecipe(
recipe: LocalExecutionContextRecipe,
): void {
const normalized = normalizeLocalExecutionContextRecipe(recipe);
const expected = createLocalExecutionContextRecipe(normalized.environment);
if (normalized.contextRef !== expected.contextRef) {
throw new TypeError(
'Local execution context recipe is not content-addressed',
);
}
}
export function createLocalExecutionContextRecipeRecord(
recipe: LocalExecutionContextRecipe,
createdAtMs: number,
): LocalExecutionContextRecipeRecord {
if (!Number.isSafeInteger(createdAtMs) || createdAtMs < 0) {
throw new RangeError('createdAtMs must be a non-negative safe integer');
}
const normalized = normalizeLocalExecutionContextRecipe(recipe);
assertContentAddressedLocalExecutionContextRecipe(normalized);
return Object.freeze({
...normalized,
contentDigest: localExecutionContextRecipeDigest(normalized),
createdAtMs,
});
}
back/runtime/domain/localSecret.ts
+270
View File
@@ -0,0 +1,270 @@
import { MAX_EXECUTION_ENVIRONMENT_VALUE_BYTES } from './executionContext';
export const LOCAL_SECRET_ALGORITHM = 'aes-256-gcm';
export const MAX_LOCAL_SECRET_NAME_LENGTH = 128;
export const MAX_LOCAL_SECRET_VERSION = 2_147_483_647;
export const MAX_LOCAL_SECRET_REF_LENGTH = 512;
export const MAX_LOCAL_SECRET_MUTATION_ID_LENGTH = 64;
export const MAX_LOCAL_SECRET_KEY_ID_LENGTH = 128;
const SECRET_REF_PREFIX = 'qlsecret:v1:';
const BASE64URL_PATTERN = /^[A-Za-z0-9_-]+$/;
const KEY_ID_PATTERN = /^[A-Za-z0-9._-]+$/;
export interface LocalSecretReference {
projectId: string;
name: string;
version?: number;
}
export interface LocalSecretEnvelope {
projectId: string;
name: string;
version: number;
mutationId: string;
keyId: string;
algorithm: typeof LOCAL_SECRET_ALGORITHM;
nonce: string;
ciphertext: string;
authTag: string;
createdAtMs: number;
}
export class InvalidLocalSecretError extends TypeError {
constructor(message: string) {
super(`Local Secret value is invalid: ${message}`);
this.name = 'InvalidLocalSecretError';
}
}
export class LocalSecretUnavailableError extends Error {
readonly code = 'LOCAL_SECRET_UNAVAILABLE';
constructor() {
super('Local Secret is unavailable');
this.name = 'LocalSecretUnavailableError';
}
}
export class LocalSecretVersionConflictError extends Error {
readonly code = 'LOCAL_SECRET_VERSION_CONFLICT';
constructor() {
super('Local Secret current version changed');
this.name = 'LocalSecretVersionConflictError';
}
}
export class LocalSecretMutationConflictError extends Error {
readonly code = 'LOCAL_SECRET_MUTATION_CONFLICT';
constructor() {
super('Local Secret mutation does not match its previous request');
this.name = 'LocalSecretMutationConflictError';
}
}
function assertIdentifier(name: string, value: string, maximum: number): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > maximum ||
/[\u0000-\u001f\u007f]/.test(value)
) {
throw new InvalidLocalSecretError(`${name} is invalid`);
}
}
export function assertLocalSecretProjectId(value: string): void {
assertIdentifier('projectId', value, 128);
}
export function assertLocalSecretName(value: string): void {
assertIdentifier('name', value, MAX_LOCAL_SECRET_NAME_LENGTH);
}
export function assertLocalSecretVersion(value: number): void {
if (
!Number.isSafeInteger(value) ||
value < 1 ||
value > MAX_LOCAL_SECRET_VERSION
) {
throw new InvalidLocalSecretError('version is invalid');
}
}
export function assertLocalSecretMutationId(value: string): void {
assertIdentifier('mutationId', value, MAX_LOCAL_SECRET_MUTATION_ID_LENGTH);
}
export function assertLocalSecretKeyId(value: string): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > MAX_LOCAL_SECRET_KEY_ID_LENGTH ||
!KEY_ID_PATTERN.test(value)
) {
throw new InvalidLocalSecretError('keyId is invalid');
}
}
export function assertLocalSecretPlaintext(value: string): void {
if (
typeof value !== 'string' ||
value.includes('\0') ||
Buffer.byteLength(value, 'utf8') > MAX_EXECUTION_ENVIRONMENT_VALUE_BYTES
) {
throw new InvalidLocalSecretError('plaintext is invalid');
}
}
function decodeBase64Url(name: string, value: string, bytes?: number): Buffer {
if (
typeof value !== 'string' ||
(value.length > 0 && !BASE64URL_PATTERN.test(value))
) {
throw new InvalidLocalSecretError(`${name} is invalid`);
}
const decoded = Buffer.from(value, 'base64url');
if (
decoded.toString('base64url') !== value ||
(bytes && decoded.length !== bytes)
) {
throw new InvalidLocalSecretError(`${name} is invalid`);
}
return decoded;
}
export function localSecretBinary(
name: 'nonce' | 'ciphertext' | 'authTag',
value: string,
): Buffer {
const decoded = decodeBase64Url(
name,
value,
name === 'nonce' ? 12 : name === 'authTag' ? 16 : undefined,
);
if (
name === 'ciphertext' &&
decoded.length > MAX_EXECUTION_ENVIRONMENT_VALUE_BYTES
) {
throw new InvalidLocalSecretError('ciphertext is too large');
}
return decoded;
}
export function createLocalSecretRef(reference: LocalSecretReference): string {
assertLocalSecretProjectId(reference.projectId);
assertLocalSecretName(reference.name);
if (reference.version !== undefined) {
assertLocalSecretVersion(reference.version);
}
const payload = JSON.stringify({
projectId: reference.projectId,
name: reference.name,
...(reference.version === undefined ? {} : { version: reference.version }),
});
const value =
SECRET_REF_PREFIX + Buffer.from(payload, 'utf8').toString('base64url');
if (value.length > MAX_LOCAL_SECRET_REF_LENGTH) {
throw new InvalidLocalSecretError('reference is too large');
}
return value;
}
export function parseLocalSecretRef(value: string): LocalSecretReference {
if (
typeof value !== 'string' ||
value.length > MAX_LOCAL_SECRET_REF_LENGTH ||
!value.startsWith(SECRET_REF_PREFIX)
) {
throw new InvalidLocalSecretError('reference is invalid');
}
const encoded = value.slice(SECRET_REF_PREFIX.length);
let parsed: unknown;
try {
parsed = JSON.parse(decodeBase64Url('reference', encoded).toString('utf8'));
} catch (error) {
if (error instanceof InvalidLocalSecretError) throw error;
throw new InvalidLocalSecretError('reference is invalid');
}
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
throw new InvalidLocalSecretError('reference is invalid');
}
const record = parsed as Record<string, unknown>;
const keys = Object.keys(record).sort();
const expectedKeys =
record.version === undefined
? ['name', 'projectId']
: ['name', 'projectId', 'version'];
if (
keys.length !== expectedKeys.length ||
keys.some((key, index) => key !== expectedKeys[index])
) {
throw new InvalidLocalSecretError('reference is invalid');
}
const reference: LocalSecretReference = {
projectId: record.projectId as string,
name: record.name as string,
...(record.version === undefined
? {}
: { version: record.version as number }),
};
if (createLocalSecretRef(reference) !== value) {
throw new InvalidLocalSecretError('reference is not canonical');
}
return Object.freeze(reference);
}
export function normalizeLocalSecretEnvelope(
envelope: LocalSecretEnvelope,
): LocalSecretEnvelope {
if (!envelope || typeof envelope !== 'object' || Array.isArray(envelope)) {
throw new InvalidLocalSecretError('envelope must be an object');
}
assertLocalSecretProjectId(envelope.projectId);
assertLocalSecretName(envelope.name);
assertLocalSecretVersion(envelope.version);
assertLocalSecretMutationId(envelope.mutationId);
assertLocalSecretKeyId(envelope.keyId);
if (envelope.algorithm !== LOCAL_SECRET_ALGORITHM) {
throw new InvalidLocalSecretError('algorithm is invalid');
}
localSecretBinary('nonce', envelope.nonce);
localSecretBinary('ciphertext', envelope.ciphertext);
localSecretBinary('authTag', envelope.authTag);
if (!Number.isSafeInteger(envelope.createdAtMs) || envelope.createdAtMs < 0) {
throw new InvalidLocalSecretError('createdAtMs is invalid');
}
return Object.freeze({
projectId: envelope.projectId,
name: envelope.name,
version: envelope.version,
mutationId: envelope.mutationId,
keyId: envelope.keyId,
algorithm: LOCAL_SECRET_ALGORITHM,
nonce: envelope.nonce,
ciphertext: envelope.ciphertext,
authTag: envelope.authTag,
createdAtMs: envelope.createdAtMs,
});
}
export function localSecretEnvelopeAad(
envelope: Pick<
LocalSecretEnvelope,
'projectId' | 'name' | 'version' | 'mutationId' | 'keyId' | 'algorithm'
>,
): Buffer {
return Buffer.from(
JSON.stringify({
projectId: envelope.projectId,
name: envelope.name,
version: envelope.version,
mutationId: envelope.mutationId,
keyId: envelope.keyId,
algorithm: envelope.algorithm,
}),
'utf8',
);
}
back/runtime/domain/projectOwnerBootstrap.ts
+231
View File
@@ -0,0 +1,231 @@
import { createHash } from 'crypto';
import {
InvalidProjectPolicyValueError,
assertProjectPolicyProjectId,
normalizePolicySubject,
type PolicySubject,
} from './projectPolicy';
export const OWNER_BOOTSTRAP_TOKEN_BYTES = 32;
export const OWNER_BOOTSTRAP_CHALLENGE_ID_BYTES = 16;
export const OWNER_BOOTSTRAP_DEFAULT_TTL_MS = 10 * 60 * 1000;
export const OWNER_BOOTSTRAP_MIN_TTL_MS = 60 * 1000;
export const OWNER_BOOTSTRAP_MAX_TTL_MS = 30 * 60 * 1000;
export const OWNER_BOOTSTRAP_MAX_VERSION = 2_147_483_647;
export const OWNER_BOOTSTRAP_SYSTEM_SUBJECT = Object.freeze({
type: 'system' as const,
id: 'owner-bootstrap',
});
const TOKEN_PATTERN = /^[A-Za-z0-9_-]{43}$/;
const CHALLENGE_ID_PATTERN = /^[A-Za-z0-9_-]{22}$/;
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
const DIGEST_DOMAIN = 'qinglong3-owner-bootstrap-v1\0';
export interface ProjectOwnerBootstrapChallengeRecord {
projectId: string;
version: number;
challengeId: string;
tokenDigest: string;
issuedAtMs: number;
expiresAtMs: number;
consumedAtMs?: number;
claimedSubject?: PolicySubject;
}
export class InvalidProjectOwnerBootstrapValueError extends TypeError {
constructor(message: string) {
super(`Project owner bootstrap value is invalid: ${message}`);
this.name = 'InvalidProjectOwnerBootstrapValueError';
}
}
export class ProjectOwnerBootstrapUnauthorizedError extends Error {
readonly code = 'PROJECT_OWNER_BOOTSTRAP_UNAUTHORIZED';
constructor() {
super('Project owner bootstrap caller is not authorized');
this.name = 'ProjectOwnerBootstrapUnauthorizedError';
}
}
export class ProjectOwnerBootstrapProjectNotFoundError extends Error {
readonly code = 'PROJECT_OWNER_BOOTSTRAP_PROJECT_NOT_FOUND';
constructor() {
super('Project does not exist');
this.name = 'ProjectOwnerBootstrapProjectNotFoundError';
}
}
export class ProjectOwnerBootstrapProjectInactiveError extends Error {
readonly code = 'PROJECT_OWNER_BOOTSTRAP_PROJECT_INACTIVE';
constructor() {
super('Project is not active');
this.name = 'ProjectOwnerBootstrapProjectInactiveError';
}
}
export class ProjectOwnerBootstrapProjectNotPristineError extends Error {
readonly code = 'PROJECT_OWNER_BOOTSTRAP_PROJECT_NOT_PRISTINE';
constructor() {
super('Project owner bootstrap is no longer available');
this.name = 'ProjectOwnerBootstrapProjectNotPristineError';
}
}
export class ProjectOwnerBootstrapChallengeActiveError extends Error {
readonly code = 'PROJECT_OWNER_BOOTSTRAP_CHALLENGE_ACTIVE';
constructor() {
super('A Project owner bootstrap challenge is already active');
this.name = 'ProjectOwnerBootstrapChallengeActiveError';
}
}
export class ProjectOwnerBootstrapClaimRejectedError extends Error {
readonly code = 'PROJECT_OWNER_BOOTSTRAP_CLAIM_REJECTED';
constructor() {
super('Project owner bootstrap claim was rejected');
this.name = 'ProjectOwnerBootstrapClaimRejectedError';
}
}
export class ProjectOwnerBootstrapUnavailableError extends Error {
readonly code = 'PROJECT_OWNER_BOOTSTRAP_UNAVAILABLE';
constructor() {
super('Project owner bootstrap is unavailable');
this.name = 'ProjectOwnerBootstrapUnavailableError';
}
}
function assertTimestamp(name: string, value: number): void {
if (!Number.isSafeInteger(value) || value < 0) {
throw new InvalidProjectOwnerBootstrapValueError(`${name} is invalid`);
}
}
export function assertProjectOwnerBootstrapToken(value: string): void {
if (typeof value !== 'string' || !TOKEN_PATTERN.test(value)) {
throw new InvalidProjectOwnerBootstrapValueError('token is invalid');
}
}
export function assertProjectOwnerBootstrapChallengeId(value: string): void {
if (typeof value !== 'string' || !CHALLENGE_ID_PATTERN.test(value)) {
throw new InvalidProjectOwnerBootstrapValueError('challengeId is invalid');
}
}
export function assertProjectOwnerBootstrapTokenDigest(value: string): void {
if (typeof value !== 'string' || !DIGEST_PATTERN.test(value)) {
throw new InvalidProjectOwnerBootstrapValueError('tokenDigest is invalid');
}
}
export function assertProjectOwnerBootstrapTtl(value: number): void {
if (
!Number.isSafeInteger(value) ||
value < OWNER_BOOTSTRAP_MIN_TTL_MS ||
value > OWNER_BOOTSTRAP_MAX_TTL_MS
) {
throw new InvalidProjectOwnerBootstrapValueError('ttlMs is invalid');
}
}
export function digestProjectOwnerBootstrapToken(
projectId: string,
challengeId: string,
token: string,
): string {
assertProjectPolicyProjectId(projectId);
assertProjectOwnerBootstrapChallengeId(challengeId);
assertProjectOwnerBootstrapToken(token);
return createHash('sha256')
.update(DIGEST_DOMAIN, 'utf8')
.update(projectId, 'utf8')
.update('\0', 'utf8')
.update(challengeId, 'utf8')
.update('\0', 'utf8')
.update(token, 'utf8')
.digest('hex');
}
export function normalizeProjectOwnerBootstrapChallengeRecord(
value: ProjectOwnerBootstrapChallengeRecord,
): Readonly<ProjectOwnerBootstrapChallengeRecord> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidProjectOwnerBootstrapValueError(
'challenge must be an object',
);
}
const consumed = value.consumedAtMs !== undefined;
const expected = consumed
? [
'challengeId',
'claimedSubject',
'consumedAtMs',
'expiresAtMs',
'issuedAtMs',
'projectId',
'tokenDigest',
'version',
]
: [
'challengeId',
'expiresAtMs',
'issuedAtMs',
'projectId',
'tokenDigest',
'version',
];
const keys = Object.keys(value).sort();
if (
keys.length !== expected.length ||
keys.some((key, index) => key !== expected[index])
) {
throw new InvalidProjectOwnerBootstrapValueError(
'challenge shape is invalid',
);
}
try {
assertProjectPolicyProjectId(value.projectId);
} catch (error) {
if (error instanceof InvalidProjectPolicyValueError) {
throw new InvalidProjectOwnerBootstrapValueError('projectId is invalid');
}
throw error;
}
if (
!Number.isSafeInteger(value.version) ||
value.version < 1 ||
value.version > OWNER_BOOTSTRAP_MAX_VERSION
) {
throw new InvalidProjectOwnerBootstrapValueError('version is invalid');
}
assertProjectOwnerBootstrapChallengeId(value.challengeId);
assertProjectOwnerBootstrapTokenDigest(value.tokenDigest);
assertTimestamp('issuedAtMs', value.issuedAtMs);
assertTimestamp('expiresAtMs', value.expiresAtMs);
if (value.expiresAtMs <= value.issuedAtMs) {
throw new InvalidProjectOwnerBootstrapValueError('lifetime is invalid');
}
if (!consumed) return Object.freeze({ ...value });
assertTimestamp('consumedAtMs', value.consumedAtMs!);
if (value.consumedAtMs! < value.issuedAtMs) {
throw new InvalidProjectOwnerBootstrapValueError(
'consumption time is invalid',
);
}
if (!value.claimedSubject) {
throw new InvalidProjectOwnerBootstrapValueError(
'claimedSubject is required',
);
}
const claimedSubject = normalizePolicySubject(value.claimedSubject);
return Object.freeze({ ...value, claimedSubject });
}
back/runtime/domain/projectPolicy.ts
+371
View File
@@ -0,0 +1,371 @@
export const POLICY_SUBJECT_TYPES = [
'user',
'api_app',
'mcp_client',
'agent',
'system',
'worker',
] as const;
export const PROJECT_STATUSES = ['active', 'archived'] as const;
export const PROJECT_ROLES = ['owner', 'admin', 'operator', 'viewer'] as const;
export const PROJECT_ROLE_BINDING_STATES = ['active', 'revoked'] as const;
export const STATIC_PROJECT_PERMISSIONS = [
'project.read',
'project.manage',
'task.read',
'task.create',
'task.update',
'task.delete',
'run.read',
'run.start',
'run.stop',
'run.retry',
'artifact.read',
'secret.use',
'secret.manage',
'worker.manage',
'policy.manage',
'approval.decide',
'approval.recover',
] as const;
export type PolicySubjectType = (typeof POLICY_SUBJECT_TYPES)[number];
export type ProjectStatus = (typeof PROJECT_STATUSES)[number];
export type ProjectRole = (typeof PROJECT_ROLES)[number];
export type ProjectRoleBindingState =
(typeof PROJECT_ROLE_BINDING_STATES)[number];
export type StaticProjectPermission =
(typeof STATIC_PROJECT_PERMISSIONS)[number];
export type ProjectPermission = StaticProjectPermission | `tool.call:${string}`;
export interface PolicySubject {
type: PolicySubjectType;
id: string;
}
export interface ProjectRecord {
id: string;
name: string;
slug: string;
status: ProjectStatus;
version: number;
createdAtMs: number;
updatedAtMs: number;
}
export interface ProjectRoleBindingRecord {
projectId: string;
subject: PolicySubject;
version: number;
state: ProjectRoleBindingState;
role?: ProjectRole;
mutationId: string;
changedBy: PolicySubject;
createdAtMs: number;
}
export interface ProjectPolicySnapshot {
project: Readonly<ProjectRecord>;
binding?: Readonly<ProjectRoleBindingRecord>;
}
export type ProjectPolicyEffect = 'allow' | 'deny' | 'require_approval';
export interface ProjectPolicyDecision {
effect: ProjectPolicyEffect;
reasons: readonly string[];
}
export interface ProjectPolicyFence {
projectVersion: number;
bindingVersion: number | null;
}
export interface ProjectPolicyDecisionWithFence {
decision: Readonly<ProjectPolicyDecision>;
fence: Readonly<ProjectPolicyFence> | null;
}
export interface ProjectPolicyRequest {
subject: PolicySubject;
projectId: string;
permission: ProjectPermission;
}
export const MAX_POLICY_SUBJECT_ID_LENGTH = 255;
export const MAX_PROJECT_ID_LENGTH = 128;
export const MAX_PROJECT_NAME_LENGTH = 255;
export const MAX_PROJECT_SLUG_LENGTH = 128;
export const MAX_PROJECT_ROLE_BINDING_VERSION = 2_147_483_647;
export const MAX_PROJECT_POLICY_MUTATION_ID_LENGTH = 64;
export const MAX_PROJECT_PERMISSION_LENGTH = 255;
const IDENTIFIER_CONTROL_PATTERN = /[\u0000-\u001f\u007f]/;
const PROJECT_SLUG_PATTERN = /^[a-z0-9](?:[a-z0-9-]{0,126}[a-z0-9])?$/;
const MUTATION_ID_PATTERN = /^[A-Za-z0-9._:-]+$/;
const TOOL_PERMISSION_PATTERN =
/^tool\.call:[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
export class InvalidProjectPolicyValueError extends TypeError {
constructor(message: string) {
super(`Project policy value is invalid: ${message}`);
this.name = 'InvalidProjectPolicyValueError';
}
}
export class ProjectRoleBindingVersionConflictError extends Error {
readonly code = 'PROJECT_ROLE_BINDING_VERSION_CONFLICT';
constructor() {
super('Project role binding current version changed');
this.name = 'ProjectRoleBindingVersionConflictError';
}
}
export class ProjectRoleBindingMutationConflictError extends Error {
readonly code = 'PROJECT_ROLE_BINDING_MUTATION_CONFLICT';
constructor() {
super('Project role binding mutation does not match its previous request');
this.name = 'ProjectRoleBindingMutationConflictError';
}
}
export class ProjectPolicyUnavailableError extends Error {
readonly code = 'PROJECT_POLICY_UNAVAILABLE';
constructor() {
super('Project policy is unavailable');
this.name = 'ProjectPolicyUnavailableError';
}
}
export class ProjectPolicyProjectNotFoundError extends Error {
readonly code = 'PROJECT_NOT_FOUND';
constructor() {
super('Project does not exist');
this.name = 'ProjectPolicyProjectNotFoundError';
}
}
function assertBoundedIdentifier(
name: string,
value: string,
maximum: number,
): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > maximum ||
IDENTIFIER_CONTROL_PATTERN.test(value)
) {
throw new InvalidProjectPolicyValueError(`${name} is invalid`);
}
}
function assertTimestamp(name: string, value: number): void {
if (!Number.isSafeInteger(value) || value < 0) {
throw new InvalidProjectPolicyValueError(`${name} is invalid`);
}
}
function assertExactKeys(
name: string,
value: object,
expected: readonly string[],
): void {
const keys = Object.keys(value).sort();
const canonical = [...expected].sort();
if (
keys.length !== canonical.length ||
keys.some((key, index) => key !== canonical[index])
) {
throw new InvalidProjectPolicyValueError(`${name} shape is invalid`);
}
}
export function assertProjectPolicyProjectId(value: string): void {
assertBoundedIdentifier('projectId', value, MAX_PROJECT_ID_LENGTH);
}
export function normalizePolicySubject(
value: PolicySubject,
): Readonly<PolicySubject> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidProjectPolicyValueError('subject must be an object');
}
assertExactKeys('subject', value, ['type', 'id']);
if (!POLICY_SUBJECT_TYPES.includes(value.type)) {
throw new InvalidProjectPolicyValueError('subject type is invalid');
}
assertBoundedIdentifier('subject id', value.id, MAX_POLICY_SUBJECT_ID_LENGTH);
return Object.freeze({ type: value.type, id: value.id });
}
export function normalizeProjectPermission(value: string): ProjectPermission {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > MAX_PROJECT_PERMISSION_LENGTH
) {
throw new InvalidProjectPolicyValueError('permission is invalid');
}
if (
STATIC_PROJECT_PERMISSIONS.includes(value as StaticProjectPermission) ||
TOOL_PERMISSION_PATTERN.test(value)
) {
return value as ProjectPermission;
}
throw new InvalidProjectPolicyValueError('permission is invalid');
}
export function normalizeProjectRecord(
value: ProjectRecord,
): Readonly<ProjectRecord> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidProjectPolicyValueError('Project must be an object');
}
assertExactKeys('Project', value, [
'id',
'name',
'slug',
'status',
'version',
'createdAtMs',
'updatedAtMs',
]);
assertProjectPolicyProjectId(value.id);
assertBoundedIdentifier('Project name', value.name, MAX_PROJECT_NAME_LENGTH);
if (
typeof value.slug !== 'string' ||
value.slug.length > MAX_PROJECT_SLUG_LENGTH ||
!PROJECT_SLUG_PATTERN.test(value.slug)
) {
throw new InvalidProjectPolicyValueError('Project slug is invalid');
}
if (!PROJECT_STATUSES.includes(value.status)) {
throw new InvalidProjectPolicyValueError('Project status is invalid');
}
if (
!Number.isSafeInteger(value.version) ||
value.version < 1 ||
value.version > MAX_PROJECT_ROLE_BINDING_VERSION
) {
throw new InvalidProjectPolicyValueError('Project version is invalid');
}
assertTimestamp('Project createdAtMs', value.createdAtMs);
assertTimestamp('Project updatedAtMs', value.updatedAtMs);
if (value.updatedAtMs < value.createdAtMs) {
throw new InvalidProjectPolicyValueError('Project timestamps are invalid');
}
return Object.freeze({ ...value });
}
export function normalizeProjectRoleBindingRecord(
value: ProjectRoleBindingRecord,
): Readonly<ProjectRoleBindingRecord> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidProjectPolicyValueError(
'Project role binding must be an object',
);
}
assertExactKeys(
'Project role binding',
value,
value.state === 'active'
? [
'projectId',
'subject',
'version',
'state',
'role',
'mutationId',
'changedBy',
'createdAtMs',
]
: [
'projectId',
'subject',
'version',
'state',
'mutationId',
'changedBy',
'createdAtMs',
],
);
assertProjectPolicyProjectId(value.projectId);
const subject = normalizePolicySubject(value.subject);
const changedBy = normalizePolicySubject(value.changedBy);
if (
!Number.isSafeInteger(value.version) ||
value.version < 1 ||
value.version > MAX_PROJECT_ROLE_BINDING_VERSION
) {
throw new InvalidProjectPolicyValueError(
'Project role binding version is invalid',
);
}
if (!PROJECT_ROLE_BINDING_STATES.includes(value.state)) {
throw new InvalidProjectPolicyValueError(
'Project role binding state is invalid',
);
}
if (
(value.state === 'active' &&
(!value.role || !PROJECT_ROLES.includes(value.role))) ||
(value.state === 'revoked' && value.role !== undefined)
) {
throw new InvalidProjectPolicyValueError(
'Project role binding role is invalid',
);
}
if (
typeof value.mutationId !== 'string' ||
value.mutationId.length < 1 ||
value.mutationId.length > MAX_PROJECT_POLICY_MUTATION_ID_LENGTH ||
!MUTATION_ID_PATTERN.test(value.mutationId)
) {
throw new InvalidProjectPolicyValueError(
'Project role binding mutationId is invalid',
);
}
assertTimestamp('Project role binding createdAtMs', value.createdAtMs);
return Object.freeze({
projectId: value.projectId,
subject,
version: value.version,
state: value.state,
...(value.role ? { role: value.role } : {}),
mutationId: value.mutationId,
changedBy,
createdAtMs: value.createdAtMs,
});
}
export function normalizeProjectPolicySnapshot(
value: ProjectPolicySnapshot,
): Readonly<ProjectPolicySnapshot> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new InvalidProjectPolicyValueError(
'Project policy snapshot must be an object',
);
}
assertExactKeys(
'Project policy snapshot',
value,
value.binding ? ['project', 'binding'] : ['project'],
);
const project = normalizeProjectRecord(value.project);
const binding = value.binding
? normalizeProjectRoleBindingRecord(value.binding)
: undefined;
if (binding && binding.projectId !== project.id) {
throw new InvalidProjectPolicyValueError(
'Project policy snapshot binding is misplaced',
);
}
return Object.freeze({ project, ...(binding ? { binding } : {}) });
}
back/runtime/domain/repositoryErrors.ts
+87
View File
@@ -0,0 +1,87 @@
export class RunRepositoryError extends Error {
constructor(
message: string,
public readonly code: string,
public readonly retryable = false,
public readonly cause?: unknown,
) {
super(message);
this.name = new.target.name;
}
}
export class DuplicateIdempotencyKeyError extends RunRepositoryError {
constructor(
public readonly projectId: string,
public readonly idempotencyKey: string,
) {
super(
'A Run with the same project idempotency key already exists',
'DUPLICATE_RUN_IDEMPOTENCY_KEY',
);
}
}
export class DuplicateRunAttemptError extends RunRepositoryError {
constructor(public readonly runId: string, public readonly attempt: number) {
super(
'A RunAttempt with the same run and attempt number already exists',
'DUPLICATE_RUN_ATTEMPT',
);
}
}
export class DuplicateRunEventError extends RunRepositoryError {
constructor(
public readonly runId: string,
public readonly dedupeKey?: string,
) {
super(
'A RunEvent with the same sequence or dedupe key already exists',
'DUPLICATE_RUN_EVENT',
);
}
}
export class RunRepositoryConstraintError extends RunRepositoryError {
constructor(
message = 'Run repository constraint violation',
cause?: unknown,
) {
super(message, 'RUN_REPOSITORY_CONSTRAINT', false, cause);
}
}
export class RunRepositoryBusyError extends RunRepositoryError {
constructor(cause?: unknown) {
super(
'Run repository is temporarily busy',
'RUN_REPOSITORY_BUSY',
true,
cause,
);
}
}
export class RunRepositoryOperationError extends RunRepositoryError {
constructor(cause?: unknown) {
super(
'Run repository operation failed',
'RUN_REPOSITORY_OPERATION_FAILED',
false,
cause,
);
}
}
export class RunEventPayloadTooLargeError extends RunRepositoryError {
constructor(
public readonly actualBytes: number,
public readonly maxBytes: number,
) {
super(
'RunEvent payload exceeds the configured size limit',
'RUN_EVENT_PAYLOAD_TOO_LARGE',
);
}
}
back/runtime/domain/run.ts
+144
View File
@@ -0,0 +1,144 @@
export const RUN_STATUSES = [
'created',
'queued',
'dispatching',
'running',
'waiting_approval',
'retry_wait',
'lost',
'succeeded',
'failed',
'cancelled',
'timed_out',
] as const;
export type RunStatus = (typeof RUN_STATUSES)[number];
export const RUN_ATTEMPT_STATUSES = [
'claimed',
'starting',
'running',
'succeeded',
'failed',
'cancelled',
'timed_out',
'lost',
] as const;
export type RunAttemptStatus = (typeof RUN_ATTEMPT_STATUSES)[number];
export const EXECUTION_ORIGINS = [
'manual',
'scheduled_system',
'scheduled_node',
'once',
'boot',
'grpc',
'subscription',
'system',
'script',
'legacy_import',
] as const;
export type ExecutionOrigin = (typeof EXECUTION_ORIGINS)[number];
export type ExecutionOwner = 'legacy' | 'runtime';
export const RUN_CANCELLATION_REASONS = [
'user',
'policy',
'shutdown',
'reconcile',
'timeout',
] as const;
export type RunCancellationReason = (typeof RUN_CANCELLATION_REASONS)[number];
export const RUN_EVENT_ACTOR_TYPES = [
'user',
'api_app',
'trigger',
'agent',
'mcp_client',
'worker',
'executor',
'system',
'legacy_shell',
'scheduler',
'reconciler',
'compatibility',
] as const;
export type RunEventActorType = (typeof RUN_EVENT_ACTOR_TYPES)[number];
export interface RunRecord {
id: string;
projectId: string;
taskId: string;
taskRevision: string;
taskName?: string;
taskSnapshotRef?: string;
legacyCronId?: number;
parentRunId?: string;
retryOfRunId?: string;
triggerId?: string;
triggerType: string;
executionOrigin: ExecutionOrigin;
executionOwner: ExecutionOwner;
triggeredBy?: string;
requestId?: string;
scheduledForMs?: number;
status: RunStatus;
version: number;
eventSequence: number;
priority: number;
idempotencyKey?: string;
inputRef?: string;
outputRef?: string;
createdAtMs: number;
queuedAtMs?: number;
startedAtMs?: number;
finishedAtMs?: number;
cancelRequestedAtMs?: number;
cancelReason?: RunCancellationReason;
errorCode?: string;
errorSummary?: string;
}
export interface RunAttemptRecord {
id: string;
runId: string;
stepRunId?: string;
attempt: number;
status: RunAttemptStatus;
executorType: string;
workerId?: string;
executorHandle?: string;
pid?: number;
logArtifactId?: string;
leaseToken?: string;
leaseExpiresAtMs?: number;
deadlineAtMs?: number;
callbackTokenHash?: string;
callbackSequence: number;
createdAtMs: number;
startedAtMs?: number;
finishedAtMs?: number;
exitCode?: number;
errorCode?: string;
errorSummary?: string;
}
export interface RunEventRecord {
id: string;
runId: string;
sequence: number;
type: string;
dedupeKey?: string;
actorType: RunEventActorType;
actorId?: string;
attemptId?: string;
stepRunId?: string;
payload: Readonly<Record<string, unknown>>;
createdAtMs: number;
}
back/runtime/domain/runDispatchCandidate.ts
+75
View File
@@ -0,0 +1,75 @@
import {
assertRunDispatchId,
assertRunDispatchLeaseVersion,
} from './runDispatchLease';
export const MAX_RUN_DISPATCH_CANDIDATE_PAGE_SIZE = 64;
export interface RunDispatchCandidateCursor {
priority: number;
queuedAtMs: number;
attemptCreatedAtMs: number;
attemptId: string;
}
export interface RunDispatchCandidate extends RunDispatchCandidateCursor {
runId: string;
projectId: string;
taskId: string;
taskRevision: string;
executorType: string;
}
function assertPriority(value: number): void {
if (!Number.isSafeInteger(value)) {
throw new TypeError(
'Run dispatch candidate priority must be a safe integer',
);
}
}
function assertExecutorType(value: string): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > 64 ||
/[\u0000-\u001f\u007f]/.test(value)
) {
throw new TypeError('Run dispatch candidate executorType is invalid');
}
}
export function assertRunDispatchCandidateCursor(
cursor: RunDispatchCandidateCursor,
): void {
assertPriority(cursor.priority);
assertRunDispatchLeaseVersion('queuedAtMs', cursor.queuedAtMs);
assertRunDispatchLeaseVersion(
'attemptCreatedAtMs',
cursor.attemptCreatedAtMs,
);
assertRunDispatchId('attemptId', cursor.attemptId);
}
export function assertRunDispatchCandidate(
candidate: RunDispatchCandidate,
): void {
assertRunDispatchCandidateCursor(candidate);
assertRunDispatchId('runId', candidate.runId);
assertRunDispatchId('projectId', candidate.projectId);
assertRunDispatchId('taskId', candidate.taskId);
assertRunDispatchId('taskRevision', candidate.taskRevision);
assertExecutorType(candidate.executorType);
}
export function assertRunDispatchCandidatePageSize(limit: number): void {
if (
!Number.isSafeInteger(limit) ||
limit < 1 ||
limit > MAX_RUN_DISPATCH_CANDIDATE_PAGE_SIZE
) {
throw new RangeError(
`Run dispatch candidate page size must be between 1 and ${MAX_RUN_DISPATCH_CANDIDATE_PAGE_SIZE}`,
);
}
}
back/runtime/domain/runDispatchLease.ts
+210
View File
@@ -0,0 +1,210 @@
import { assertWorkerId, assertWorkerSessionId } from './worker';
export const RUN_DISPATCH_LEASE_STATUSES = [
'leased',
'released',
'completed',
] as const;
export type RunDispatchLeaseStatus =
(typeof RUN_DISPATCH_LEASE_STATUSES)[number];
export const RUN_DISPATCH_RELEASE_REASONS = [
'declined',
'shutdown',
'start_failed',
'capacity_changed',
'lease_expired',
] as const;
export type RunDispatchReleaseReason =
(typeof RUN_DISPATCH_RELEASE_REASONS)[number];
export const MIN_RUN_DISPATCH_LEASE_DURATION_MS = 5_000;
export const MAX_RUN_DISPATCH_LEASE_DURATION_MS = 10 * 60_000;
export interface RunDispatchLeaseRecord {
attemptId: string;
runId: string;
status: RunDispatchLeaseStatus;
version: number;
leaseGeneration: number;
workerId: string;
workerSessionId: string;
workerGeneration: number;
leaseToken: string;
acquiredAtMs: number;
renewedAtMs: number;
expiresAtMs: number;
releasedAtMs?: number;
releaseReason?: RunDispatchReleaseReason;
completedAtMs?: number;
updatedAtMs: number;
}
export type RunDispatchLeaseFenceReason =
| 'missing'
| 'run_mismatch'
| 'not_leased'
| 'worker_mismatch'
| 'worker_session_mismatch'
| 'worker_generation_mismatch'
| 'lease_generation_mismatch'
| 'lease_token_mismatch'
| 'version_mismatch'
| 'lease_expired'
| 'worker_unavailable';
export class InvalidRunDispatchLeaseValueError extends TypeError {
constructor(message: string) {
super(`Run dispatch lease value is invalid: ${message}`);
this.name = 'InvalidRunDispatchLeaseValueError';
}
}
export class RunDispatchLeaseFenceRejectedError extends Error {
constructor(
readonly attemptId: string,
readonly reason: RunDispatchLeaseFenceReason,
) {
super(`Run dispatch lease for Attempt ${attemptId} was fenced: ${reason}`);
this.name = 'RunDispatchLeaseFenceRejectedError';
}
}
export function assertRunDispatchId(name: string, value: string): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > 128 ||
/[\u0000-\u001f\u007f]/.test(value)
) {
throw new InvalidRunDispatchLeaseValueError(`${name} is invalid`);
}
}
export function assertRunDispatchLeaseToken(value: string): void {
if (
typeof value !== 'string' ||
value.length < 32 ||
value.length > 128 ||
!/^[A-Za-z0-9_-]+$/.test(value)
) {
throw new InvalidRunDispatchLeaseValueError('leaseToken is invalid');
}
}
export function assertRunDispatchLeaseVersion(
name: string,
value: number,
positive = false,
): void {
if (!Number.isSafeInteger(value) || value < (positive ? 1 : 0)) {
throw new InvalidRunDispatchLeaseValueError(
`${name} must be a ${
positive ? 'positive' : 'non-negative'
} safe integer`,
);
}
}
export function assertRunDispatchWorkerFence(value: {
workerId: string;
workerSessionId: string;
workerGeneration: number;
}): void {
assertWorkerId(value.workerId);
assertWorkerSessionId(value.workerSessionId);
assertRunDispatchLeaseVersion(
'workerGeneration',
value.workerGeneration,
true,
);
}
export function assertRunDispatchLeaseDuration(value: number): void {
if (
!Number.isSafeInteger(value) ||
value < MIN_RUN_DISPATCH_LEASE_DURATION_MS ||
value > MAX_RUN_DISPATCH_LEASE_DURATION_MS
) {
throw new InvalidRunDispatchLeaseValueError(
`leaseDurationMs must be between ${MIN_RUN_DISPATCH_LEASE_DURATION_MS} and ${MAX_RUN_DISPATCH_LEASE_DURATION_MS}`,
);
}
}
export function assertRunDispatchLeaseRecord(
value: RunDispatchLeaseRecord,
): void {
assertRunDispatchId('attemptId', value.attemptId);
assertRunDispatchId('runId', value.runId);
if (!RUN_DISPATCH_LEASE_STATUSES.includes(value.status)) {
throw new InvalidRunDispatchLeaseValueError('status is invalid');
}
assertRunDispatchLeaseVersion('version', value.version);
assertRunDispatchLeaseVersion('leaseGeneration', value.leaseGeneration, true);
assertRunDispatchWorkerFence(value);
assertRunDispatchLeaseToken(value.leaseToken);
assertRunDispatchLeaseVersion('acquiredAtMs', value.acquiredAtMs);
assertRunDispatchLeaseVersion('renewedAtMs', value.renewedAtMs);
assertRunDispatchLeaseVersion('expiresAtMs', value.expiresAtMs);
assertRunDispatchLeaseVersion('updatedAtMs', value.updatedAtMs);
if (
value.renewedAtMs < value.acquiredAtMs ||
value.expiresAtMs <= value.renewedAtMs ||
value.updatedAtMs < value.acquiredAtMs
) {
throw new InvalidRunDispatchLeaseValueError('timestamps are inconsistent');
}
if (value.status === 'leased') {
if (
value.releasedAtMs !== undefined ||
value.releaseReason !== undefined ||
value.completedAtMs !== undefined
) {
throw new InvalidRunDispatchLeaseValueError(
'active lease has terminal metadata',
);
}
return;
}
if (value.status === 'released') {
if (
value.releasedAtMs === undefined ||
value.completedAtMs !== undefined ||
value.releaseReason === undefined ||
!RUN_DISPATCH_RELEASE_REASONS.includes(value.releaseReason)
) {
throw new InvalidRunDispatchLeaseValueError(
'released lease metadata is inconsistent',
);
}
assertRunDispatchLeaseVersion('releasedAtMs', value.releasedAtMs);
return;
}
if (
value.completedAtMs === undefined ||
value.releasedAtMs !== undefined ||
value.releaseReason !== undefined
) {
throw new InvalidRunDispatchLeaseValueError(
'completed lease metadata is inconsistent',
);
}
assertRunDispatchLeaseVersion('completedAtMs', value.completedAtMs);
}
export function runDispatchLeaseExpiration(
nowMs: number,
durationMs: number,
): number {
assertRunDispatchLeaseVersion('nowMs', nowMs);
assertRunDispatchLeaseDuration(durationMs);
if (nowMs > Number.MAX_SAFE_INTEGER - durationMs) {
throw new InvalidRunDispatchLeaseValueError(
'lease expiration exceeds the safe integer range',
);
}
return nowMs + durationMs;
}
back/runtime/domain/runDispatchOffer.ts
+108
View File
@@ -0,0 +1,108 @@
import { createHash } from 'crypto';
import type { ExecutionSpec } from './execution';
import { cloneExecutionSpec } from './executionSpec';
import type { RunDispatchCandidate } from './runDispatchCandidate';
import {
assertRunDispatchLeaseRecord,
type RunDispatchLeaseRecord,
} from './runDispatchLease';
export const RUN_DISPATCH_OFFER_ID_PATTERN = /^[0-9a-f]{64}$/;
export interface RunDispatchPlan {
/** Untrusted until normalized by the Dispatcher. */
placement: unknown;
executionSpec: ExecutionSpec;
}
export interface ClaimedExecutionOffer {
/** Stable for one Attempt lease generation, including crash recovery. */
offerId: string;
/** Worker dedupe must reject the same offerId with a different digest. */
executionSpecDigest: string;
deliveryKind: 'new_claim' | 'lease_recovery';
candidate: RunDispatchCandidate;
worker: {
id: string;
sessionId: string;
generation: number;
};
/** Contains the opaque lease capability; never log or persist outside its owner. */
lease: RunDispatchLeaseRecord;
executionSpec: ExecutionSpec;
placementScore?: number;
}
export interface RunDispatcherStats {
recoveryPages: number;
recoveriesScanned: number;
recoveryPlansUnavailable: number;
candidatePages: number;
candidatesScanned: number;
workerPages: number;
workersScanned: number;
plansUnavailable: number;
matchingWorkers: number;
claimAttempts: number;
claimRaces: number;
}
export type RunDispatcherIdleReason =
| 'recovery_plans_unavailable'
| 'recovery_scan_budget_exhausted'
| 'no_candidates'
| 'no_workers'
| 'plans_unavailable'
| 'no_match'
| 'claim_raced'
| 'claim_budget_exhausted'
| 'scan_budget_exhausted';
export type RunDispatcherResult =
| {
status: 'offered';
offer: ClaimedExecutionOffer;
stats: RunDispatcherStats;
truncated: boolean;
}
| {
status: 'idle';
reason: RunDispatcherIdleReason;
stats: RunDispatcherStats;
truncated: boolean;
};
export function createRunDispatchOfferId(
lease: RunDispatchLeaseRecord,
): string {
assertRunDispatchLeaseRecord(lease);
if (lease.status !== 'leased') {
throw new TypeError(
'Execution offer requires an active Run dispatch lease',
);
}
return createHash('sha256')
.update('qinglong-run-dispatch-offer-v1\0', 'utf8')
.update(lease.attemptId, 'utf8')
.update('\0', 'utf8')
.update(String(lease.leaseGeneration), 'utf8')
.update('\0', 'utf8')
.update(lease.workerId, 'utf8')
.update('\0', 'utf8')
.update(lease.workerSessionId, 'utf8')
.update('\0', 'utf8')
.update(String(lease.workerGeneration), 'utf8')
.digest('hex');
}
export function createExecutionSpecDigest(spec: ExecutionSpec): string {
return createHash('sha256')
.update(JSON.stringify(cloneExecutionSpec(spec)), 'utf8')
.digest('hex');
}
export function assertRunDispatchOfferId(value: string): void {
if (!RUN_DISPATCH_OFFER_ID_PATTERN.test(value)) {
throw new TypeError('Run dispatch offer ID is invalid');
}
}
back/runtime/domain/runDispatchPlan.ts
+23
View File
@@ -0,0 +1,23 @@
import type { ExecutionSpec } from './execution';
import { cloneExecutionSpec } from './executionSpec';
import type { RunDispatchCandidate } from './runDispatchCandidate';
/** Normalizes a trusted plan and binds it to one persisted candidate identity. */
export function executionSpecForRunDispatchCandidate(
candidate: RunDispatchCandidate,
value: ExecutionSpec,
): ExecutionSpec {
const executionSpec = cloneExecutionSpec(value);
if (
executionSpec.runId !== candidate.runId ||
executionSpec.attemptId !== candidate.attemptId ||
executionSpec.projectId !== candidate.projectId ||
executionSpec.taskId !== candidate.taskId ||
executionSpec.taskRevision !== candidate.taskRevision
) {
throw new TypeError(
'ExecutionSpec identity does not match its dispatch candidate',
);
}
return executionSpec;
}
back/runtime/domain/runDispatchRecovery.ts
+58
View File
@@ -0,0 +1,58 @@
import {
assertRunDispatchCandidate,
assertRunDispatchCandidatePageSize,
type RunDispatchCandidate,
} from './runDispatchCandidate';
import {
assertRunDispatchId,
assertRunDispatchLeaseRecord,
assertRunDispatchLeaseVersion,
type RunDispatchLeaseRecord,
} from './runDispatchLease';
export const MAX_RUN_DISPATCH_RECOVERY_PAGE_SIZE = 64;
export interface RunDispatchRecoveryCursor {
expiresAtMs: number;
attemptId: string;
}
export interface RecoverableRunDispatch {
candidate: RunDispatchCandidate;
lease: RunDispatchLeaseRecord;
}
export function assertRunDispatchRecoveryCursor(
cursor: RunDispatchRecoveryCursor,
): void {
assertRunDispatchLeaseVersion('expiresAtMs', cursor.expiresAtMs);
assertRunDispatchId('attemptId', cursor.attemptId);
}
export function assertRecoverableRunDispatch(
recovery: RecoverableRunDispatch,
): void {
if (!recovery || typeof recovery !== 'object' || Array.isArray(recovery)) {
throw new TypeError('Recoverable Run dispatch must be an object');
}
assertRunDispatchCandidate(recovery.candidate);
assertRunDispatchLeaseRecord(recovery.lease);
if (
recovery.lease.status !== 'leased' ||
recovery.lease.runId !== recovery.candidate.runId ||
recovery.lease.attemptId !== recovery.candidate.attemptId
) {
throw new TypeError(
'Recoverable Run dispatch candidate and active lease do not match',
);
}
}
export function assertRunDispatchRecoveryPageSize(limit: number): void {
assertRunDispatchCandidatePageSize(limit);
if (limit > MAX_RUN_DISPATCH_RECOVERY_PAGE_SIZE) {
throw new RangeError(
`Run dispatch recovery page size must not exceed ${MAX_RUN_DISPATCH_RECOVERY_PAGE_SIZE}`,
);
}
}
back/runtime/domain/runRetryPolicy.ts
+138
View File
@@ -0,0 +1,138 @@
export const RUN_RETRY_SAFETIES = [
'unknown',
'idempotent',
'deduplicated',
] as const;
export type RunRetrySafety = (typeof RUN_RETRY_SAFETIES)[number];
export const MAX_RUN_ATTEMPTS = 16;
export const MAX_RUN_RETRY_BACKOFF_MS = 24 * 60 * 60 * 1000;
export interface RunRetryPolicyDefinition {
maxAttempts: number;
retryOnLost: boolean;
safety: RunRetrySafety;
backoffBaseMs: number;
backoffMaxMs: number;
}
export interface RunRetryPolicyRecord extends RunRetryPolicyDefinition {
runId: string;
nextAttemptAtMs?: number;
version: number;
createdAtMs: number;
updatedAtMs: number;
}
export const NO_AUTOMATIC_RUN_RETRY: Readonly<RunRetryPolicyDefinition> = {
maxAttempts: 1,
retryOnLost: false,
safety: 'unknown',
backoffBaseMs: 0,
backoffMaxMs: 0,
};
export class InvalidRunRetryPolicyError extends TypeError {
readonly code = 'INVALID_RUN_RETRY_POLICY';
constructor(message: string) {
super(message);
this.name = 'InvalidRunRetryPolicyError';
}
}
function assertNonNegativeTime(name: string, value: number): void {
if (!Number.isSafeInteger(value) || value < 0) {
throw new InvalidRunRetryPolicyError(
`${name} must be a non-negative safe integer`,
);
}
}
export function assertRunRetryPolicyDefinition(
policy: RunRetryPolicyDefinition,
): void {
if (
!Number.isSafeInteger(policy.maxAttempts) ||
policy.maxAttempts < 1 ||
policy.maxAttempts > MAX_RUN_ATTEMPTS
) {
throw new InvalidRunRetryPolicyError(
`maxAttempts must be between 1 and ${MAX_RUN_ATTEMPTS}`,
);
}
if (typeof policy.retryOnLost !== 'boolean') {
throw new InvalidRunRetryPolicyError('retryOnLost must be boolean');
}
if (!RUN_RETRY_SAFETIES.includes(policy.safety)) {
throw new InvalidRunRetryPolicyError('safety is not supported');
}
assertNonNegativeTime('backoffBaseMs', policy.backoffBaseMs);
assertNonNegativeTime('backoffMaxMs', policy.backoffMaxMs);
if (
policy.backoffBaseMs > MAX_RUN_RETRY_BACKOFF_MS ||
policy.backoffMaxMs > MAX_RUN_RETRY_BACKOFF_MS
) {
throw new InvalidRunRetryPolicyError(
`retry backoff cannot exceed ${MAX_RUN_RETRY_BACKOFF_MS}ms`,
);
}
if (policy.backoffMaxMs < policy.backoffBaseMs) {
throw new InvalidRunRetryPolicyError(
'backoffMaxMs cannot be smaller than backoffBaseMs',
);
}
}
export function assertAdmittedRunRetryPolicy(
policy: RunRetryPolicyDefinition,
): void {
assertRunRetryPolicyDefinition(policy);
if (
policy.retryOnLost &&
policy.maxAttempts > 1 &&
policy.safety === 'unknown'
) {
throw new InvalidRunRetryPolicyError(
'automatic lost retry requires idempotent or deduplicated safety',
);
}
}
export function assertRunRetryPolicyRecord(policy: RunRetryPolicyRecord): void {
assertRunRetryPolicyDefinition(policy);
if (!policy.runId) {
throw new InvalidRunRetryPolicyError('runId is required');
}
if (!Number.isSafeInteger(policy.version) || policy.version < 0) {
throw new InvalidRunRetryPolicyError(
'version must be a non-negative safe integer',
);
}
assertNonNegativeTime('createdAtMs', policy.createdAtMs);
assertNonNegativeTime('updatedAtMs', policy.updatedAtMs);
if (policy.updatedAtMs < policy.createdAtMs) {
throw new InvalidRunRetryPolicyError(
'updatedAtMs cannot be earlier than createdAtMs',
);
}
if (policy.nextAttemptAtMs !== undefined) {
assertNonNegativeTime('nextAttemptAtMs', policy.nextAttemptAtMs);
}
}
export function runRetryDelayMs(
policy: RunRetryPolicyDefinition,
lostAttempt: number,
): number {
assertRunRetryPolicyDefinition(policy);
if (!Number.isSafeInteger(lostAttempt) || lostAttempt < 1) {
throw new InvalidRunRetryPolicyError(
'lostAttempt must be a positive safe integer',
);
}
if (policy.backoffBaseMs === 0) return 0;
const exponent = Math.min(lostAttempt - 1, MAX_RUN_ATTEMPTS - 1);
return Math.min(policy.backoffMaxMs, policy.backoffBaseMs * 2 ** exponent);
}
back/runtime/domain/runStateMachine.ts
+559
View File
@@ -0,0 +1,559 @@
import type {
RunAttemptRecord,
RunAttemptStatus,
RunCancellationReason,
RunRecord,
RunStatus,
} from './run';
import { RUN_CANCELLATION_REASONS } from './run';
import {
InvalidRunAttemptTransitionError,
InvalidRunTransitionError,
InvalidTransitionMetadataError,
InvalidTransitionTimestampError,
RunVersionConflictError,
} from './stateMachineErrors';
export const MAX_RUN_ERROR_CODE_LENGTH = 128;
export const MAX_RUN_ERROR_SUMMARY_LENGTH = 1024;
export const MAX_EXECUTOR_HANDLE_LENGTH = 2048;
export const MAX_LOG_ARTIFACT_ID_LENGTH = 36;
export const RUN_TRANSITIONS: Readonly<
Record<RunStatus, readonly RunStatus[]>
> = {
created: ['queued', 'cancelled'],
queued: ['dispatching', 'cancelled', 'timed_out'],
dispatching: [
'running',
'retry_wait',
'failed',
'cancelled',
'timed_out',
'lost',
],
running: [
'waiting_approval',
'retry_wait',
'succeeded',
'failed',
'cancelled',
'timed_out',
'lost',
],
waiting_approval: ['running', 'cancelled', 'timed_out'],
retry_wait: ['queued', 'cancelled', 'timed_out'],
lost: ['retry_wait', 'queued', 'failed', 'cancelled'],
succeeded: [],
failed: [],
cancelled: [],
timed_out: [],
};
export const RUN_ATTEMPT_TRANSITIONS: Readonly<
Record<RunAttemptStatus, readonly RunAttemptStatus[]>
> = {
claimed: ['starting', 'cancelled', 'lost'],
starting: ['running', 'failed', 'cancelled', 'timed_out', 'lost'],
running: ['succeeded', 'failed', 'cancelled', 'timed_out', 'lost'],
succeeded: [],
failed: [],
cancelled: [],
timed_out: [],
lost: [],
};
const TERMINAL_RUN_STATUSES = new Set<RunStatus>([
'succeeded',
'failed',
'cancelled',
'timed_out',
]);
const TERMINAL_RUN_ATTEMPT_STATUSES = new Set<RunAttemptStatus>([
'succeeded',
'failed',
'cancelled',
'timed_out',
'lost',
]);
const ERROR_RUN_STATUSES = new Set<RunStatus>([
'retry_wait',
'failed',
'cancelled',
'timed_out',
'lost',
]);
const ERROR_RUN_ATTEMPT_STATUSES = new Set<RunAttemptStatus>([
'failed',
'cancelled',
'timed_out',
'lost',
]);
export interface RunDomainEventDraft {
sequence: number;
type: string;
payload: Readonly<Record<string, unknown>>;
}
export interface RunTransitionCommand {
to: RunStatus;
expectedVersion: number;
atMs: number;
errorCode?: string;
errorSummary?: string;
}
export interface RunAttemptTransitionCommand {
to: RunAttemptStatus;
expectedRunVersion: number;
atMs: number;
executorHandle?: string;
pid?: number;
logArtifactId?: string;
deadlineAtMs?: number;
callbackTokenHash?: string;
callbackSequence?: number;
exitCode?: number;
errorCode?: string;
errorSummary?: string;
}
export interface RunTransitionDecision {
run: RunRecord;
event: RunDomainEventDraft;
}
export interface RunAttemptTransitionDecision {
run: RunRecord;
attempt: RunAttemptRecord;
event: RunDomainEventDraft;
}
export interface RunCancellationRequestCommand {
expectedVersion: number;
atMs: number;
reason: RunCancellationReason;
}
export type RunCancellationRequestDecision =
| {
status: 'accepted';
run: RunRecord;
event: RunDomainEventDraft;
}
| {
status: 'already_requested' | 'already_terminal';
run: RunRecord;
};
export function isTerminalRunStatus(status: RunStatus): boolean {
return TERMINAL_RUN_STATUSES.has(status);
}
export function isTerminalRunAttemptStatus(status: RunAttemptStatus): boolean {
return TERMINAL_RUN_ATTEMPT_STATUSES.has(status);
}
function assertVersion(run: RunRecord, expectedVersion: number): void {
if (!Number.isInteger(expectedVersion) || expectedVersion < 0) {
throw new InvalidTransitionMetadataError(
'expectedVersion must be a non-negative integer',
);
}
if (run.version !== expectedVersion) {
throw new RunVersionConflictError(run.id, expectedVersion, run.version);
}
}
function assertTimestamp(
atMs: number,
createdAtMs: number,
startedAtMs?: number,
): void {
if (!Number.isSafeInteger(atMs) || atMs < 0) {
throw new InvalidTransitionTimestampError(
'Transition timestamp must be a non-negative safe integer',
);
}
if (atMs < createdAtMs) {
throw new InvalidTransitionTimestampError(
'Transition timestamp cannot be earlier than creation time',
);
}
if (startedAtMs !== undefined && atMs < startedAtMs) {
throw new InvalidTransitionTimestampError(
'Transition timestamp cannot be earlier than start time',
);
}
}
function assertErrorMetadata(
status: RunStatus | RunAttemptStatus,
errorStatuses: ReadonlySet<RunStatus | RunAttemptStatus>,
errorCode?: string,
errorSummary?: string,
): void {
if (
(errorCode !== undefined || errorSummary !== undefined) &&
!errorStatuses.has(status)
) {
throw new InvalidTransitionMetadataError(
'Error metadata is not allowed for the target status',
);
}
if (errorCode !== undefined && errorCode.length > MAX_RUN_ERROR_CODE_LENGTH) {
throw new InvalidTransitionMetadataError('errorCode is too long');
}
if (
errorSummary !== undefined &&
errorSummary.length > MAX_RUN_ERROR_SUMMARY_LENGTH
) {
throw new InvalidTransitionMetadataError('errorSummary is too long');
}
}
function assertAttemptExecutionMetadata(
attempt: RunAttemptRecord,
command: RunAttemptTransitionCommand,
): void {
const hasExecutionMetadata =
command.executorHandle !== undefined ||
command.pid !== undefined ||
command.logArtifactId !== undefined ||
command.deadlineAtMs !== undefined ||
command.callbackTokenHash !== undefined;
if (
hasExecutionMetadata &&
command.to !== 'starting' &&
command.to !== 'running'
) {
throw new InvalidTransitionMetadataError(
'Execution metadata is only allowed while an Attempt is starting or running',
);
}
if (
command.executorHandle !== undefined &&
(command.executorHandle.length < 1 ||
command.executorHandle.length > MAX_EXECUTOR_HANDLE_LENGTH)
) {
throw new InvalidTransitionMetadataError(
'executorHandle has an invalid length',
);
}
if (
command.pid !== undefined &&
(!Number.isSafeInteger(command.pid) || command.pid < 1)
) {
throw new InvalidTransitionMetadataError(
'pid must be a positive safe integer',
);
}
if (
command.logArtifactId !== undefined &&
(command.logArtifactId.length < 1 ||
command.logArtifactId.length > MAX_LOG_ARTIFACT_ID_LENGTH)
) {
throw new InvalidTransitionMetadataError(
'logArtifactId has an invalid length',
);
}
if (command.deadlineAtMs !== undefined) {
if (command.to !== 'starting') {
throw new InvalidTransitionMetadataError(
'deadlineAtMs is only allowed when an Attempt starts',
);
}
if (
!Number.isSafeInteger(command.deadlineAtMs) ||
command.deadlineAtMs <= command.atMs
) {
throw new InvalidTransitionMetadataError(
'deadlineAtMs must be a safe integer after the transition time',
);
}
if (
attempt.deadlineAtMs !== undefined &&
attempt.deadlineAtMs !== command.deadlineAtMs
) {
throw new InvalidTransitionMetadataError(
'deadlineAtMs cannot replace an existing Attempt deadline',
);
}
}
if (command.callbackTokenHash !== undefined) {
if (command.to !== 'starting') {
throw new InvalidTransitionMetadataError(
'callbackTokenHash is only allowed when an Attempt starts',
);
}
if (!/^[a-f0-9]{64}$/.test(command.callbackTokenHash)) {
throw new InvalidTransitionMetadataError(
'callbackTokenHash must be a lowercase SHA-256 digest',
);
}
if (
attempt.callbackTokenHash !== undefined &&
attempt.callbackTokenHash !== command.callbackTokenHash
) {
throw new InvalidTransitionMetadataError(
'callbackTokenHash cannot replace an existing Attempt token',
);
}
}
}
function assertAttemptCallbackSequence(
attempt: RunAttemptRecord,
command: RunAttemptTransitionCommand,
): void {
if (command.callbackSequence === undefined) return;
if (!isTerminalRunAttemptStatus(command.to)) {
throw new InvalidTransitionMetadataError(
'callbackSequence is only allowed for terminal Attempt states',
);
}
if (
!Number.isSafeInteger(command.callbackSequence) ||
command.callbackSequence !== attempt.callbackSequence + 1
) {
throw new InvalidTransitionMetadataError(
'callbackSequence must advance the Attempt sequence exactly once',
);
}
}
export function reserveRunEvent(
run: RunRecord,
expectedVersion: number,
): { run: RunRecord; sequence: number } {
assertVersion(run, expectedVersion);
const sequence = run.eventSequence + 1;
if (!Number.isSafeInteger(sequence) || sequence < 1) {
throw new InvalidTransitionMetadataError(
'Run event sequence exceeds the supported range',
);
}
return {
run: {
...run,
version: run.version + 1,
eventSequence: sequence,
},
sequence,
};
}
export function requestRunCancellation(
current: RunRecord,
command: RunCancellationRequestCommand,
): RunCancellationRequestDecision {
assertTimestamp(command.atMs, current.createdAtMs, current.startedAtMs);
if (!RUN_CANCELLATION_REASONS.includes(command.reason)) {
throw new InvalidTransitionMetadataError(
'Cancellation reason is not supported',
);
}
assertVersion(current, command.expectedVersion);
if (isTerminalRunStatus(current.status)) {
return { status: 'already_terminal', run: current };
}
if (current.cancelRequestedAtMs !== undefined) {
return { status: 'already_requested', run: current };
}
const reserved = reserveRunEvent(current, command.expectedVersion);
const run: RunRecord = {
...reserved.run,
cancelRequestedAtMs: command.atMs,
cancelReason: command.reason,
};
return {
status: 'accepted',
run,
event: {
sequence: reserved.sequence,
type: 'run.cancel_requested',
payload: {
status: current.status,
reason: command.reason,
requested_at_ms: command.atMs,
version: run.version,
},
},
};
}
export function transitionRun(
current: RunRecord,
command: RunTransitionCommand,
): RunTransitionDecision {
assertTimestamp(command.atMs, current.createdAtMs, current.startedAtMs);
assertErrorMetadata(
command.to,
ERROR_RUN_STATUSES,
command.errorCode,
command.errorSummary,
);
if (!RUN_TRANSITIONS[current.status].includes(command.to)) {
throw new InvalidRunTransitionError(current.id, current.status, command.to);
}
const reserved = reserveRunEvent(current, command.expectedVersion);
const next: RunRecord = {
...reserved.run,
status: command.to,
};
if (command.to === 'queued') {
next.queuedAtMs = command.atMs;
delete next.errorCode;
delete next.errorSummary;
}
if (command.to === 'running' && next.startedAtMs === undefined) {
next.startedAtMs = command.atMs;
}
if (isTerminalRunStatus(command.to)) {
next.finishedAtMs = command.atMs;
}
if (ERROR_RUN_STATUSES.has(command.to)) {
if (command.errorCode !== undefined) next.errorCode = command.errorCode;
if (command.errorSummary !== undefined)
next.errorSummary = command.errorSummary;
} else if (command.to === 'succeeded') {
delete next.errorCode;
delete next.errorSummary;
}
return {
run: next,
event: {
sequence: reserved.sequence,
type: `run.${command.to}`,
payload: {
from_status: current.status,
to_status: command.to,
version: next.version,
...(command.errorCode ? { error_code: command.errorCode } : {}),
},
},
};
}
export function transitionRunAttempt(
currentRun: RunRecord,
currentAttempt: RunAttemptRecord,
command: RunAttemptTransitionCommand,
): RunAttemptTransitionDecision {
if (currentAttempt.runId !== currentRun.id) {
throw new InvalidTransitionMetadataError(
'RunAttempt does not belong to the supplied Run',
);
}
assertTimestamp(
command.atMs,
currentAttempt.createdAtMs,
currentAttempt.startedAtMs,
);
assertErrorMetadata(
command.to,
ERROR_RUN_ATTEMPT_STATUSES,
command.errorCode,
command.errorSummary,
);
assertAttemptExecutionMetadata(currentAttempt, command);
assertAttemptCallbackSequence(currentAttempt, command);
if (!RUN_ATTEMPT_TRANSITIONS[currentAttempt.status].includes(command.to)) {
throw new InvalidRunAttemptTransitionError(
currentAttempt.id,
currentAttempt.status,
command.to,
);
}
if (isTerminalRunStatus(currentRun.status)) {
throw new InvalidTransitionMetadataError(
'Cannot transition an Attempt after its Run is terminal',
);
}
if (
command.exitCode !== undefined &&
!isTerminalRunAttemptStatus(command.to)
) {
throw new InvalidTransitionMetadataError(
'exitCode is only allowed for terminal Attempt states',
);
}
const reserved = reserveRunEvent(currentRun, command.expectedRunVersion);
const nextAttempt: RunAttemptRecord = {
...currentAttempt,
status: command.to,
};
if (command.to === 'running' && nextAttempt.startedAtMs === undefined) {
nextAttempt.startedAtMs = command.atMs;
}
if (isTerminalRunAttemptStatus(command.to)) {
nextAttempt.finishedAtMs = command.atMs;
}
if (command.exitCode !== undefined) {
nextAttempt.exitCode = command.exitCode;
}
if (command.executorHandle !== undefined) {
nextAttempt.executorHandle = command.executorHandle;
}
if (command.pid !== undefined) nextAttempt.pid = command.pid;
if (command.logArtifactId !== undefined) {
nextAttempt.logArtifactId = command.logArtifactId;
}
if (command.deadlineAtMs !== undefined) {
nextAttempt.deadlineAtMs = command.deadlineAtMs;
}
if (command.callbackTokenHash !== undefined) {
nextAttempt.callbackTokenHash = command.callbackTokenHash;
}
if (command.callbackSequence !== undefined) {
nextAttempt.callbackSequence = command.callbackSequence;
}
if (ERROR_RUN_ATTEMPT_STATUSES.has(command.to)) {
if (command.errorCode !== undefined)
nextAttempt.errorCode = command.errorCode;
if (command.errorSummary !== undefined)
nextAttempt.errorSummary = command.errorSummary;
} else if (command.to === 'succeeded') {
delete nextAttempt.errorCode;
delete nextAttempt.errorSummary;
}
return {
run: reserved.run,
attempt: nextAttempt,
event: {
sequence: reserved.sequence,
type: `attempt.${command.to}`,
payload: {
attempt_id: currentAttempt.id,
attempt: currentAttempt.attempt,
from_status: currentAttempt.status,
to_status: command.to,
version: reserved.run.version,
...(command.exitCode !== undefined
? { exit_code: command.exitCode }
: {}),
...(command.errorCode ? { error_code: command.errorCode } : {}),
...(command.deadlineAtMs === undefined
? {}
: { deadline_at_ms: command.deadlineAtMs }),
...(command.callbackSequence === undefined
? {}
: { callback_sequence: command.callbackSequence }),
},
},
};
}
back/runtime/domain/runtimeRollout.ts
+82
View File
@@ -0,0 +1,82 @@
import {
EXECUTION_ORIGINS,
type ExecutionOrigin,
type ExecutionOwner,
} from './run';
export const COMPATIBILITY_MODES = ['off', 'shadow', 'primary'] as const;
export type CompatibilityMode = (typeof COMPATIBILITY_MODES)[number];
export interface RuntimeRolloutConfig {
defaultMode: CompatibilityMode;
origins: Partial<Record<ExecutionOrigin, CompatibilityMode>>;
allowLegacyFallbackBeforeStart: boolean;
}
export interface ExecutionOwnershipDecision {
origin: ExecutionOrigin;
mode: CompatibilityMode;
owner: ExecutionOwner;
}
const VALID_ORIGINS = new Set<ExecutionOrigin>(EXECUTION_ORIGINS);
const VALID_MODES = new Set<CompatibilityMode>(COMPATIBILITY_MODES);
function assertConfig(config: RuntimeRolloutConfig): void {
if (!VALID_MODES.has(config.defaultMode)) {
throw new TypeError('Runtime rollout defaultMode is invalid');
}
for (const [origin, mode] of Object.entries(config.origins)) {
if (!VALID_ORIGINS.has(origin as ExecutionOrigin)) {
throw new TypeError(`Runtime rollout origin is invalid: ${origin}`);
}
if (!VALID_MODES.has(mode as CompatibilityMode)) {
throw new TypeError(`Runtime rollout mode is invalid: ${String(mode)}`);
}
}
}
export class RuntimeRolloutPolicy {
private readonly config: RuntimeRolloutConfig;
constructor(config: RuntimeRolloutConfig) {
assertConfig(config);
this.config = {
defaultMode: config.defaultMode,
origins: { ...config.origins },
allowLegacyFallbackBeforeStart: config.allowLegacyFallbackBeforeStart,
};
}
modeFor(origin: ExecutionOrigin): CompatibilityMode {
return this.config.origins[origin] ?? this.config.defaultMode;
}
decide(origin: ExecutionOrigin): ExecutionOwnershipDecision {
const mode = this.modeFor(origin);
return {
origin,
mode,
owner: mode === 'primary' ? 'runtime' : 'legacy',
};
}
snapshot(): RuntimeRolloutConfig {
return {
defaultMode: this.config.defaultMode,
origins: { ...this.config.origins },
allowLegacyFallbackBeforeStart:
this.config.allowLegacyFallbackBeforeStart,
};
}
}
export function shadowOnlyRollout(
origins: readonly ExecutionOrigin[],
): RuntimeRolloutPolicy {
return new RuntimeRolloutPolicy({
defaultMode: 'off',
origins: Object.fromEntries(origins.map((origin) => [origin, 'shadow'])),
allowLegacyFallbackBeforeStart: false,
});
}
back/runtime/domain/runtimeRolloutManifest.ts
+231
View File
@@ -0,0 +1,231 @@
import type { ExecutionOrigin } from './run';
import {
RuntimeRolloutPolicy,
type CompatibilityMode,
type RuntimeRolloutConfig,
} from './runtimeRollout';
export const RUNTIME_ROLLOUT_MANIFEST_VERSION = 1;
export const MAX_RUNTIME_ROLLOUT_APPROVAL_MS = 30 * 24 * 60 * 60 * 1000;
export const REQUIRED_RUNTIME_ROLLOUT_GATES = [
'durableCancellation',
'startupReconciliation',
'atomicLegacyProjection',
'rollbackDrill',
'edgeBudget',
] as const;
export type RuntimeRolloutGate =
(typeof REQUIRED_RUNTIME_ROLLOUT_GATES)[number];
export interface DisabledRuntimeRolloutManifest {
schemaVersion: typeof RUNTIME_ROLLOUT_MANIFEST_VERSION;
revision: string;
enabled: false;
}
export interface EnabledRuntimeRolloutManifest {
schemaVersion: typeof RUNTIME_ROLLOUT_MANIFEST_VERSION;
revision: string;
enabled: true;
approvedBy: string;
approvedAtMs: number;
expiresAtMs: number;
rollbackPlanRef: string;
rollout: RuntimeRolloutConfig;
gates: Record<RuntimeRolloutGate, 'passed'>;
}
export type RuntimeRolloutManifest =
| DisabledRuntimeRolloutManifest
| EnabledRuntimeRolloutManifest;
export interface RuntimeRolloutManifestDecision {
manifest: RuntimeRolloutManifest;
policy: RuntimeRolloutPolicy;
}
const MANUAL_ORIGIN: ExecutionOrigin = 'manual';
const ALLOWED_MANUAL_MODES = new Set<CompatibilityMode>([
'off',
'shadow',
'primary',
]);
function asObject(value: unknown, name: string): Record<string, unknown> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new TypeError(`${name} must be a JSON object`);
}
return value as Record<string, unknown>;
}
function assertExactKeys(
value: Record<string, unknown>,
expected: readonly string[],
name: string,
): void {
const allowed = new Set(expected);
for (const key of Object.keys(value)) {
if (!allowed.has(key)) throw new TypeError(`${name}.${key} is not allowed`);
}
for (const key of expected) {
if (!(key in value)) throw new TypeError(`${name}.${key} is required`);
}
}
function boundedString(
value: unknown,
name: string,
maxLength: number,
): string {
if (
typeof value !== 'string' ||
value.trim() !== value ||
value.length < 1 ||
value.length > maxLength ||
/[\u0000-\u001f\u007f]/.test(value)
) {
throw new TypeError(`${name} is invalid`);
}
return value;
}
function safeTimestamp(value: unknown, name: string): number {
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
throw new TypeError(`${name} must be a non-negative safe integer`);
}
return value;
}
function disabledPolicy(): RuntimeRolloutPolicy {
return new RuntimeRolloutPolicy({
defaultMode: 'off',
origins: {},
allowLegacyFallbackBeforeStart: false,
});
}
export function parseRuntimeRolloutManifest(
value: unknown,
evaluatedAtMs: number,
): RuntimeRolloutManifestDecision {
const now = safeTimestamp(evaluatedAtMs, 'evaluatedAtMs');
const object = asObject(value, 'manifest');
if (object.enabled === false) {
assertExactKeys(
object,
['schemaVersion', 'revision', 'enabled'],
'manifest',
);
if (object.schemaVersion !== RUNTIME_ROLLOUT_MANIFEST_VERSION) {
throw new TypeError('manifest.schemaVersion is unsupported');
}
const manifest: DisabledRuntimeRolloutManifest = {
schemaVersion: RUNTIME_ROLLOUT_MANIFEST_VERSION,
revision: boundedString(object.revision, 'manifest.revision', 128),
enabled: false,
};
return { manifest, policy: disabledPolicy() };
}
assertExactKeys(
object,
[
'schemaVersion',
'revision',
'enabled',
'approvedBy',
'approvedAtMs',
'expiresAtMs',
'rollbackPlanRef',
'rollout',
'gates',
],
'manifest',
);
if (object.schemaVersion !== RUNTIME_ROLLOUT_MANIFEST_VERSION) {
throw new TypeError('manifest.schemaVersion is unsupported');
}
if (object.enabled !== true) {
throw new TypeError('manifest.enabled must be a boolean');
}
const approvedAtMs = safeTimestamp(
object.approvedAtMs,
'manifest.approvedAtMs',
);
const expiresAtMs = safeTimestamp(object.expiresAtMs, 'manifest.expiresAtMs');
if (approvedAtMs > now) {
throw new TypeError('manifest approval is not active yet');
}
if (expiresAtMs <= now) {
throw new TypeError('manifest approval has expired');
}
if (
expiresAtMs <= approvedAtMs ||
expiresAtMs - approvedAtMs > MAX_RUNTIME_ROLLOUT_APPROVAL_MS
) {
throw new TypeError('manifest approval window is invalid');
}
const rolloutObject = asObject(object.rollout, 'manifest.rollout');
assertExactKeys(
rolloutObject,
['defaultMode', 'origins', 'allowLegacyFallbackBeforeStart'],
'manifest.rollout',
);
if (rolloutObject.defaultMode !== 'off') {
throw new TypeError('manifest.rollout.defaultMode must remain off');
}
if (rolloutObject.allowLegacyFallbackBeforeStart !== false) {
throw new TypeError(
'manifest.rollout.allowLegacyFallbackBeforeStart must remain false',
);
}
const origins = asObject(rolloutObject.origins, 'manifest.rollout.origins');
assertExactKeys(origins, [MANUAL_ORIGIN], 'manifest.rollout.origins');
if (!ALLOWED_MANUAL_MODES.has(origins.manual as CompatibilityMode)) {
throw new TypeError('manifest.rollout.origins.manual is invalid');
}
const gatesObject = asObject(object.gates, 'manifest.gates');
assertExactKeys(
gatesObject,
REQUIRED_RUNTIME_ROLLOUT_GATES,
'manifest.gates',
);
for (const gate of REQUIRED_RUNTIME_ROLLOUT_GATES) {
if (gatesObject[gate] !== 'passed') {
throw new TypeError(`manifest.gates.${gate} must be passed`);
}
}
const rollout: RuntimeRolloutConfig = {
defaultMode: 'off',
origins: { manual: origins.manual as CompatibilityMode },
allowLegacyFallbackBeforeStart: false,
};
const manifest: EnabledRuntimeRolloutManifest = {
schemaVersion: RUNTIME_ROLLOUT_MANIFEST_VERSION,
revision: boundedString(object.revision, 'manifest.revision', 128),
enabled: true,
approvedBy: boundedString(object.approvedBy, 'manifest.approvedBy', 128),
approvedAtMs,
expiresAtMs,
rollbackPlanRef: boundedString(
object.rollbackPlanRef,
'manifest.rollbackPlanRef',
512,
),
rollout,
gates: Object.fromEntries(
REQUIRED_RUNTIME_ROLLOUT_GATES.map((gate) => [gate, 'passed']),
) as Record<RuntimeRolloutGate, 'passed'>,
};
return { manifest, policy: new RuntimeRolloutPolicy(rollout) };
}
export function defaultOffRuntimeRolloutPolicy(): RuntimeRolloutPolicy {
return disabledPolicy();
}
back/runtime/domain/stateMachineErrors.ts
+59
View File
@@ -0,0 +1,59 @@
import type { RunAttemptStatus, RunStatus } from './run';
export class RunStateMachineError extends Error {
constructor(message: string, public readonly code: string) {
super(message);
this.name = new.target.name;
}
}
export class RunVersionConflictError extends RunStateMachineError {
constructor(
public readonly runId: string,
public readonly expectedVersion: number,
public readonly actualVersion: number,
) {
super(
'Run version does not match the expected version',
'RUN_VERSION_CONFLICT',
);
}
}
export class InvalidRunTransitionError extends RunStateMachineError {
constructor(
public readonly runId: string,
public readonly from: RunStatus,
public readonly to: RunStatus,
) {
super(
`Run cannot transition from ${from} to ${to}`,
'INVALID_RUN_TRANSITION',
);
}
}
export class InvalidRunAttemptTransitionError extends RunStateMachineError {
constructor(
public readonly attemptId: string,
public readonly from: RunAttemptStatus,
public readonly to: RunAttemptStatus,
) {
super(
`RunAttempt cannot transition from ${from} to ${to}`,
'INVALID_RUN_ATTEMPT_TRANSITION',
);
}
}
export class InvalidTransitionTimestampError extends RunStateMachineError {
constructor(message: string) {
super(message, 'INVALID_TRANSITION_TIMESTAMP');
}
}
export class InvalidTransitionMetadataError extends RunStateMachineError {
constructor(message: string) {
super(message, 'INVALID_TRANSITION_METADATA');
}
}
back/runtime/domain/taskExecutionRevision.ts
+118
View File
@@ -0,0 +1,118 @@
import type {
ExecutionCommand,
ExecutionResourcePolicy,
ExecutionSpec,
ExecutorType,
} from './execution';
import { EXECUTOR_TYPES } from './execution';
import { cloneExecutionSpec } from './executionSpec';
import type { RunDispatchCandidate } from './runDispatchCandidate';
export interface TaskExecutionSpecTemplate {
command: ExecutionCommand;
workingDirectory?: string;
environmentPolicy: ExecutionSpec['environmentPolicy'];
timeoutMs?: number;
terminationGraceMs: number;
resourcePolicy?: ExecutionResourcePolicy;
}
export interface PinnedTaskExecutionRevision {
projectId: string;
taskId: string;
taskRevision: string;
executorType: ExecutorType;
execution: TaskExecutionSpecTemplate;
/** Opaque immutable recipe for Secret, output and callback capabilities. */
contextRef: string;
}
function assertIdentifier(name: string, value: string, maximum: number): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > maximum ||
/[\u0000-\u001f\u007f]/.test(value)
) {
throw new TypeError(`${name} is invalid`);
}
}
function templateSpec(
revision: PinnedTaskExecutionRevision,
identity: Pick<
ExecutionSpec,
'runId' | 'attemptId' | 'projectId' | 'taskId' | 'taskRevision'
>,
): ExecutionSpec {
const execution = revision.execution;
return {
runId: identity.runId,
attemptId: identity.attemptId,
projectId: identity.projectId,
taskId: identity.taskId,
taskRevision: identity.taskRevision,
command: execution.command,
...(execution.workingDirectory === undefined
? {}
: { workingDirectory: execution.workingDirectory }),
environmentPolicy: execution.environmentPolicy,
...(execution.timeoutMs === undefined
? {}
: { timeoutMs: execution.timeoutMs }),
terminationGraceMs: execution.terminationGraceMs,
...(execution.resourcePolicy === undefined
? {}
: { resourcePolicy: execution.resourcePolicy }),
};
}
export function assertPinnedTaskExecutionRevision(
revision: PinnedTaskExecutionRevision,
): void {
if (!revision || typeof revision !== 'object' || Array.isArray(revision)) {
throw new TypeError('Pinned Task execution revision must be an object');
}
assertIdentifier('projectId', revision.projectId, 128);
assertIdentifier('taskId', revision.taskId, 255);
assertIdentifier('taskRevision', revision.taskRevision, 128);
assertIdentifier('contextRef', revision.contextRef, 512);
if (!EXECUTOR_TYPES.includes(revision.executorType)) {
throw new TypeError('Pinned Task executorType is invalid');
}
cloneExecutionSpec(
templateSpec(revision, {
runId: 'revision-validation-run',
attemptId: 'revision-validation-attempt',
projectId: revision.projectId,
taskId: revision.taskId,
taskRevision: revision.taskRevision,
}),
);
}
export function executionSpecFromPinnedTaskRevision(
candidate: RunDispatchCandidate,
revision: PinnedTaskExecutionRevision,
): ExecutionSpec {
assertPinnedTaskExecutionRevision(revision);
if (
revision.projectId !== candidate.projectId ||
revision.taskId !== candidate.taskId ||
revision.taskRevision !== candidate.taskRevision ||
revision.executorType !== candidate.executorType
) {
throw new TypeError(
'Pinned Task execution revision does not match its dispatch candidate',
);
}
return cloneExecutionSpec(
templateSpec(revision, {
runId: candidate.runId,
attemptId: candidate.attemptId,
projectId: candidate.projectId,
taskId: candidate.taskId,
taskRevision: candidate.taskRevision,
}),
);
}
back/runtime/domain/taskExecutionRevisionRecord.ts
+137
View File
@@ -0,0 +1,137 @@
import { createHash } from 'crypto';
import type {
ExecutionCommand,
ExecutionResourcePolicy,
ExecutionSpec,
} from './execution';
import { cloneExecutionSpec } from './executionSpec';
import {
assertPinnedTaskExecutionRevision,
type PinnedTaskExecutionRevision,
type TaskExecutionSpecTemplate,
} from './taskExecutionRevision';
export interface PinnedTaskExecutionRevisionRecord
extends PinnedTaskExecutionRevision {
contentDigest: string;
createdAtMs: number;
}
export class TaskExecutionRevisionCorruptError extends Error {
constructor(message: string) {
super(message);
this.name = 'TaskExecutionRevisionCorruptError';
}
}
function frozenCommand(command: ExecutionCommand): ExecutionCommand {
return command.kind === 'argv'
? Object.freeze({
kind: 'argv' as const,
file: command.file,
args: Object.freeze([...command.args]),
})
: Object.freeze({
kind: 'shell' as const,
command: command.command,
...(command.shell === undefined ? {} : { shell: command.shell }),
});
}
function frozenResourcePolicy(
policy: ExecutionResourcePolicy | undefined,
): ExecutionResourcePolicy | undefined {
if (policy === undefined) return undefined;
return Object.freeze({
...(policy.memoryBytes === undefined
? {}
: { memoryBytes: Object.freeze({ ...policy.memoryBytes }) }),
...(policy.cpuMillisPerSecond === undefined
? {}
: {
cpuMillisPerSecond: Object.freeze({
...policy.cpuMillisPerSecond,
}),
}),
...(policy.filesystemIsolation === undefined
? {}
: { filesystemIsolation: policy.filesystemIsolation }),
...(policy.networkIsolation === undefined
? {}
: { networkIsolation: policy.networkIsolation }),
});
}
function normalizedTemplate(
revision: PinnedTaskExecutionRevision,
): TaskExecutionSpecTemplate {
const spec: ExecutionSpec = cloneExecutionSpec({
runId: 'task-revision-normalization-run',
attemptId: 'task-revision-normalization-attempt',
projectId: revision.projectId,
taskId: revision.taskId,
taskRevision: revision.taskRevision,
command: revision.execution.command,
...(revision.execution.workingDirectory === undefined
? {}
: { workingDirectory: revision.execution.workingDirectory }),
environmentPolicy: revision.execution.environmentPolicy,
...(revision.execution.timeoutMs === undefined
? {}
: { timeoutMs: revision.execution.timeoutMs }),
terminationGraceMs: revision.execution.terminationGraceMs,
...(revision.execution.resourcePolicy === undefined
? {}
: { resourcePolicy: revision.execution.resourcePolicy }),
});
const resourcePolicy = frozenResourcePolicy(spec.resourcePolicy);
return Object.freeze({
command: frozenCommand(spec.command),
...(spec.workingDirectory === undefined
? {}
: { workingDirectory: spec.workingDirectory }),
environmentPolicy: spec.environmentPolicy,
...(spec.timeoutMs === undefined ? {} : { timeoutMs: spec.timeoutMs }),
terminationGraceMs: spec.terminationGraceMs,
...(resourcePolicy === undefined ? {} : { resourcePolicy }),
});
}
/** Removes unknown fields, deep-clones mutable values and freezes the result. */
export function normalizePinnedTaskExecutionRevision(
revision: PinnedTaskExecutionRevision,
): PinnedTaskExecutionRevision {
assertPinnedTaskExecutionRevision(revision);
return Object.freeze({
projectId: revision.projectId,
taskId: revision.taskId,
taskRevision: revision.taskRevision,
executorType: revision.executorType,
execution: normalizedTemplate(revision),
contextRef: revision.contextRef,
});
}
export function taskExecutionRevisionDigest(
revision: PinnedTaskExecutionRevision,
): string {
const normalized = normalizePinnedTaskExecutionRevision(revision);
return createHash('sha256')
.update(JSON.stringify(normalized), 'utf8')
.digest('hex');
}
export function createPinnedTaskExecutionRevisionRecord(
revision: PinnedTaskExecutionRevision,
createdAtMs: number,
): PinnedTaskExecutionRevisionRecord {
if (!Number.isSafeInteger(createdAtMs) || createdAtMs < 0) {
throw new RangeError('createdAtMs must be a non-negative safe integer');
}
const normalized = normalizePinnedTaskExecutionRevision(revision);
return Object.freeze({
...normalized,
contentDigest: taskExecutionRevisionDigest(normalized),
createdAtMs,
});
}
back/runtime/domain/worker.ts
+402
View File
@@ -0,0 +1,402 @@
import { createHash } from 'crypto';
export const WORKER_STATUSES = ['online', 'draining', 'offline'] as const;
export type WorkerStatus = (typeof WORKER_STATUSES)[number];
export const MAX_WORKER_ID_LENGTH = 128;
export const MAX_WORKER_CAPABILITIES_BYTES = 16 * 1024;
export const MAX_WORKER_EXECUTORS = 16;
export const MAX_WORKER_RUNTIMES = 32;
export const MAX_WORKER_LABELS = 32;
export const MAX_WORKER_FEATURES = 32;
export const MAX_WORKER_GPUS = 8;
export const MAX_WORKER_CONCURRENT_RUNS = 1024;
const UUID_V7_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
const WORKER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/;
const CAPABILITY_NAME_PATTERN = /^[a-z0-9][a-z0-9._+-]*$/;
const LABEL_KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._/-]*$/;
export interface WorkerRuntimeCapability {
name: string;
version: string;
}
export interface WorkerGpuCapability {
vendor: string;
model?: string;
memoryBytes?: number;
}
export interface WorkerCapacity {
cpuCores?: number;
memoryBytes?: number;
diskBytes?: number;
gpu?: readonly WorkerGpuCapability[];
}
export interface WorkerCapabilities {
architecture: string;
operatingSystem: string;
executors: readonly string[];
runtimes: readonly WorkerRuntimeCapability[];
labels: Readonly<Record<string, string>>;
capacity: WorkerCapacity;
features: readonly string[];
}
export interface WorkerRecord {
id: string;
sessionId: string;
generation: number;
status: WorkerStatus;
version: number;
capabilities: WorkerCapabilities;
capabilitiesHash: string;
maxConcurrentRuns: number;
availableSlots: number;
registeredAtMs: number;
lastHeartbeatAtMs: number;
leaseExpiresAtMs: number;
updatedAtMs: number;
}
export class InvalidWorkerValueError extends Error {
constructor(message: string) {
super(message);
this.name = 'InvalidWorkerValueError';
}
}
export class WorkerFenceRejectedError extends Error {
constructor(
readonly workerId: string,
readonly reason:
| 'missing'
| 'session_mismatch'
| 'generation_mismatch'
| 'version_mismatch'
| 'lease_expired'
| 'offline',
) {
super(`Worker ${workerId} fence rejected: ${reason}`);
this.name = 'WorkerFenceRejectedError';
}
}
export class WorkerSessionConflictError extends Error {
constructor(readonly workerId: string) {
super(`Worker ${workerId} session replay conflicts with persisted data`);
this.name = 'WorkerSessionConflictError';
}
}
function invalid(message: string): never {
throw new InvalidWorkerValueError(message);
}
function record(value: unknown, name: string): Record<string, unknown> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
invalid(`${name} must be an object`);
}
return value as Record<string, unknown>;
}
function assertKeys(
value: Record<string, unknown>,
name: string,
required: readonly string[],
optional: readonly string[] = [],
): void {
const allowed = new Set([...required, ...optional]);
if (
required.some((key) => !Object.hasOwn(value, key)) ||
Object.keys(value).some((key) => !allowed.has(key))
) {
invalid(`${name} fields do not match the supported schema`);
}
}
function boundedString(
value: unknown,
name: string,
maximum: number,
pattern?: RegExp,
): string {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > maximum ||
value.includes('\0') ||
/[\u0000-\u001f\u007f]/.test(value) ||
(pattern && !pattern.test(value))
) {
invalid(`${name} is invalid`);
}
return value;
}
function positiveInteger(
value: unknown,
name: string,
maximum = Number.MAX_SAFE_INTEGER,
): number {
if (
!Number.isSafeInteger(value) ||
(value as number) < 1 ||
(value as number) > maximum
) {
invalid(`${name} must be a positive safe integer`);
}
return value as number;
}
function optionalPositiveInteger(
value: unknown,
name: string,
): number | undefined {
return value === undefined ? undefined : positiveInteger(value, name);
}
function uniqueSortedStrings(
value: unknown,
name: string,
maximumItems: number,
): string[] {
if (!Array.isArray(value) || value.length > maximumItems) {
invalid(`${name} must be an array with at most ${maximumItems} items`);
}
const normalized = value.map((item, index) =>
boundedString(item, `${name}[${index}]`, 64, CAPABILITY_NAME_PATTERN),
);
if (new Set(normalized).size !== normalized.length) {
invalid(`${name} must not contain duplicates`);
}
return normalized.sort();
}
function normalizeRuntimes(value: unknown): WorkerRuntimeCapability[] {
if (!Array.isArray(value) || value.length > MAX_WORKER_RUNTIMES) {
invalid(`runtimes must contain at most ${MAX_WORKER_RUNTIMES} items`);
}
const runtimes = value.map((candidate, index) => {
const item = record(candidate, `runtimes[${index}]`);
assertKeys(item, `runtimes[${index}]`, ['name', 'version']);
return {
name: boundedString(
item.name,
`runtimes[${index}].name`,
64,
CAPABILITY_NAME_PATTERN,
),
version: boundedString(item.version, `runtimes[${index}].version`, 64),
};
});
const keys = runtimes.map((item) => `${item.name}\0${item.version}`);
if (new Set(keys).size !== keys.length) {
invalid('runtimes must not contain duplicates');
}
return runtimes.sort(
(left, right) =>
left.name.localeCompare(right.name) ||
left.version.localeCompare(right.version),
);
}
function normalizeLabels(value: unknown): Record<string, string> {
const labels = record(value, 'labels');
const entries = Object.entries(labels);
if (entries.length > MAX_WORKER_LABELS) {
invalid(`labels must contain at most ${MAX_WORKER_LABELS} entries`);
}
return Object.fromEntries(
entries
.map(([key, candidate]) => [
boundedString(key, 'label key', 128, LABEL_KEY_PATTERN),
boundedString(candidate, `labels.${key}`, 256),
])
.sort(([left], [right]) => left.localeCompare(right)),
);
}
function normalizeGpu(value: unknown): WorkerGpuCapability[] | undefined {
if (value === undefined) return undefined;
if (!Array.isArray(value) || value.length > MAX_WORKER_GPUS) {
invalid(`capacity.gpu must contain at most ${MAX_WORKER_GPUS} items`);
}
return value.map((candidate, index) => {
const item = record(candidate, `capacity.gpu[${index}]`);
assertKeys(
item,
`capacity.gpu[${index}]`,
['vendor'],
['model', 'memoryBytes'],
);
return {
vendor: boundedString(
item.vendor,
`capacity.gpu[${index}].vendor`,
64,
CAPABILITY_NAME_PATTERN,
),
...(item.model === undefined
? {}
: {
model: boundedString(
item.model,
`capacity.gpu[${index}].model`,
128,
),
}),
...(item.memoryBytes === undefined
? {}
: {
memoryBytes: positiveInteger(
item.memoryBytes,
`capacity.gpu[${index}].memoryBytes`,
),
}),
};
});
}
function normalizeCapacity(value: unknown): WorkerCapacity {
const capacity = record(value, 'capacity');
assertKeys(
capacity,
'capacity',
[],
['cpuCores', 'memoryBytes', 'diskBytes', 'gpu'],
);
const cpuCores =
capacity.cpuCores === undefined
? undefined
: positiveInteger(capacity.cpuCores, 'capacity.cpuCores', 4096);
const memoryBytes = optionalPositiveInteger(
capacity.memoryBytes,
'capacity.memoryBytes',
);
const diskBytes = optionalPositiveInteger(
capacity.diskBytes,
'capacity.diskBytes',
);
const gpu = normalizeGpu(capacity.gpu);
return {
...(cpuCores === undefined ? {} : { cpuCores }),
...(memoryBytes === undefined ? {} : { memoryBytes }),
...(diskBytes === undefined ? {} : { diskBytes }),
...(gpu === undefined ? {} : { gpu }),
};
}
export function assertWorkerId(value: string): void {
boundedString(value, 'workerId', MAX_WORKER_ID_LENGTH, WORKER_ID_PATTERN);
}
export function assertWorkerSessionId(value: string): void {
if (!UUID_V7_PATTERN.test(value))
invalid('sessionId must be a lowercase UUIDv7');
}
export function normalizeWorkerCapabilities(
value: unknown,
): WorkerCapabilities {
const capabilities = record(value, 'capabilities');
assertKeys(capabilities, 'capabilities', [
'architecture',
'operatingSystem',
'executors',
'runtimes',
'labels',
'capacity',
'features',
]);
const normalized: WorkerCapabilities = {
architecture: boundedString(
capabilities.architecture,
'architecture',
32,
CAPABILITY_NAME_PATTERN,
),
operatingSystem: boundedString(
capabilities.operatingSystem,
'operatingSystem',
32,
CAPABILITY_NAME_PATTERN,
),
executors: uniqueSortedStrings(
capabilities.executors,
'executors',
MAX_WORKER_EXECUTORS,
),
runtimes: normalizeRuntimes(capabilities.runtimes),
labels: normalizeLabels(capabilities.labels),
capacity: normalizeCapacity(capabilities.capacity),
features: uniqueSortedStrings(
capabilities.features,
'features',
MAX_WORKER_FEATURES,
),
};
const serialized = JSON.stringify(normalized);
if (Buffer.byteLength(serialized, 'utf8') > MAX_WORKER_CAPABILITIES_BYTES) {
invalid('capabilities exceed the byte limit');
}
return normalized;
}
export function serializeWorkerCapabilities(value: unknown): string {
return JSON.stringify(normalizeWorkerCapabilities(value));
}
export function parseWorkerCapabilities(value: string): WorkerCapabilities {
if (
Buffer.byteLength(value, 'utf8') < 2 ||
Buffer.byteLength(value, 'utf8') > MAX_WORKER_CAPABILITIES_BYTES
) {
invalid('capabilities size is outside the allowed range');
}
let parsed: unknown;
try {
parsed = JSON.parse(value);
} catch {
return invalid('capabilities are not valid JSON');
}
return normalizeWorkerCapabilities(parsed);
}
export function hashWorkerCapabilities(serialized: string): string {
const canonical = serializeWorkerCapabilities(
parseWorkerCapabilities(serialized),
);
return createHash('sha256').update(canonical).digest('hex');
}
export function assertWorkerConcurrency(
maxConcurrentRuns: number,
availableSlots: number,
): void {
positiveInteger(
maxConcurrentRuns,
'maxConcurrentRuns',
MAX_WORKER_CONCURRENT_RUNS,
);
if (
!Number.isSafeInteger(availableSlots) ||
availableSlots < 0 ||
availableSlots > maxConcurrentRuns
) {
invalid('availableSlots must be between 0 and maxConcurrentRuns');
}
}
export function isWorkerLeaseActive(
worker: WorkerRecord,
observedAtMs: number,
): boolean {
if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0) {
invalid('observedAtMs must be a non-negative safe integer');
}
return worker.status !== 'offline' && worker.leaseExpiresAtMs > observedAtMs;
}
back/runtime/domain/workerExecutionCompletionReceiptAuthentication.ts
+50
View File
@@ -0,0 +1,50 @@
import { createHash, timingSafeEqual } from 'crypto';
import type { CompletionReceipt } from './completionReceipt';
import type { ExecutionContext } from './execution';
export const WORKER_COMPLETION_RECEIPT_TOKEN_DIGEST_PATTERN = /^[0-9a-f]{64}$/;
const CALLBACK_TOKEN_PATTERN = /^[A-Za-z0-9_-]{32,128}$/;
export interface WorkerExecutionCompletionReceiptAuthentication {
callbackSequence: number;
tokenDigest: string;
}
export function createWorkerExecutionCompletionReceiptAuthentication(
callback: ExecutionContext['completionCallback'],
): WorkerExecutionCompletionReceiptAuthentication | undefined {
if (callback === undefined) return undefined;
if (!CALLBACK_TOKEN_PATTERN.test(callback.token)) {
throw new TypeError('Worker completion callback token is invalid');
}
if (
!Number.isSafeInteger(callback.callbackSequence) ||
callback.callbackSequence < 1
) {
throw new TypeError(
'Worker completion callback sequence must be a positive safe integer',
);
}
return {
callbackSequence: callback.callbackSequence,
tokenDigest: createHash('sha256')
.update(callback.token, 'utf8')
.digest('hex'),
};
}
export function matchesWorkerExecutionCompletionReceiptAuthentication(
receipt: CompletionReceipt,
expected: WorkerExecutionCompletionReceiptAuthentication,
): boolean {
if (
receipt.callbackSequence !== expected.callbackSequence ||
!WORKER_COMPLETION_RECEIPT_TOKEN_DIGEST_PATTERN.test(expected.tokenDigest)
) {
return false;
}
const actual = createHash('sha256').update(receipt.token, 'utf8').digest();
const expectedBytes = Buffer.from(expected.tokenDigest, 'hex');
return timingSafeEqual(actual, expectedBytes);
}
back/runtime/domain/workerExecutionOffer.ts
+450
View File
@@ -0,0 +1,450 @@
import type { ExecutionHandle } from './execution';
import { cloneExecutionSpec } from './executionSpec';
import {
assertRunDispatchCandidate,
type RunDispatchCandidate,
} from './runDispatchCandidate';
import {
assertRunDispatchLeaseRecord,
type RunDispatchLeaseRecord,
} from './runDispatchLease';
import {
assertRunDispatchOfferId,
createExecutionSpecDigest,
createRunDispatchOfferId,
type ClaimedExecutionOffer,
} from './runDispatchOffer';
import {
MAX_EXECUTOR_HANDLE_LENGTH,
MAX_LOG_ARTIFACT_ID_LENGTH,
} from './runStateMachine';
import { WORKER_COMPLETION_RECEIPT_TOKEN_DIGEST_PATTERN } from './workerExecutionCompletionReceiptAuthentication';
export const WORKER_EXECUTION_OFFER_JOURNAL_STATES = [
'accepted',
'starting_acknowledged',
'launching',
'started',
'running_acknowledged',
'start_failed',
'start_failure_acknowledged',
'completion_acknowledged',
'recovery_required',
] as const;
export type WorkerExecutionOfferJournalState =
(typeof WORKER_EXECUTION_OFFER_JOURNAL_STATES)[number];
export const MAX_WORKER_EXECUTION_OFFER_RECORD_BYTES = 192 * 1024;
export const MAX_WORKER_EXECUTION_OFFER_JOURNAL_ENTRIES = 1024;
export const MAX_WORKER_EXECUTION_OFFER_JOURNAL_PAGE_SIZE = 64;
export interface WorkerExecutionOfferJournalRecord {
schemaVersion: 1;
revision: number;
state: WorkerExecutionOfferJournalState;
offer: ClaimedExecutionOffer;
acceptedAtMs: number;
updatedAtMs: number;
executorHandle?: string;
executorStartedAtMs?: number;
logArtifactId?: string;
completionReceiptCallbackSequence?: number;
completionReceiptTokenDigest?: string;
completionAcknowledgedAtMs?: number;
recoveryReason?:
| 'launch_outcome_unknown'
| 'control_plane_already_running'
| 'control_plane_terminal'
| 'lease_lost_local_execution_stopped'
| 'lease_lost_local_execution_unverified';
}
export class InvalidWorkerExecutionOfferError extends TypeError {
constructor(message: string) {
super(`Worker execution offer is invalid: ${message}`);
this.name = 'InvalidWorkerExecutionOfferError';
}
}
export class WorkerExecutionOfferConflictError extends Error {
constructor(readonly offerId: string) {
super(`Worker execution offer ${offerId} conflicts with durable state`);
this.name = 'WorkerExecutionOfferConflictError';
}
}
function invalid(message: string): never {
throw new InvalidWorkerExecutionOfferError(message);
}
function safeInteger(value: unknown, name: string): number {
if (!Number.isSafeInteger(value) || (value as number) < 0) {
invalid(`${name} must be a non-negative safe integer`);
}
return value as number;
}
function boundedString(value: unknown, name: string, maximum: number): string {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > maximum ||
/[\u0000-\u001f\u007f]/.test(value)
) {
invalid(`${name} is invalid`);
}
return value;
}
function sameCandidate(
left: RunDispatchCandidate,
right: RunDispatchCandidate,
): boolean {
return (
left.runId === right.runId &&
left.attemptId === right.attemptId &&
left.projectId === right.projectId &&
left.taskId === right.taskId &&
left.taskRevision === right.taskRevision &&
left.executorType === right.executorType &&
left.priority === right.priority &&
left.queuedAtMs === right.queuedAtMs &&
left.attemptCreatedAtMs === right.attemptCreatedAtMs
);
}
function sameLeaseAuthority(
left: RunDispatchLeaseRecord,
right: RunDispatchLeaseRecord,
): boolean {
return (
left.attemptId === right.attemptId &&
left.runId === right.runId &&
left.workerId === right.workerId &&
left.workerSessionId === right.workerSessionId &&
left.workerGeneration === right.workerGeneration &&
left.leaseGeneration === right.leaseGeneration &&
left.leaseToken === right.leaseToken
);
}
export function assertClaimedExecutionOffer(
offer: ClaimedExecutionOffer,
): void {
if (!offer || typeof offer !== 'object' || Array.isArray(offer)) {
invalid('offer must be an object');
}
assertRunDispatchOfferId(offer.offerId);
assertRunDispatchCandidate(offer.candidate);
assertRunDispatchLeaseRecord(offer.lease);
if (offer.lease.status !== 'leased') invalid('lease must be active');
cloneExecutionSpec(offer.executionSpec);
if (
offer.deliveryKind !== 'new_claim' &&
offer.deliveryKind !== 'lease_recovery'
) {
invalid('deliveryKind is invalid');
}
if (
!offer.worker ||
typeof offer.worker !== 'object' ||
offer.worker.id !== offer.lease.workerId ||
offer.worker.sessionId !== offer.lease.workerSessionId ||
offer.worker.generation !== offer.lease.workerGeneration
) {
invalid('Worker target does not match the lease fence');
}
if (
offer.candidate.runId !== offer.lease.runId ||
offer.candidate.attemptId !== offer.lease.attemptId ||
offer.executionSpec.runId !== offer.candidate.runId ||
offer.executionSpec.attemptId !== offer.candidate.attemptId ||
offer.executionSpec.projectId !== offer.candidate.projectId ||
offer.executionSpec.taskId !== offer.candidate.taskId ||
offer.executionSpec.taskRevision !== offer.candidate.taskRevision
) {
invalid('Run, Attempt, Project, Task or revision identity drifted');
}
if (createRunDispatchOfferId(offer.lease) !== offer.offerId) {
invalid('offerId does not match the lease authority');
}
if (
createExecutionSpecDigest(offer.executionSpec) !== offer.executionSpecDigest
) {
invalid('ExecutionSpec digest does not match the payload');
}
if (
offer.placementScore !== undefined &&
(!Number.isFinite(offer.placementScore) || offer.placementScore < 0)
) {
invalid('placementScore is invalid');
}
}
export function cloneClaimedExecutionOffer(
offer: ClaimedExecutionOffer,
): ClaimedExecutionOffer {
assertClaimedExecutionOffer(offer);
return {
offerId: offer.offerId,
executionSpecDigest: offer.executionSpecDigest,
deliveryKind: offer.deliveryKind,
candidate: { ...offer.candidate },
worker: { ...offer.worker },
lease: { ...offer.lease },
executionSpec: cloneExecutionSpec(offer.executionSpec),
...(offer.placementScore === undefined
? {}
: { placementScore: offer.placementScore }),
};
}
export function assertSameWorkerExecutionOffer(
persisted: ClaimedExecutionOffer,
delivered: ClaimedExecutionOffer,
): void {
assertClaimedExecutionOffer(persisted);
assertClaimedExecutionOffer(delivered);
if (
persisted.offerId !== delivered.offerId ||
persisted.executionSpecDigest !== delivered.executionSpecDigest ||
!sameCandidate(persisted.candidate, delivered.candidate) ||
!sameLeaseAuthority(persisted.lease, delivered.lease) ||
persisted.worker.id !== delivered.worker.id ||
persisted.worker.sessionId !== delivered.worker.sessionId ||
persisted.worker.generation !== delivered.worker.generation
) {
throw new WorkerExecutionOfferConflictError(delivered.offerId);
}
}
export function mergeWorkerExecutionOffer(
persisted: ClaimedExecutionOffer,
delivered: ClaimedExecutionOffer,
): ClaimedExecutionOffer {
assertSameWorkerExecutionOffer(persisted, delivered);
const selectedLease =
delivered.lease.version > persisted.lease.version
? delivered.lease
: persisted.lease;
return cloneClaimedExecutionOffer({
...persisted,
deliveryKind: delivered.deliveryKind,
lease: selectedLease,
});
}
export function createWorkerExecutionOfferJournalRecord(
offer: ClaimedExecutionOffer,
acceptedAtMs: number,
): WorkerExecutionOfferJournalRecord {
safeInteger(acceptedAtMs, 'acceptedAtMs');
return {
schemaVersion: 1,
revision: 0,
state: 'accepted',
offer: cloneClaimedExecutionOffer(offer),
acceptedAtMs,
updatedAtMs: acceptedAtMs,
};
}
export function cloneWorkerExecutionOfferJournalRecord(
record: WorkerExecutionOfferJournalRecord,
): WorkerExecutionOfferJournalRecord {
assertWorkerExecutionOfferJournalRecord(record);
return {
schemaVersion: 1,
revision: record.revision,
state: record.state,
offer: cloneClaimedExecutionOffer(record.offer),
acceptedAtMs: record.acceptedAtMs,
updatedAtMs: record.updatedAtMs,
...(record.executorHandle === undefined
? {}
: { executorHandle: record.executorHandle }),
...(record.executorStartedAtMs === undefined
? {}
: { executorStartedAtMs: record.executorStartedAtMs }),
...(record.logArtifactId === undefined
? {}
: { logArtifactId: record.logArtifactId }),
...(record.completionReceiptCallbackSequence === undefined
? {}
: {
completionReceiptCallbackSequence:
record.completionReceiptCallbackSequence,
}),
...(record.completionReceiptTokenDigest === undefined
? {}
: {
completionReceiptTokenDigest: record.completionReceiptTokenDigest,
}),
...(record.completionAcknowledgedAtMs === undefined
? {}
: {
completionAcknowledgedAtMs: record.completionAcknowledgedAtMs,
}),
...(record.recoveryReason === undefined
? {}
: { recoveryReason: record.recoveryReason }),
};
}
export function assertWorkerExecutionOfferJournalRecord(
record: WorkerExecutionOfferJournalRecord,
): void {
if (!record || typeof record !== 'object' || Array.isArray(record)) {
invalid('journal record must be an object');
}
if (record.schemaVersion !== 1) invalid('schemaVersion is unsupported');
safeInteger(record.revision, 'revision');
safeInteger(record.acceptedAtMs, 'acceptedAtMs');
safeInteger(record.updatedAtMs, 'updatedAtMs');
if (record.updatedAtMs < record.acceptedAtMs) {
invalid('journal timestamps are inconsistent');
}
if (!WORKER_EXECUTION_OFFER_JOURNAL_STATES.includes(record.state)) {
invalid('journal state is invalid');
}
assertClaimedExecutionOffer(record.offer);
const hasExecutorMetadata =
record.executorHandle !== undefined ||
record.executorStartedAtMs !== undefined ||
record.logArtifactId !== undefined;
const requiresExecutorMetadata =
record.state === 'started' || record.state === 'running_acknowledged';
if (requiresExecutorMetadata || hasExecutorMetadata) {
boundedString(
record.executorHandle,
'executorHandle',
MAX_EXECUTOR_HANDLE_LENGTH,
);
safeInteger(record.executorStartedAtMs, 'executorStartedAtMs');
if (
!requiresExecutorMetadata &&
record.state !== 'recovery_required' &&
record.state !== 'completion_acknowledged'
) {
invalid('executor metadata is not allowed in this journal state');
}
}
if (record.logArtifactId !== undefined) {
boundedString(
record.logArtifactId,
'logArtifactId',
MAX_LOG_ARTIFACT_ID_LENGTH,
);
}
const hasCompletionCallbackSequence =
record.completionReceiptCallbackSequence !== undefined;
const hasCompletionTokenDigest =
record.completionReceiptTokenDigest !== undefined;
if (hasCompletionCallbackSequence !== hasCompletionTokenDigest) {
invalid('completion receipt authentication metadata must be complete');
}
if (hasCompletionCallbackSequence) {
if (
!Number.isSafeInteger(record.completionReceiptCallbackSequence) ||
record.completionReceiptCallbackSequence! < 1
) {
invalid('completionReceiptCallbackSequence must be positive');
}
if (
typeof record.completionReceiptTokenDigest !== 'string' ||
!WORKER_COMPLETION_RECEIPT_TOKEN_DIGEST_PATTERN.test(
record.completionReceiptTokenDigest,
)
) {
invalid('completionReceiptTokenDigest is invalid');
}
if (
record.state === 'accepted' ||
record.state === 'starting_acknowledged'
) {
invalid(
'completion receipt authentication is not allowed before launching',
);
}
}
if (record.state === 'completion_acknowledged') {
if (!hasCompletionCallbackSequence) {
invalid('completion acknowledgement requires authentication metadata');
}
safeInteger(
record.completionAcknowledgedAtMs,
'completionAcknowledgedAtMs',
);
if (record.completionAcknowledgedAtMs !== record.updatedAtMs) {
invalid('completion acknowledgement timestamp must match updatedAtMs');
}
} else if (record.completionAcknowledgedAtMs !== undefined) {
invalid(
'completionAcknowledgedAtMs is only allowed for completion_acknowledged',
);
}
if (record.state === 'recovery_required') {
if (
record.recoveryReason !== 'launch_outcome_unknown' &&
record.recoveryReason !== 'control_plane_already_running' &&
record.recoveryReason !== 'control_plane_terminal' &&
record.recoveryReason !== 'lease_lost_local_execution_stopped' &&
record.recoveryReason !== 'lease_lost_local_execution_unverified'
) {
invalid('recoveryReason is invalid');
}
} else if (record.recoveryReason !== undefined) {
invalid('recoveryReason is only allowed for recovery_required');
}
}
export function workerExecutionHandleMetadata(handle: ExecutionHandle): {
executorHandle: string;
executorStartedAtMs: number;
} {
const executorHandle = boundedString(
handle.durableHandle ?? handle.id,
'executorHandle',
MAX_EXECUTOR_HANDLE_LENGTH,
);
const executorStartedAtMs = safeInteger(handle.startedAtMs, 'startedAtMs');
return { executorHandle, executorStartedAtMs };
}
export function serializeWorkerExecutionOfferJournalRecord(
record: WorkerExecutionOfferJournalRecord,
): string {
const serialized = JSON.stringify(
cloneWorkerExecutionOfferJournalRecord(record),
);
if (
Buffer.byteLength(serialized, 'utf8') >
MAX_WORKER_EXECUTION_OFFER_RECORD_BYTES
) {
invalid('journal record exceeds the byte limit');
}
return serialized;
}
export function parseWorkerExecutionOfferJournalRecord(
value: Uint8Array | string,
): WorkerExecutionOfferJournalRecord {
const bytes =
typeof value === 'string' ? Buffer.from(value) : Buffer.from(value);
if (
bytes.length < 2 ||
bytes.length > MAX_WORKER_EXECUTION_OFFER_RECORD_BYTES
) {
invalid('serialized journal record size is outside the allowed range');
}
let parsed: unknown;
try {
parsed = JSON.parse(bytes.toString('utf8'));
} catch {
return invalid('serialized journal record is not valid JSON');
}
return cloneWorkerExecutionOfferJournalRecord(
parsed as WorkerExecutionOfferJournalRecord,
);
}
back/runtime/domain/workerPlacement.ts
+430
View File
@@ -0,0 +1,430 @@
import { satisfies, valid, validRange } from 'semver';
import {
isWorkerLeaseActive,
type WorkerCapabilities,
type WorkerRecord,
} from './worker';
export const MAX_PLACEMENT_VALUES = 16;
export const MAX_PLACEMENT_PREFERENCES = 16;
export const MAX_PLACEMENT_CANDIDATES = 64;
export interface WorkerRuntimeRequirement {
name: string;
versionRange?: string;
}
export interface WorkerPlacementRequired {
architectures?: readonly string[];
operatingSystems?: readonly string[];
executors?: readonly string[];
runtimes?: readonly WorkerRuntimeRequirement[];
labels?: Readonly<Record<string, string>>;
minMemoryBytes?: number;
minDiskBytes?: number;
gpuVendor?: string;
features?: readonly string[];
}
export interface WorkerPlacementPreference {
labels: Readonly<Record<string, string>>;
weight: number;
}
export interface WorkerPlacementSpec {
required?: WorkerPlacementRequired;
preferred?: readonly WorkerPlacementPreference[];
}
export type WorkerPlacementMismatch =
| 'worker_unavailable'
| 'architecture'
| 'operating_system'
| 'executor'
| 'runtime'
| 'label'
| 'memory'
| 'disk'
| 'gpu'
| 'feature';
export interface WorkerPlacementDecision {
matches: boolean;
score: number;
mismatches: readonly WorkerPlacementMismatch[];
}
export interface WorkerPlacementCandidate {
worker: WorkerRecord;
score: number;
}
function invalid(message: string): never {
throw new TypeError(`Worker PlacementSpec is invalid: ${message}`);
}
function object(value: unknown, name: string): Record<string, unknown> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
invalid(`${name} must be an object`);
}
return value as Record<string, unknown>;
}
function assertKeys(
value: Record<string, unknown>,
name: string,
keys: readonly string[],
): void {
if (Object.keys(value).some((key) => !keys.includes(key))) {
invalid(`${name} contains an unknown field`);
}
}
function stringValue(value: unknown, name: string, maximum = 128): string {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > maximum ||
value.includes('\0') ||
/[\u0000-\u001f\u007f]/.test(value)
) {
invalid(`${name} is invalid`);
}
return value;
}
function stringList(value: unknown, name: string): string[] | undefined {
if (value === undefined) return undefined;
if (!Array.isArray(value) || value.length > MAX_PLACEMENT_VALUES) {
invalid(`${name} must contain at most ${MAX_PLACEMENT_VALUES} values`);
}
const values = value.map((item, index) =>
stringValue(item, `${name}[${index}]`, 64),
);
if (new Set(values).size !== values.length) {
invalid(`${name} must not contain duplicates`);
}
return values.sort();
}
function labels(
value: unknown,
name: string,
): Record<string, string> | undefined {
if (value === undefined) return undefined;
const source = object(value, name);
const entries = Object.entries(source);
if (entries.length > MAX_PLACEMENT_VALUES) {
invalid(`${name} must contain at most ${MAX_PLACEMENT_VALUES} labels`);
}
return Object.fromEntries(
entries
.map(([key, candidate]) => [
stringValue(key, `${name} key`, 128),
stringValue(candidate, `${name}.${key}`, 256),
])
.sort(([left], [right]) => left.localeCompare(right)),
);
}
function positiveInteger(value: unknown, name: string): number | undefined {
if (value === undefined) return undefined;
if (!Number.isSafeInteger(value) || (value as number) < 1) {
invalid(`${name} must be a positive safe integer`);
}
return value as number;
}
function runtimes(value: unknown): WorkerRuntimeRequirement[] | undefined {
if (value === undefined) return undefined;
if (!Array.isArray(value) || value.length > MAX_PLACEMENT_VALUES) {
invalid(
`required.runtimes must contain at most ${MAX_PLACEMENT_VALUES} values`,
);
}
const requirements = value.map((candidate, index) => {
const requirement = object(candidate, `required.runtimes[${index}]`);
assertKeys(requirement, `required.runtimes[${index}]`, [
'name',
'versionRange',
]);
const name = stringValue(
requirement.name,
`required.runtimes[${index}].name`,
64,
);
if (requirement.versionRange === undefined) return { name };
const versionRange = stringValue(
requirement.versionRange,
`required.runtimes[${index}].versionRange`,
128,
);
if (!validRange(versionRange)) {
invalid(`required.runtimes[${index}].versionRange is not semver`);
}
return { name, versionRange };
});
if (
new Set(requirements.map((item) => item.name)).size !== requirements.length
) {
invalid('required.runtimes must not repeat a runtime name');
}
return requirements.sort((left, right) =>
left.name.localeCompare(right.name),
);
}
function normalizeRequired(
value: unknown,
): WorkerPlacementRequired | undefined {
if (value === undefined) return undefined;
const required = object(value, 'required');
assertKeys(required, 'required', [
'architectures',
'operatingSystems',
'executors',
'runtimes',
'labels',
'minMemoryBytes',
'minDiskBytes',
'gpuVendor',
'features',
]);
const architectures = stringList(
required.architectures,
'required.architectures',
);
const operatingSystems = stringList(
required.operatingSystems,
'required.operatingSystems',
);
const executors = stringList(required.executors, 'required.executors');
const runtimeRequirements = runtimes(required.runtimes);
const requiredLabels = labels(required.labels, 'required.labels');
const minMemoryBytes = positiveInteger(
required.minMemoryBytes,
'required.minMemoryBytes',
);
const minDiskBytes = positiveInteger(
required.minDiskBytes,
'required.minDiskBytes',
);
const gpuVendor =
required.gpuVendor === undefined
? undefined
: stringValue(required.gpuVendor, 'required.gpuVendor', 64);
const features = stringList(required.features, 'required.features');
return {
...(architectures === undefined ? {} : { architectures }),
...(operatingSystems === undefined ? {} : { operatingSystems }),
...(executors === undefined ? {} : { executors }),
...(runtimeRequirements === undefined
? {}
: { runtimes: runtimeRequirements }),
...(requiredLabels === undefined ? {} : { labels: requiredLabels }),
...(minMemoryBytes === undefined ? {} : { minMemoryBytes }),
...(minDiskBytes === undefined ? {} : { minDiskBytes }),
...(gpuVendor === undefined ? {} : { gpuVendor }),
...(features === undefined ? {} : { features }),
};
}
function normalizePreferred(
value: unknown,
): WorkerPlacementPreference[] | undefined {
if (value === undefined) return undefined;
if (!Array.isArray(value) || value.length > MAX_PLACEMENT_PREFERENCES) {
invalid(
`preferred must contain at most ${MAX_PLACEMENT_PREFERENCES} values`,
);
}
return value.map((candidate, index) => {
const preference = object(candidate, `preferred[${index}]`);
assertKeys(preference, `preferred[${index}]`, ['labels', 'weight']);
const preferredLabels = labels(
preference.labels,
`preferred[${index}].labels`,
);
if (!preferredLabels || Object.keys(preferredLabels).length === 0) {
invalid(`preferred[${index}].labels must not be empty`);
}
if (
!Number.isSafeInteger(preference.weight) ||
(preference.weight as number) < 1 ||
(preference.weight as number) > 100
) {
invalid(`preferred[${index}].weight must be between 1 and 100`);
}
return {
labels: preferredLabels,
weight: preference.weight as number,
};
});
}
export function normalizeWorkerPlacementSpec(
value: unknown,
): WorkerPlacementSpec {
const placement = object(value, 'placement');
assertKeys(placement, 'placement', ['required', 'preferred']);
const required = normalizeRequired(placement.required);
const preferred = normalizePreferred(placement.preferred);
return {
...(required === undefined ? {} : { required }),
...(preferred === undefined ? {} : { preferred }),
};
}
function hasLabels(
capabilities: WorkerCapabilities,
expected: Readonly<Record<string, string>>,
): boolean {
return Object.entries(expected).every(
([key, value]) => capabilities.labels[key] === value,
);
}
function hasRuntime(
capabilities: WorkerCapabilities,
requirement: WorkerRuntimeRequirement,
): boolean {
return capabilities.runtimes.some((runtime) => {
if (runtime.name !== requirement.name) return false;
if (!requirement.versionRange) return true;
return (
valid(runtime.version) !== null &&
satisfies(runtime.version, requirement.versionRange, {
includePrerelease: true,
})
);
});
}
function matchNormalizedWorkerPlacement(
worker: WorkerRecord,
placement: WorkerPlacementSpec,
observedAtMs: number,
): WorkerPlacementDecision {
const required = placement.required ?? {};
const capabilities = worker.capabilities;
const mismatches: WorkerPlacementMismatch[] = [];
if (
worker.status !== 'online' ||
worker.availableSlots < 1 ||
!isWorkerLeaseActive(worker, observedAtMs)
) {
mismatches.push('worker_unavailable');
}
if (
required.architectures?.length &&
!required.architectures.includes(capabilities.architecture)
) {
mismatches.push('architecture');
}
if (
required.operatingSystems?.length &&
!required.operatingSystems.includes(capabilities.operatingSystem)
) {
mismatches.push('operating_system');
}
if (
required.executors?.some(
(executor) => !capabilities.executors.includes(executor),
)
) {
mismatches.push('executor');
}
if (
required.runtimes?.some((runtime) => !hasRuntime(capabilities, runtime))
) {
mismatches.push('runtime');
}
if (required.labels && !hasLabels(capabilities, required.labels)) {
mismatches.push('label');
}
if (
required.minMemoryBytes !== undefined &&
(capabilities.capacity.memoryBytes ?? 0) < required.minMemoryBytes
) {
mismatches.push('memory');
}
if (
required.minDiskBytes !== undefined &&
(capabilities.capacity.diskBytes ?? 0) < required.minDiskBytes
) {
mismatches.push('disk');
}
if (
required.gpuVendor !== undefined &&
!capabilities.capacity.gpu?.some((gpu) => gpu.vendor === required.gpuVendor)
) {
mismatches.push('gpu');
}
if (
required.features?.some(
(feature) => !capabilities.features.includes(feature),
)
) {
mismatches.push('feature');
}
const score = (placement.preferred ?? []).reduce(
(total, preference) =>
total +
(hasLabels(capabilities, preference.labels) ? preference.weight : 0),
0,
);
return { matches: mismatches.length === 0, score, mismatches };
}
export function matchesWorkerPlacement(
worker: WorkerRecord,
placementValue: unknown,
observedAtMs: number,
): WorkerPlacementDecision {
return matchNormalizedWorkerPlacement(
worker,
normalizeWorkerPlacementSpec(placementValue),
observedAtMs,
);
}
export function selectWorkerCandidates(
workers: readonly WorkerRecord[],
placement: unknown,
observedAtMs: number,
limit = 16,
): WorkerPlacementCandidate[] {
if (workers.length > MAX_PLACEMENT_CANDIDATES) {
throw new RangeError(
`workers must contain at most ${MAX_PLACEMENT_CANDIDATES} candidates`,
);
}
if (
!Number.isSafeInteger(limit) ||
limit < 1 ||
limit > MAX_PLACEMENT_CANDIDATES
) {
throw new RangeError(
`limit must be between 1 and ${MAX_PLACEMENT_CANDIDATES}`,
);
}
const normalizedPlacement = normalizeWorkerPlacementSpec(placement);
return workers
.map((worker) => ({
worker,
decision: matchNormalizedWorkerPlacement(
worker,
normalizedPlacement,
observedAtMs,
),
}))
.filter((candidate) => candidate.decision.matches)
.sort(
(left, right) =>
right.decision.score - left.decision.score ||
right.worker.availableSlots - left.worker.availableSlots ||
left.worker.id.localeCompare(right.worker.id),
)
.slice(0, limit)
.map(({ worker, decision }) => ({ worker, score: decision.score }));
}