mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-22 19:29:13 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,347 @@
|
||||
import {
|
||||
ApprovalUnavailableError,
|
||||
normalizeApprovedActionBinding,
|
||||
normalizeApprovalRequestRecord,
|
||||
type ApprovalDecision,
|
||||
type ApprovalRequestRecord,
|
||||
type ApprovalRequestRepository,
|
||||
type ApprovedActionBinding,
|
||||
type DecideApprovalRequestResult,
|
||||
} from './approvedAction';
|
||||
import { ProjectPolicyEngine } from '../security/project-policy/projectPolicy';
|
||||
import {
|
||||
normalizeSecurityPrincipal,
|
||||
type SecurityPolicyFence,
|
||||
type SecurityPrincipal,
|
||||
} from '../security/security';
|
||||
import type { SecurityAuditRecord } from '../security/audit/securityAudit';
|
||||
|
||||
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const REASON_PATTERN = /^[a-z][a-z0-9_]{0,63}$/;
|
||||
const UUID_V4_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
||||
const STRONG_HUMAN_ASSURANCES = new Set([
|
||||
'multi_factor',
|
||||
'hardware',
|
||||
'local_console',
|
||||
]);
|
||||
|
||||
export interface ApprovalDecisionRequest {
|
||||
readonly projectId: string;
|
||||
readonly approvalRequestId: string;
|
||||
readonly expectedVersion: number;
|
||||
readonly expectedAction: Readonly<ApprovedActionBinding>;
|
||||
readonly decisionId: string;
|
||||
readonly decision: ApprovalDecision;
|
||||
readonly reasonCode: string;
|
||||
readonly auditEventId: string;
|
||||
readonly requestId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
}
|
||||
|
||||
export interface ApprovalDecisionService {
|
||||
decide(
|
||||
request: ApprovalDecisionRequest,
|
||||
): Promise<Readonly<DecideApprovalRequestResult>>;
|
||||
}
|
||||
|
||||
export interface ApprovalDecisionServiceOptions {
|
||||
readonly approvals: Pick<
|
||||
ApprovalRequestRepository,
|
||||
'findById' | 'decide'
|
||||
>;
|
||||
readonly policy: Pick<ProjectPolicyEngine, 'authorize'>;
|
||||
readonly confirmAuthorization?: () => void | Promise<void>;
|
||||
readonly now?: () => number;
|
||||
}
|
||||
|
||||
export class ApprovalDecisionRequestError extends TypeError {
|
||||
readonly code = 'APPROVAL_DECISION_REQUEST_INVALID';
|
||||
|
||||
constructor(message: string) {
|
||||
super(`Approval decision request is invalid: ${message}`);
|
||||
this.name = 'ApprovalDecisionRequestError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ApprovalDecisionAuthorizationError extends Error {
|
||||
readonly code = 'APPROVAL_DECISION_AUTHORIZATION_REJECTED';
|
||||
|
||||
constructor() {
|
||||
super('Approval decision authorization was rejected');
|
||||
this.name = 'ApprovalDecisionAuthorizationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ApprovalDecisionTargetUnavailableError extends Error {
|
||||
readonly code = 'APPROVAL_DECISION_TARGET_UNAVAILABLE';
|
||||
|
||||
constructor() {
|
||||
super('Approval decision target is unavailable');
|
||||
this.name = 'ApprovalDecisionTargetUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ApprovalDecisionBindingConflictError extends Error {
|
||||
readonly code = 'APPROVAL_DECISION_BINDING_CONFLICT';
|
||||
|
||||
constructor() {
|
||||
super('Approval decision does not match the reviewed action binding');
|
||||
this.name = 'ApprovalDecisionBindingConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ApprovalDecisionUnavailableError extends Error {
|
||||
readonly code = 'APPROVAL_DECISION_UNAVAILABLE';
|
||||
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('Approval decision authority is unavailable', options);
|
||||
this.name = 'ApprovalDecisionUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function exactObject(
|
||||
value: unknown,
|
||||
expected: readonly string[],
|
||||
label: string,
|
||||
): asserts value is Record<string, unknown> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
Object.keys(value).sort().join('\0') !== [...expected].sort().join('\0')
|
||||
) {
|
||||
throw new ApprovalDecisionRequestError(`${label} shape is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
function identifier(value: unknown, label: string): string {
|
||||
if (typeof value !== 'string' || !IDENTIFIER_PATTERN.test(value)) {
|
||||
throw new ApprovalDecisionRequestError(`${label} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function action(
|
||||
value: Readonly<ApprovedActionBinding>,
|
||||
): Readonly<ApprovedActionBinding> {
|
||||
try {
|
||||
return normalizeApprovedActionBinding(value);
|
||||
} catch {
|
||||
throw new ApprovalDecisionRequestError('expected action is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function sameAction(
|
||||
left: Readonly<ApprovedActionBinding>,
|
||||
right: Readonly<ApprovedActionBinding>,
|
||||
): boolean {
|
||||
return (
|
||||
left.permission === right.permission &&
|
||||
left.actionType === right.actionType &&
|
||||
left.actionRef === right.actionRef &&
|
||||
left.actionDigest === right.actionDigest &&
|
||||
left.previewDigest === right.previewDigest
|
||||
);
|
||||
}
|
||||
|
||||
function sameSubject(
|
||||
left: Readonly<SecurityPrincipal>['subject'],
|
||||
right: Readonly<SecurityPrincipal>['subject'],
|
||||
): boolean {
|
||||
return left.type === right.type && left.id === right.id;
|
||||
}
|
||||
|
||||
function observedTime(now: () => number): number {
|
||||
const value = now();
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new ApprovalDecisionUnavailableError();
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function audit(
|
||||
request: Readonly<ApprovalDecisionRequest>,
|
||||
principal: Readonly<SecurityPrincipal>,
|
||||
fence: Readonly<SecurityPolicyFence>,
|
||||
occurredAtMs: number,
|
||||
): Readonly<SecurityAuditRecord> {
|
||||
return Object.freeze({
|
||||
eventId: request.auditEventId,
|
||||
requestId: request.requestId,
|
||||
operationId: 'approval.decide',
|
||||
projectId: request.projectId,
|
||||
subject: principal.subject,
|
||||
authenticationId: principal.authenticationId,
|
||||
outcome: 'allowed',
|
||||
reasons: Object.freeze(['human_approval_decision']),
|
||||
fence,
|
||||
occurredAtMs,
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeRequest(
|
||||
value: ApprovalDecisionRequest,
|
||||
): Readonly<ApprovalDecisionRequest> {
|
||||
exactObject(
|
||||
value,
|
||||
[
|
||||
'projectId',
|
||||
'approvalRequestId',
|
||||
'expectedVersion',
|
||||
'expectedAction',
|
||||
'decisionId',
|
||||
'decision',
|
||||
'reasonCode',
|
||||
'auditEventId',
|
||||
'requestId',
|
||||
'principal',
|
||||
],
|
||||
'request',
|
||||
);
|
||||
const projectId = identifier(value.projectId, 'projectId');
|
||||
const approvalRequestId = identifier(
|
||||
value.approvalRequestId,
|
||||
'approvalRequestId',
|
||||
);
|
||||
const requestId = identifier(value.requestId, 'requestId');
|
||||
const decisionId = identifier(value.decisionId, 'decisionId');
|
||||
if (value.expectedVersion !== 1) {
|
||||
throw new ApprovalDecisionRequestError('expectedVersion must be 1');
|
||||
}
|
||||
if (value.decision !== 'approved' && value.decision !== 'rejected') {
|
||||
throw new ApprovalDecisionRequestError('decision is invalid');
|
||||
}
|
||||
if (
|
||||
typeof value.reasonCode !== 'string' ||
|
||||
!REASON_PATTERN.test(value.reasonCode)
|
||||
) {
|
||||
throw new ApprovalDecisionRequestError('reasonCode is invalid');
|
||||
}
|
||||
if (
|
||||
typeof value.auditEventId !== 'string' ||
|
||||
!UUID_V4_PATTERN.test(value.auditEventId)
|
||||
) {
|
||||
throw new ApprovalDecisionRequestError('auditEventId is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
projectId,
|
||||
approvalRequestId,
|
||||
expectedVersion: 1,
|
||||
expectedAction: action(value.expectedAction),
|
||||
decisionId,
|
||||
decision: value.decision,
|
||||
reasonCode: value.reasonCode,
|
||||
auditEventId: value.auditEventId,
|
||||
requestId,
|
||||
principal: value.principal,
|
||||
});
|
||||
}
|
||||
|
||||
export function createApprovalDecisionService(
|
||||
candidateOptions: ApprovalDecisionServiceOptions,
|
||||
): Readonly<ApprovalDecisionService> {
|
||||
exactObject(
|
||||
candidateOptions,
|
||||
[
|
||||
'approvals',
|
||||
'policy',
|
||||
...(candidateOptions?.confirmAuthorization === undefined
|
||||
? []
|
||||
: ['confirmAuthorization']),
|
||||
...(candidateOptions?.now === undefined ? [] : ['now']),
|
||||
],
|
||||
'options',
|
||||
);
|
||||
if (
|
||||
typeof candidateOptions.approvals?.findById !== 'function' ||
|
||||
typeof candidateOptions.approvals?.decide !== 'function' ||
|
||||
typeof candidateOptions.policy?.authorize !== 'function' ||
|
||||
(candidateOptions.confirmAuthorization !== undefined &&
|
||||
typeof candidateOptions.confirmAuthorization !== 'function') ||
|
||||
(candidateOptions.now !== undefined && typeof candidateOptions.now !== 'function')
|
||||
) {
|
||||
throw new ApprovalDecisionRequestError('options are invalid');
|
||||
}
|
||||
const now = candidateOptions.now ?? Date.now;
|
||||
return Object.freeze({
|
||||
async decide(requestValue: ApprovalDecisionRequest) {
|
||||
const request = normalizeRequest(requestValue);
|
||||
const decidedAtMs = observedTime(now);
|
||||
let principal: Readonly<SecurityPrincipal>;
|
||||
try {
|
||||
principal = normalizeSecurityPrincipal(request.principal, decidedAtMs);
|
||||
} catch {
|
||||
throw new ApprovalDecisionAuthorizationError();
|
||||
}
|
||||
if (
|
||||
principal.subject.type !== 'user' ||
|
||||
!STRONG_HUMAN_ASSURANCES.has(principal.assurance)
|
||||
) {
|
||||
throw new ApprovalDecisionAuthorizationError();
|
||||
}
|
||||
let authorization;
|
||||
try {
|
||||
authorization = await candidateOptions.policy.authorize(
|
||||
principal,
|
||||
request.projectId,
|
||||
'approval.decide',
|
||||
);
|
||||
} catch (error) {
|
||||
throw new ApprovalDecisionUnavailableError({
|
||||
cause: error instanceof Error ? error : undefined,
|
||||
});
|
||||
}
|
||||
if (authorization.effect !== 'allow' || authorization.fence === null) {
|
||||
throw new ApprovalDecisionAuthorizationError();
|
||||
}
|
||||
let current: Readonly<ApprovalRequestRecord> | null;
|
||||
try {
|
||||
const found = await candidateOptions.approvals.findById(
|
||||
request.approvalRequestId,
|
||||
);
|
||||
current = found ? normalizeApprovalRequestRecord(found) : null;
|
||||
} catch (error) {
|
||||
throw new ApprovalDecisionUnavailableError({
|
||||
cause: error instanceof Error ? error : undefined,
|
||||
});
|
||||
}
|
||||
if (!current || current.projectId !== request.projectId) {
|
||||
throw new ApprovalDecisionTargetUnavailableError();
|
||||
}
|
||||
if (!sameAction(current.action, request.expectedAction)) {
|
||||
throw new ApprovalDecisionBindingConflictError();
|
||||
}
|
||||
if (
|
||||
current.version === 2 &&
|
||||
current.decisionId === request.decisionId &&
|
||||
current.decision === request.decision &&
|
||||
current.decisionReasonCode === request.reasonCode &&
|
||||
current.decidedBy !== null &&
|
||||
sameSubject(current.decidedBy, principal.subject)
|
||||
) {
|
||||
await candidateOptions.confirmAuthorization?.();
|
||||
return Object.freeze({ status: 'existing' as const, request: current });
|
||||
}
|
||||
await candidateOptions.confirmAuthorization?.();
|
||||
try {
|
||||
return await candidateOptions.approvals.decide({
|
||||
requestId: request.approvalRequestId,
|
||||
expectedVersion: request.expectedVersion,
|
||||
decisionId: request.decisionId,
|
||||
decision: request.decision,
|
||||
reasonCode: request.reasonCode,
|
||||
principal,
|
||||
decidedAtMs,
|
||||
authorizationFence: authorization.fence,
|
||||
audit: audit(request, principal, authorization.fence, decidedAtMs),
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof ApprovalUnavailableError) {
|
||||
throw new ApprovalDecisionUnavailableError({ cause: error });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
import type { ApprovalRequestRecord } from './approvedAction';
|
||||
|
||||
export const MAX_APPROVAL_REQUEST_PAGE_SIZE = 64;
|
||||
export const MAX_APPROVAL_DETAIL_PREVIEW_BYTES = 8 * 1024;
|
||||
|
||||
export interface ApprovalRequestCursor {
|
||||
readonly updatedAtMs: number;
|
||||
readonly requestId: string;
|
||||
}
|
||||
|
||||
export interface ApprovalRequestPage {
|
||||
readonly requests: readonly Readonly<ApprovalRequestRecord>[];
|
||||
readonly truncated: boolean;
|
||||
readonly next?: Readonly<ApprovalRequestCursor>;
|
||||
}
|
||||
|
||||
export interface ApprovalRequestSource {
|
||||
listApprovalRequests(options: {
|
||||
readonly projectId: string;
|
||||
readonly limit: number;
|
||||
readonly after?: Readonly<ApprovalRequestCursor>;
|
||||
}): Promise<Readonly<ApprovalRequestPage>>;
|
||||
}
|
||||
|
||||
export interface ApprovalRequestDetail {
|
||||
readonly request: Readonly<ApprovalRequestRecord>;
|
||||
readonly preview: Readonly<ApprovalDetailPreview> | null;
|
||||
}
|
||||
|
||||
export interface ApprovalDetailPreviewField {
|
||||
readonly kind: 'count' | 'identifier' | 'redacted' | 'text';
|
||||
readonly label: string;
|
||||
readonly value: string | null;
|
||||
}
|
||||
|
||||
export interface ApprovalDetailPreview {
|
||||
readonly title: string;
|
||||
readonly summary: string;
|
||||
readonly fields: readonly Readonly<ApprovalDetailPreviewField>[];
|
||||
readonly warnings: readonly string[];
|
||||
}
|
||||
|
||||
export interface ApprovalRequestDetailSource {
|
||||
getApprovalRequestDetail(options: {
|
||||
readonly projectId: string;
|
||||
readonly requestId: string;
|
||||
}): Promise<Readonly<ApprovalRequestDetail> | null>;
|
||||
}
|
||||
|
||||
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/;
|
||||
const WARNING_PATTERN = /^[a-z][a-z0-9_]{0,63}$/;
|
||||
const PREVIEW_FIELD_KINDS = ['count', 'identifier', 'redacted', 'text'] as const;
|
||||
|
||||
export class InvalidApprovalDiscoveryValueError extends TypeError {
|
||||
constructor(message: string) {
|
||||
super(`Approval discovery value is invalid: ${message}`);
|
||||
this.name = 'InvalidApprovalDiscoveryValueError';
|
||||
}
|
||||
}
|
||||
|
||||
function identifier(value: unknown, name: string): string {
|
||||
if (typeof value !== 'string' || !IDENTIFIER_PATTERN.test(value)) {
|
||||
throw new InvalidApprovalDiscoveryValueError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function timestamp(value: unknown, name: string): number {
|
||||
if (!Number.isSafeInteger(value) || Number(value) < 0) {
|
||||
throw new InvalidApprovalDiscoveryValueError(`${name} is invalid`);
|
||||
}
|
||||
return Number(value);
|
||||
}
|
||||
|
||||
export function assertApprovalRequestPageSize(value: number): void {
|
||||
if (
|
||||
!Number.isSafeInteger(value) ||
|
||||
value < 1 ||
|
||||
value > MAX_APPROVAL_REQUEST_PAGE_SIZE
|
||||
) {
|
||||
throw new InvalidApprovalDiscoveryValueError('page size is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
export function assertApprovalDiscoveryProjectId(value: string): void {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
value.length > 128 ||
|
||||
CONTROL_PATTERN.test(value)
|
||||
) {
|
||||
throw new InvalidApprovalDiscoveryValueError('projectId is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
export function assertApprovalDiscoveryRequestId(value: string): void {
|
||||
identifier(value, 'requestId');
|
||||
}
|
||||
|
||||
export function normalizeApprovalRequestCursor(
|
||||
value: ApprovalRequestCursor,
|
||||
): Readonly<ApprovalRequestCursor> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new InvalidApprovalDiscoveryValueError('cursor is invalid');
|
||||
}
|
||||
const keys = Object.keys(value);
|
||||
if (
|
||||
keys.length !== 2 ||
|
||||
!keys.includes('updatedAtMs') ||
|
||||
!keys.includes('requestId')
|
||||
) {
|
||||
throw new InvalidApprovalDiscoveryValueError('cursor shape is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
updatedAtMs: timestamp(value.updatedAtMs, 'cursor updatedAtMs'),
|
||||
requestId: identifier(value.requestId, 'cursor requestId'),
|
||||
});
|
||||
}
|
||||
|
||||
function boundedText(value: unknown, maximumBytes: number, name: string): string {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length === 0 ||
|
||||
CONTROL_PATTERN.test(value) ||
|
||||
Buffer.byteLength(value, 'utf8') > maximumBytes
|
||||
) {
|
||||
throw new InvalidApprovalDiscoveryValueError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function normalizeApprovalDetailPreview(
|
||||
value: ApprovalDetailPreview,
|
||||
): Readonly<ApprovalDetailPreview> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new InvalidApprovalDiscoveryValueError('preview is invalid');
|
||||
}
|
||||
const keys = Reflect.ownKeys(value);
|
||||
if (
|
||||
keys.length !== 4 ||
|
||||
!['title', 'summary', 'fields', 'warnings'].every((key) =>
|
||||
Object.hasOwn(value, key),
|
||||
) ||
|
||||
!Array.isArray(value.fields) ||
|
||||
value.fields.length > 16 ||
|
||||
!Array.isArray(value.warnings) ||
|
||||
value.warnings.length > 8
|
||||
) {
|
||||
throw new InvalidApprovalDiscoveryValueError('preview shape is invalid');
|
||||
}
|
||||
const fields = value.fields.map((field) => {
|
||||
if (
|
||||
!field ||
|
||||
typeof field !== 'object' ||
|
||||
Array.isArray(field) ||
|
||||
Reflect.ownKeys(field).length !== 3 ||
|
||||
!Object.hasOwn(field, 'kind') ||
|
||||
!Object.hasOwn(field, 'label') ||
|
||||
!Object.hasOwn(field, 'value') ||
|
||||
!PREVIEW_FIELD_KINDS.includes(field.kind) ||
|
||||
(field.kind === 'redacted' && field.value !== null) ||
|
||||
(field.kind !== 'redacted' && field.value === null)
|
||||
) {
|
||||
throw new InvalidApprovalDiscoveryValueError('preview field is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
kind: field.kind,
|
||||
label: boundedText(field.label, 128, 'preview field label'),
|
||||
value:
|
||||
field.value === null
|
||||
? null
|
||||
: boundedText(field.value, 512, 'preview field value'),
|
||||
});
|
||||
});
|
||||
const warnings = value.warnings.map((warning) => {
|
||||
if (typeof warning !== 'string' || !WARNING_PATTERN.test(warning)) {
|
||||
throw new InvalidApprovalDiscoveryValueError('preview warning is invalid');
|
||||
}
|
||||
return warning;
|
||||
});
|
||||
if (new Set(warnings).size !== warnings.length) {
|
||||
throw new InvalidApprovalDiscoveryValueError('preview warnings are duplicated');
|
||||
}
|
||||
const normalized = Object.freeze({
|
||||
title: boundedText(value.title, 256, 'preview title'),
|
||||
summary: boundedText(value.summary, 2048, 'preview summary'),
|
||||
fields: Object.freeze(fields),
|
||||
warnings: Object.freeze([...warnings].sort()),
|
||||
});
|
||||
if (
|
||||
Buffer.byteLength(JSON.stringify(normalized), 'utf8') >
|
||||
MAX_APPROVAL_DETAIL_PREVIEW_BYTES
|
||||
) {
|
||||
throw new InvalidApprovalDiscoveryValueError('preview byte budget exceeded');
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function approvalRequestUpdatedAtMs(
|
||||
request: Readonly<ApprovalRequestRecord>,
|
||||
): number {
|
||||
return timestamp(
|
||||
request.consumedAtMs ?? request.decidedAtMs ?? request.requestedAtMs,
|
||||
'request updatedAtMs',
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
import { normalizeApprovalRequestRecord } from './approvedAction';
|
||||
import {
|
||||
normalizeApprovalDetailPreview,
|
||||
type ApprovalRequestDetail,
|
||||
type ApprovalRequestDetailSource,
|
||||
} from './approvalDiscovery';
|
||||
import { ProjectPolicyEngine } from '../security/project-policy/projectPolicy';
|
||||
import {
|
||||
normalizeSecurityPrincipal,
|
||||
type SecurityPolicyFence,
|
||||
type SecurityPrincipal,
|
||||
} from '../security/security';
|
||||
import type { SecurityAuditSink } from '../security/audit/securityAudit';
|
||||
|
||||
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const UUID_V4_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
||||
const STRONG_HUMAN_ASSURANCES = new Set([
|
||||
'multi_factor',
|
||||
'hardware',
|
||||
'local_console',
|
||||
]);
|
||||
|
||||
export interface ApprovalInspectionRequest {
|
||||
readonly projectId: string;
|
||||
readonly approvalRequestId: string;
|
||||
readonly auditEventId: string;
|
||||
readonly requestId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
}
|
||||
|
||||
export interface ApprovalInspectionService {
|
||||
inspect(
|
||||
request: ApprovalInspectionRequest,
|
||||
): Promise<Readonly<ApprovalRequestDetail> | null>;
|
||||
}
|
||||
|
||||
export interface ApprovalInspectionServiceOptions {
|
||||
readonly source: Pick<ApprovalRequestDetailSource, 'getApprovalRequestDetail'>;
|
||||
readonly policy: Pick<ProjectPolicyEngine, 'authorize'>;
|
||||
readonly audit: Pick<SecurityAuditSink, 'record'>;
|
||||
readonly confirmAuthorization?: () => void | Promise<void>;
|
||||
readonly now?: () => number;
|
||||
}
|
||||
|
||||
export class ApprovalInspectionRequestError extends TypeError {
|
||||
readonly code = 'APPROVAL_INSPECTION_REQUEST_INVALID';
|
||||
|
||||
constructor(message: string) {
|
||||
super(`Approval inspection request is invalid: ${message}`);
|
||||
this.name = 'ApprovalInspectionRequestError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ApprovalInspectionAuthorizationError extends Error {
|
||||
readonly code = 'APPROVAL_INSPECTION_AUTHORIZATION_REJECTED';
|
||||
|
||||
constructor() {
|
||||
super('Approval inspection authorization was rejected');
|
||||
this.name = 'ApprovalInspectionAuthorizationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ApprovalInspectionUnavailableError extends Error {
|
||||
readonly code = 'APPROVAL_INSPECTION_UNAVAILABLE';
|
||||
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('Approval inspection authority is unavailable', options);
|
||||
this.name = 'ApprovalInspectionUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function exactObject(
|
||||
value: unknown,
|
||||
expected: readonly string[],
|
||||
label: string,
|
||||
): asserts value is Record<string, unknown> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
Object.keys(value).sort().join('\0') !== [...expected].sort().join('\0')
|
||||
) {
|
||||
throw new ApprovalInspectionRequestError(`${label} shape is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
function identifier(value: unknown, label: string): string {
|
||||
if (typeof value !== 'string' || !IDENTIFIER_PATTERN.test(value)) {
|
||||
throw new ApprovalInspectionRequestError(`${label} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function sameFence(
|
||||
left: Readonly<SecurityPolicyFence>,
|
||||
right: Readonly<SecurityPolicyFence>,
|
||||
): boolean {
|
||||
return (
|
||||
left.projectVersion === right.projectVersion &&
|
||||
left.bindingVersion === right.bindingVersion
|
||||
);
|
||||
}
|
||||
|
||||
function observedTime(now: () => number): number {
|
||||
const value = now();
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new ApprovalInspectionUnavailableError();
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function normalizedRequest(
|
||||
value: ApprovalInspectionRequest,
|
||||
): Readonly<ApprovalInspectionRequest> {
|
||||
exactObject(
|
||||
value,
|
||||
[
|
||||
'projectId',
|
||||
'approvalRequestId',
|
||||
'auditEventId',
|
||||
'requestId',
|
||||
'principal',
|
||||
],
|
||||
'request',
|
||||
);
|
||||
if (
|
||||
typeof value.auditEventId !== 'string' ||
|
||||
!UUID_V4_PATTERN.test(value.auditEventId)
|
||||
) {
|
||||
throw new ApprovalInspectionRequestError('auditEventId is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
projectId: identifier(value.projectId, 'projectId'),
|
||||
approvalRequestId: identifier(
|
||||
value.approvalRequestId,
|
||||
'approvalRequestId',
|
||||
),
|
||||
auditEventId: value.auditEventId,
|
||||
requestId: identifier(value.requestId, 'requestId'),
|
||||
principal: value.principal,
|
||||
});
|
||||
}
|
||||
|
||||
export function createApprovalInspectionService(
|
||||
options: ApprovalInspectionServiceOptions,
|
||||
): Readonly<ApprovalInspectionService> {
|
||||
exactObject(
|
||||
options,
|
||||
[
|
||||
'source',
|
||||
'policy',
|
||||
'audit',
|
||||
...(options?.confirmAuthorization === undefined
|
||||
? []
|
||||
: ['confirmAuthorization']),
|
||||
...(options?.now === undefined ? [] : ['now']),
|
||||
],
|
||||
'options',
|
||||
);
|
||||
if (
|
||||
typeof options.source?.getApprovalRequestDetail !== 'function' ||
|
||||
typeof options.policy?.authorize !== 'function' ||
|
||||
typeof options.audit?.record !== 'function' ||
|
||||
(options.confirmAuthorization !== undefined &&
|
||||
typeof options.confirmAuthorization !== 'function') ||
|
||||
(options.now !== undefined && typeof options.now !== 'function')
|
||||
) {
|
||||
throw new ApprovalInspectionRequestError('options are invalid');
|
||||
}
|
||||
const now = options.now ?? Date.now;
|
||||
return Object.freeze({
|
||||
async inspect(requestValue: ApprovalInspectionRequest) {
|
||||
const request = normalizedRequest(requestValue);
|
||||
const occurredAtMs = observedTime(now);
|
||||
let principal: Readonly<SecurityPrincipal>;
|
||||
try {
|
||||
principal = normalizeSecurityPrincipal(request.principal, occurredAtMs);
|
||||
} catch {
|
||||
throw new ApprovalInspectionAuthorizationError();
|
||||
}
|
||||
if (
|
||||
principal.subject.type !== 'user' ||
|
||||
!STRONG_HUMAN_ASSURANCES.has(principal.assurance)
|
||||
) {
|
||||
throw new ApprovalInspectionAuthorizationError();
|
||||
}
|
||||
let approvalRead;
|
||||
let artifactRead;
|
||||
try {
|
||||
[approvalRead, artifactRead] = await Promise.all([
|
||||
options.policy.authorize(
|
||||
principal,
|
||||
request.projectId,
|
||||
'approval.read',
|
||||
),
|
||||
options.policy.authorize(
|
||||
principal,
|
||||
request.projectId,
|
||||
'artifact.read',
|
||||
),
|
||||
]);
|
||||
} catch (error) {
|
||||
throw new ApprovalInspectionUnavailableError({
|
||||
cause: error instanceof Error ? error : undefined,
|
||||
});
|
||||
}
|
||||
if (
|
||||
approvalRead.effect !== 'allow' ||
|
||||
approvalRead.fence === null ||
|
||||
artifactRead.effect !== 'allow' ||
|
||||
artifactRead.fence === null ||
|
||||
!sameFence(approvalRead.fence, artifactRead.fence)
|
||||
) {
|
||||
throw new ApprovalInspectionAuthorizationError();
|
||||
}
|
||||
await options.confirmAuthorization?.();
|
||||
let detail: Readonly<ApprovalRequestDetail> | null;
|
||||
try {
|
||||
const found = await options.source.getApprovalRequestDetail({
|
||||
projectId: request.projectId,
|
||||
requestId: request.approvalRequestId,
|
||||
});
|
||||
if (!found) {
|
||||
detail = null;
|
||||
} else {
|
||||
const approval = normalizeApprovalRequestRecord(found.request);
|
||||
if (
|
||||
approval.projectId !== request.projectId ||
|
||||
approval.id !== request.approvalRequestId
|
||||
) {
|
||||
throw new ApprovalInspectionUnavailableError();
|
||||
}
|
||||
detail = Object.freeze({
|
||||
request: approval,
|
||||
preview: found.preview
|
||||
? normalizeApprovalDetailPreview(found.preview)
|
||||
: null,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof ApprovalInspectionUnavailableError) throw error;
|
||||
throw new ApprovalInspectionUnavailableError({
|
||||
cause: error instanceof Error ? error : undefined,
|
||||
});
|
||||
}
|
||||
try {
|
||||
await options.audit.record({
|
||||
eventId: request.auditEventId,
|
||||
requestId: request.requestId,
|
||||
operationId: 'approval.inspect',
|
||||
projectId: request.projectId,
|
||||
subject: principal.subject,
|
||||
authenticationId: principal.authenticationId,
|
||||
outcome: 'allowed',
|
||||
reasons: Object.freeze(['human_approval_inspection']),
|
||||
fence: approvalRead.fence,
|
||||
occurredAtMs,
|
||||
});
|
||||
} catch (error) {
|
||||
throw new ApprovalInspectionUnavailableError({
|
||||
cause: error instanceof Error ? error : undefined,
|
||||
});
|
||||
}
|
||||
return detail;
|
||||
},
|
||||
});
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,563 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
import {
|
||||
approvedActionExecutionEffectiveStatus,
|
||||
type ApprovedActionExecutionCursor,
|
||||
type ApprovedActionExecutionRecord,
|
||||
type ApprovedActionExecutionRepository,
|
||||
type ApprovedActionExecutionSnapshot,
|
||||
} from './approvedActionExecution';
|
||||
import type { ApprovedActionDispatchRecord } from './approvedAction';
|
||||
|
||||
const DEFAULT_LEASE_DURATION_MS = 30_000;
|
||||
const DEFAULT_RETRY_BASE_MS = 1_000;
|
||||
const DEFAULT_RETRY_MAX_MS = 60_000;
|
||||
const DEFAULT_BATCH_SIZE = 4;
|
||||
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const RESULT_CODE_PATTERN = /^[a-z][a-z0-9_]{0,63}$/;
|
||||
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
|
||||
|
||||
export type ApprovedActionHandlerInspection =
|
||||
| Readonly<{ status: 'ready'; actionDigest: string }>
|
||||
| Readonly<{ status: 'retry' | 'blocked'; resultCode: string }>;
|
||||
|
||||
export interface ApprovedActionHandlerExecutionContext {
|
||||
readonly dispatch: Readonly<ApprovedActionDispatchRecord>;
|
||||
readonly execution: Readonly<ApprovedActionExecutionRecord>;
|
||||
readonly idempotencyKey: string;
|
||||
readonly fence: Readonly<{
|
||||
owner: string;
|
||||
leaseToken: string;
|
||||
version: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface ApprovedActionHandlerResult {
|
||||
readonly outcome: 'succeeded' | 'failed' | 'indeterminate';
|
||||
readonly resultCode: string;
|
||||
readonly resultDigest?: string;
|
||||
}
|
||||
|
||||
export interface ApprovedActionHandler {
|
||||
readonly actionType: string;
|
||||
inspect(
|
||||
dispatch: Readonly<ApprovedActionDispatchRecord>,
|
||||
): Promise<ApprovedActionHandlerInspection>;
|
||||
execute(
|
||||
context: Readonly<ApprovedActionHandlerExecutionContext>,
|
||||
): Promise<Readonly<ApprovedActionHandlerResult>>;
|
||||
}
|
||||
|
||||
export interface ApprovedActionDispatcherOptions {
|
||||
readonly owner: string;
|
||||
readonly leaseDurationMs?: number;
|
||||
readonly retryBaseMs?: number;
|
||||
readonly retryMaxMs?: number;
|
||||
readonly defaultBatchSize?: number;
|
||||
readonly clock?: () => number;
|
||||
readonly createId?: () => string;
|
||||
}
|
||||
|
||||
export interface ApprovedActionDispatchBatchSummary {
|
||||
readonly scanned: number;
|
||||
readonly claimed: number;
|
||||
readonly started: number;
|
||||
readonly succeeded: number;
|
||||
readonly failed: number;
|
||||
readonly blocked: number;
|
||||
readonly retrying: number;
|
||||
readonly deferred: number;
|
||||
readonly recoveryRequired: number;
|
||||
readonly alreadyTerminal: number;
|
||||
readonly unavailable: number;
|
||||
readonly truncated: boolean;
|
||||
readonly nextCursor?: Readonly<ApprovedActionExecutionCursor>;
|
||||
}
|
||||
|
||||
interface MutableSummary {
|
||||
scanned: number;
|
||||
claimed: number;
|
||||
started: number;
|
||||
succeeded: number;
|
||||
failed: number;
|
||||
blocked: number;
|
||||
retrying: number;
|
||||
deferred: number;
|
||||
recoveryRequired: number;
|
||||
alreadyTerminal: number;
|
||||
unavailable: number;
|
||||
truncated: boolean;
|
||||
nextCursor?: Readonly<ApprovedActionExecutionCursor>;
|
||||
}
|
||||
|
||||
function positiveInteger(
|
||||
value: number,
|
||||
label: string,
|
||||
maximum = Number.MAX_SAFE_INTEGER,
|
||||
): number {
|
||||
if (!Number.isSafeInteger(value) || value < 1 || value > maximum) {
|
||||
throw new RangeError(`${label} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function identifier(value: string, label: string): string {
|
||||
if (typeof value !== 'string' || !IDENTIFIER_PATTERN.test(value)) {
|
||||
throw new TypeError(`${label} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function resultCode(value: string): string {
|
||||
if (typeof value !== 'string' || !RESULT_CODE_PATTERN.test(value)) {
|
||||
throw new TypeError('Approved Action result code is invalid');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function exactKeys(
|
||||
value: object,
|
||||
expected: readonly string[],
|
||||
optional: readonly string[] = [],
|
||||
): boolean {
|
||||
const required = new Set(expected);
|
||||
const allowed = new Set([...expected, ...optional]);
|
||||
const actual = Object.keys(value);
|
||||
return (
|
||||
expected.every((key) => Object.hasOwn(value, key)) &&
|
||||
actual.every((key) => allowed.has(key)) &&
|
||||
[...required].length === expected.length
|
||||
);
|
||||
}
|
||||
|
||||
function sameFence(
|
||||
snapshot: Readonly<ApprovedActionExecutionSnapshot>,
|
||||
owner: string,
|
||||
leaseToken: string,
|
||||
version: number,
|
||||
): boolean {
|
||||
return (
|
||||
snapshot.execution.status === 'executing' &&
|
||||
snapshot.execution.leaseOwner === owner &&
|
||||
snapshot.execution.leaseToken === leaseToken &&
|
||||
snapshot.execution.version === version
|
||||
);
|
||||
}
|
||||
|
||||
export class ApprovedActionDispatcher {
|
||||
readonly #handlers = new Map<string, ApprovedActionHandler>();
|
||||
readonly #owner: string;
|
||||
readonly #leaseDurationMs: number;
|
||||
readonly #retryBaseMs: number;
|
||||
readonly #retryMaxMs: number;
|
||||
readonly #defaultBatchSize: number;
|
||||
readonly #clock: () => number;
|
||||
readonly #createId: () => string;
|
||||
|
||||
constructor(
|
||||
readonly repository: ApprovedActionExecutionRepository,
|
||||
handlers: readonly ApprovedActionHandler[],
|
||||
options: ApprovedActionDispatcherOptions,
|
||||
) {
|
||||
if (
|
||||
!repository ||
|
||||
typeof repository.listDueExecutions !== 'function' ||
|
||||
typeof repository.claimExecution !== 'function' ||
|
||||
typeof repository.startExecution !== 'function' ||
|
||||
typeof repository.releaseExecutionBeforeStart !== 'function' ||
|
||||
typeof repository.completeExecution !== 'function' ||
|
||||
typeof repository.findExecutionByDispatchId !== 'function'
|
||||
) {
|
||||
throw new TypeError('Approved Action execution repository is invalid');
|
||||
}
|
||||
if (!Array.isArray(handlers) || !options || typeof options !== 'object') {
|
||||
throw new TypeError('Approved Action dispatcher options are invalid');
|
||||
}
|
||||
this.#owner = identifier(options.owner, 'dispatcher owner');
|
||||
this.#leaseDurationMs = positiveInteger(
|
||||
options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS,
|
||||
'lease duration',
|
||||
10 * 60 * 1000,
|
||||
);
|
||||
this.#retryBaseMs = positiveInteger(
|
||||
options.retryBaseMs ?? DEFAULT_RETRY_BASE_MS,
|
||||
'retry base',
|
||||
);
|
||||
this.#retryMaxMs = positiveInteger(
|
||||
options.retryMaxMs ?? DEFAULT_RETRY_MAX_MS,
|
||||
'retry maximum',
|
||||
);
|
||||
if (this.#retryMaxMs < this.#retryBaseMs) {
|
||||
throw new RangeError('retry maximum precedes retry base');
|
||||
}
|
||||
this.#defaultBatchSize = positiveInteger(
|
||||
options.defaultBatchSize ?? DEFAULT_BATCH_SIZE,
|
||||
'default batch size',
|
||||
64,
|
||||
);
|
||||
this.#clock = options.clock ?? Date.now;
|
||||
this.#createId = options.createId ?? randomUUID;
|
||||
for (const handler of handlers) {
|
||||
if (
|
||||
!handler ||
|
||||
typeof handler !== 'object' ||
|
||||
typeof handler.actionType !== 'string' ||
|
||||
handler.actionType.length < 1 ||
|
||||
handler.actionType.length > 128 ||
|
||||
typeof handler.inspect !== 'function' ||
|
||||
typeof handler.execute !== 'function'
|
||||
) {
|
||||
throw new TypeError('Approved Action handler is invalid');
|
||||
}
|
||||
if (this.#handlers.has(handler.actionType)) {
|
||||
throw new TypeError('Approved Action handler is duplicated');
|
||||
}
|
||||
this.#handlers.set(handler.actionType, handler);
|
||||
}
|
||||
}
|
||||
|
||||
async dispatchBatch(
|
||||
options: Readonly<{
|
||||
cursor?: ApprovedActionExecutionCursor;
|
||||
limit?: number;
|
||||
}> = {},
|
||||
): Promise<Readonly<ApprovedActionDispatchBatchSummary>> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
!exactKeys(options, [], ['cursor', 'limit'])
|
||||
) {
|
||||
throw new TypeError('Approved Action dispatch batch is invalid');
|
||||
}
|
||||
const limit = positiveInteger(
|
||||
options.limit ?? this.#defaultBatchSize,
|
||||
'batch size',
|
||||
64,
|
||||
);
|
||||
const page = await this.repository.listDueExecutions({
|
||||
nowMs: this.#now(),
|
||||
limit,
|
||||
actionTypes: Object.freeze([...this.#handlers.keys()].sort()),
|
||||
...(options.cursor ? { cursor: options.cursor } : {}),
|
||||
});
|
||||
const summary: MutableSummary = {
|
||||
scanned: page.executions.length,
|
||||
claimed: 0,
|
||||
started: 0,
|
||||
succeeded: 0,
|
||||
failed: 0,
|
||||
blocked: 0,
|
||||
retrying: 0,
|
||||
deferred: 0,
|
||||
recoveryRequired: 0,
|
||||
alreadyTerminal: 0,
|
||||
unavailable: 0,
|
||||
truncated: page.truncated,
|
||||
...(page.nextCursor ? { nextCursor: page.nextCursor } : {}),
|
||||
};
|
||||
for (const candidate of page.executions) {
|
||||
await this.#dispatchOne(candidate.dispatch.id, summary);
|
||||
}
|
||||
return Object.freeze({ ...summary });
|
||||
}
|
||||
|
||||
async #dispatchOne(
|
||||
dispatchId: string,
|
||||
summary: MutableSummary,
|
||||
): Promise<void> {
|
||||
const leaseToken = this.#id('lease token');
|
||||
let claim;
|
||||
try {
|
||||
claim = await this.repository.claimExecution({
|
||||
dispatchId,
|
||||
owner: this.#owner,
|
||||
leaseToken,
|
||||
nowMs: this.#now(),
|
||||
leaseDurationMs: this.#leaseDurationMs,
|
||||
});
|
||||
} catch {
|
||||
summary.unavailable += 1;
|
||||
return;
|
||||
}
|
||||
if (claim.status === 'not_found') {
|
||||
summary.unavailable += 1;
|
||||
return;
|
||||
}
|
||||
if (claim.status !== 'claimed') {
|
||||
if (claim.status === 'recovery_required') {
|
||||
summary.recoveryRequired += 1;
|
||||
} else if (
|
||||
claim.status === 'succeeded' ||
|
||||
claim.status === 'failed' ||
|
||||
claim.status === 'blocked'
|
||||
) {
|
||||
summary.alreadyTerminal += 1;
|
||||
} else {
|
||||
summary.deferred += 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
summary.claimed += 1;
|
||||
const handler = this.#handlers.get(
|
||||
claim.snapshot.dispatch.action.actionType,
|
||||
);
|
||||
if (!handler) {
|
||||
await this.#release(
|
||||
claim.snapshot.execution,
|
||||
'handler_unavailable',
|
||||
false,
|
||||
summary,
|
||||
);
|
||||
return;
|
||||
}
|
||||
let inspection: ApprovedActionHandlerInspection;
|
||||
try {
|
||||
inspection = await handler.inspect(claim.snapshot.dispatch);
|
||||
this.#assertInspection(inspection);
|
||||
} catch {
|
||||
await this.#release(
|
||||
claim.snapshot.execution,
|
||||
'handler_inspection_failed',
|
||||
true,
|
||||
summary,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (inspection.status !== 'ready') {
|
||||
await this.#release(
|
||||
claim.snapshot.execution,
|
||||
inspection.resultCode,
|
||||
inspection.status === 'retry',
|
||||
summary,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
inspection.actionDigest !== claim.snapshot.dispatch.action.actionDigest
|
||||
) {
|
||||
await this.#release(
|
||||
claim.snapshot.execution,
|
||||
'action_digest_mismatch',
|
||||
false,
|
||||
summary,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let started: Readonly<ApprovedActionExecutionSnapshot>;
|
||||
try {
|
||||
started = await this.repository.startExecution({
|
||||
dispatchId,
|
||||
approvalRequestId: claim.snapshot.dispatch.approvalRequestId,
|
||||
actionDigest: inspection.actionDigest,
|
||||
owner: this.#owner,
|
||||
leaseToken,
|
||||
expectedVersion: claim.snapshot.execution.version,
|
||||
startedAtMs: this.#now(),
|
||||
});
|
||||
} catch {
|
||||
const converged = await this.#find(dispatchId);
|
||||
if (
|
||||
!converged ||
|
||||
!sameFence(
|
||||
converged,
|
||||
this.#owner,
|
||||
leaseToken,
|
||||
claim.snapshot.execution.version + 1,
|
||||
)
|
||||
) {
|
||||
summary.unavailable += 1;
|
||||
if (
|
||||
converged?.execution.status === 'executing' ||
|
||||
(converged &&
|
||||
approvedActionExecutionEffectiveStatus(
|
||||
converged.execution,
|
||||
this.#now(),
|
||||
) === 'recovery_required')
|
||||
) {
|
||||
summary.recoveryRequired += 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
started = converged;
|
||||
}
|
||||
summary.started += 1;
|
||||
|
||||
let result: Readonly<ApprovedActionHandlerResult>;
|
||||
try {
|
||||
result = await handler.execute(
|
||||
Object.freeze({
|
||||
dispatch: started.dispatch,
|
||||
execution: started.execution,
|
||||
idempotencyKey: started.dispatch.id,
|
||||
fence: Object.freeze({
|
||||
owner: this.#owner,
|
||||
leaseToken,
|
||||
version: started.execution.version,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
this.#assertResult(result);
|
||||
} catch {
|
||||
result = Object.freeze({
|
||||
outcome: 'indeterminate',
|
||||
resultCode: 'handler_failed_after_start',
|
||||
});
|
||||
}
|
||||
|
||||
const resultMutationId = this.#id('result mutation id');
|
||||
try {
|
||||
const completed = await this.repository.completeExecution({
|
||||
dispatchId,
|
||||
owner: this.#owner,
|
||||
leaseToken,
|
||||
expectedVersion: started.execution.version,
|
||||
resultMutationId,
|
||||
outcome: result.outcome,
|
||||
resultCode: result.resultCode,
|
||||
...(result.resultDigest === undefined
|
||||
? {}
|
||||
: { resultDigest: result.resultDigest }),
|
||||
completedAtMs: this.#now(),
|
||||
});
|
||||
this.#countCompletion(completed.execution, summary);
|
||||
} catch {
|
||||
const converged = await this.#find(dispatchId);
|
||||
if (
|
||||
converged &&
|
||||
converged.execution.resultMutationId === resultMutationId &&
|
||||
converged.execution.resultCode === result.resultCode &&
|
||||
converged.execution.resultDigest === (result.resultDigest ?? null)
|
||||
) {
|
||||
this.#countCompletion(converged.execution, summary);
|
||||
return;
|
||||
}
|
||||
summary.unavailable += 1;
|
||||
summary.recoveryRequired += 1;
|
||||
}
|
||||
}
|
||||
|
||||
async #release(
|
||||
execution: Readonly<ApprovedActionExecutionRecord>,
|
||||
code: string,
|
||||
retry: boolean,
|
||||
summary: MutableSummary,
|
||||
): Promise<void> {
|
||||
const atMs = this.#now();
|
||||
try {
|
||||
const released = await this.repository.releaseExecutionBeforeStart({
|
||||
dispatchId: execution.dispatchId,
|
||||
owner: this.#owner,
|
||||
leaseToken: execution.leaseToken!,
|
||||
expectedVersion: execution.version,
|
||||
resultMutationId: this.#id('result mutation id'),
|
||||
resultCode: resultCode(code),
|
||||
atMs,
|
||||
...(retry
|
||||
? { retryAtMs: this.#nextRetryAt(atMs, execution.attemptCount) }
|
||||
: {}),
|
||||
});
|
||||
if (released.execution.status === 'retry_wait') summary.retrying += 1;
|
||||
else summary.blocked += 1;
|
||||
} catch {
|
||||
summary.unavailable += 1;
|
||||
}
|
||||
}
|
||||
|
||||
async #find(
|
||||
dispatchId: string,
|
||||
): Promise<Readonly<ApprovedActionExecutionSnapshot> | null> {
|
||||
try {
|
||||
return await this.repository.findExecutionByDispatchId(dispatchId);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
#countCompletion(
|
||||
execution: Readonly<ApprovedActionExecutionRecord>,
|
||||
summary: MutableSummary,
|
||||
): void {
|
||||
if (execution.status === 'succeeded') summary.succeeded += 1;
|
||||
else if (execution.status === 'failed') summary.failed += 1;
|
||||
else summary.blocked += 1;
|
||||
}
|
||||
|
||||
#assertInspection(
|
||||
value: unknown,
|
||||
): asserts value is ApprovedActionHandlerInspection {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new TypeError('Approved Action inspection is invalid');
|
||||
}
|
||||
if (
|
||||
'status' in value &&
|
||||
value.status === 'ready' &&
|
||||
exactKeys(value, ['status', 'actionDigest']) &&
|
||||
'actionDigest' in value &&
|
||||
typeof value.actionDigest === 'string' &&
|
||||
DIGEST_PATTERN.test(value.actionDigest)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
'status' in value &&
|
||||
(value.status === 'retry' || value.status === 'blocked') &&
|
||||
exactKeys(value, ['status', 'resultCode']) &&
|
||||
'resultCode' in value &&
|
||||
typeof value.resultCode === 'string'
|
||||
) {
|
||||
resultCode(value.resultCode);
|
||||
return;
|
||||
}
|
||||
throw new TypeError('Approved Action inspection is invalid');
|
||||
}
|
||||
|
||||
#assertResult(value: unknown): asserts value is ApprovedActionHandlerResult {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
!exactKeys(value, ['outcome', 'resultCode'], ['resultDigest']) ||
|
||||
!('outcome' in value) ||
|
||||
!['succeeded', 'failed', 'indeterminate'].includes(
|
||||
value.outcome as string,
|
||||
) ||
|
||||
!('resultCode' in value) ||
|
||||
typeof value.resultCode !== 'string'
|
||||
) {
|
||||
throw new TypeError('Approved Action result is invalid');
|
||||
}
|
||||
resultCode(value.resultCode);
|
||||
const digest =
|
||||
'resultDigest' in value ? (value.resultDigest as unknown) : undefined;
|
||||
if (
|
||||
(value.outcome === 'succeeded' &&
|
||||
(typeof digest !== 'string' || !DIGEST_PATTERN.test(digest))) ||
|
||||
(value.outcome !== 'succeeded' && digest !== undefined)
|
||||
) {
|
||||
throw new TypeError('Approved Action result digest is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
#nextRetryAt(atMs: number, attemptCount: number): number {
|
||||
const exponent = Math.max(0, Math.min(attemptCount - 1, 30));
|
||||
const delay = Math.min(
|
||||
this.#retryMaxMs,
|
||||
this.#retryBaseMs * 2 ** exponent,
|
||||
);
|
||||
return Math.min(Number.MAX_SAFE_INTEGER, atMs + delay);
|
||||
}
|
||||
|
||||
#now(): number {
|
||||
const nowMs = this.#clock();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new RangeError('Approved Action clock is invalid');
|
||||
}
|
||||
return nowMs;
|
||||
}
|
||||
|
||||
#id(label: string): string {
|
||||
return identifier(this.#createId(), label);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,958 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
import {
|
||||
approvedActionDispatchDigest,
|
||||
normalizeApprovedActionDispatchRecord,
|
||||
type ApprovedActionDispatchRecord,
|
||||
} from './approvedAction';
|
||||
|
||||
export const APPROVED_ACTION_EXECUTION_SCHEMA =
|
||||
'qinglong/approved-action-execution@v1' as const;
|
||||
export const APPROVED_ACTION_EXECUTION_STATUSES = [
|
||||
'pending',
|
||||
'leased',
|
||||
'executing',
|
||||
'retry_wait',
|
||||
'succeeded',
|
||||
'failed',
|
||||
'blocked',
|
||||
] as const;
|
||||
|
||||
export type ApprovedActionExecutionStatus =
|
||||
(typeof APPROVED_ACTION_EXECUTION_STATUSES)[number];
|
||||
export type ApprovedActionExecutionEffectiveStatus =
|
||||
| ApprovedActionExecutionStatus
|
||||
| 'recovery_required';
|
||||
|
||||
export interface ApprovedActionExecutionRecord {
|
||||
readonly schema: typeof APPROVED_ACTION_EXECUTION_SCHEMA;
|
||||
readonly dispatchId: string;
|
||||
readonly dispatchDigest: string;
|
||||
readonly projectId: string;
|
||||
readonly status: ApprovedActionExecutionStatus;
|
||||
readonly version: number;
|
||||
readonly attemptCount: number;
|
||||
readonly maxAttempts: number;
|
||||
readonly eligibleAtMs: number | null;
|
||||
readonly nextAttemptAtMs: number | null;
|
||||
readonly leaseOwner: string | null;
|
||||
readonly leaseToken: string | null;
|
||||
readonly leaseExpiresAtMs: number | null;
|
||||
readonly startedAtMs: number | null;
|
||||
readonly resultMutationId: string | null;
|
||||
readonly resultCode: string | null;
|
||||
readonly resultDigest: string | null;
|
||||
readonly completedAtMs: number | null;
|
||||
readonly createdAtMs: number;
|
||||
readonly updatedAtMs: number;
|
||||
readonly executionDigest: string;
|
||||
}
|
||||
|
||||
export interface ApprovedActionExecutionSnapshot {
|
||||
readonly dispatch: Readonly<ApprovedActionDispatchRecord>;
|
||||
readonly execution: Readonly<ApprovedActionExecutionRecord>;
|
||||
}
|
||||
|
||||
export interface ApprovedActionExecutionCursor {
|
||||
readonly eligibleAtMs: number;
|
||||
readonly dispatchId: string;
|
||||
}
|
||||
|
||||
export interface ListDueApprovedActionExecutionsQuery {
|
||||
readonly nowMs: number;
|
||||
readonly limit: number;
|
||||
readonly actionTypes: readonly string[];
|
||||
readonly cursor?: ApprovedActionExecutionCursor;
|
||||
}
|
||||
|
||||
export interface ListDueApprovedActionExecutionsResult {
|
||||
readonly executions: readonly Readonly<ApprovedActionExecutionSnapshot>[];
|
||||
readonly truncated: boolean;
|
||||
readonly nextCursor?: Readonly<ApprovedActionExecutionCursor>;
|
||||
}
|
||||
|
||||
export interface ClaimApprovedActionExecutionCommand {
|
||||
readonly dispatchId: string;
|
||||
readonly owner: string;
|
||||
readonly leaseToken: string;
|
||||
readonly nowMs: number;
|
||||
readonly leaseDurationMs: number;
|
||||
}
|
||||
|
||||
export type ClaimApprovedActionExecutionResult =
|
||||
| Readonly<{
|
||||
status: 'claimed';
|
||||
snapshot: Readonly<ApprovedActionExecutionSnapshot>;
|
||||
}>
|
||||
| Readonly<{ status: 'not_found' }>
|
||||
| Readonly<{
|
||||
status:
|
||||
| 'not_due'
|
||||
| 'leased'
|
||||
| 'executing'
|
||||
| 'recovery_required'
|
||||
| 'succeeded'
|
||||
| 'failed'
|
||||
| 'blocked';
|
||||
snapshot: Readonly<ApprovedActionExecutionSnapshot>;
|
||||
}>;
|
||||
|
||||
export interface StartApprovedActionExecutionCommand {
|
||||
readonly dispatchId: string;
|
||||
readonly approvalRequestId: string;
|
||||
readonly actionDigest: string;
|
||||
readonly owner: string;
|
||||
readonly leaseToken: string;
|
||||
readonly expectedVersion: number;
|
||||
readonly startedAtMs: number;
|
||||
}
|
||||
|
||||
export interface RenewApprovedActionExecutionCommand {
|
||||
readonly dispatchId: string;
|
||||
readonly owner: string;
|
||||
readonly leaseToken: string;
|
||||
readonly expectedVersion: number;
|
||||
readonly nowMs: number;
|
||||
readonly leaseDurationMs: number;
|
||||
}
|
||||
|
||||
export interface ReleaseApprovedActionExecutionBeforeStartCommand {
|
||||
readonly dispatchId: string;
|
||||
readonly owner: string;
|
||||
readonly leaseToken: string;
|
||||
readonly expectedVersion: number;
|
||||
readonly resultMutationId: string;
|
||||
readonly resultCode: string;
|
||||
readonly atMs: number;
|
||||
readonly retryAtMs?: number;
|
||||
}
|
||||
|
||||
export interface CompleteApprovedActionExecutionCommand {
|
||||
readonly dispatchId: string;
|
||||
readonly owner: string;
|
||||
readonly leaseToken: string;
|
||||
readonly expectedVersion: number;
|
||||
readonly resultMutationId: string;
|
||||
readonly outcome: 'succeeded' | 'failed' | 'indeterminate';
|
||||
readonly resultCode: string;
|
||||
readonly resultDigest?: string;
|
||||
readonly completedAtMs: number;
|
||||
}
|
||||
|
||||
export interface ApprovedActionExecutionRepository {
|
||||
findExecutionByDispatchId(
|
||||
dispatchId: string,
|
||||
): Promise<Readonly<ApprovedActionExecutionSnapshot> | null>;
|
||||
listDueExecutions(
|
||||
query: ListDueApprovedActionExecutionsQuery,
|
||||
): Promise<ListDueApprovedActionExecutionsResult>;
|
||||
claimExecution(
|
||||
command: ClaimApprovedActionExecutionCommand,
|
||||
): Promise<ClaimApprovedActionExecutionResult>;
|
||||
startExecution(
|
||||
command: StartApprovedActionExecutionCommand,
|
||||
): Promise<Readonly<ApprovedActionExecutionSnapshot>>;
|
||||
renewExecution(
|
||||
command: RenewApprovedActionExecutionCommand,
|
||||
): Promise<Readonly<ApprovedActionExecutionSnapshot>>;
|
||||
releaseExecutionBeforeStart(
|
||||
command: ReleaseApprovedActionExecutionBeforeStartCommand,
|
||||
): Promise<Readonly<ApprovedActionExecutionSnapshot>>;
|
||||
completeExecution(
|
||||
command: CompleteApprovedActionExecutionCommand,
|
||||
): Promise<Readonly<ApprovedActionExecutionSnapshot>>;
|
||||
}
|
||||
|
||||
export const DEFAULT_APPROVED_ACTION_MAX_ATTEMPTS = 5;
|
||||
export const MAX_APPROVED_ACTION_ATTEMPTS = 16;
|
||||
export const MAX_APPROVED_ACTION_EXECUTION_VERSION = 2_147_483_647;
|
||||
export const MAX_APPROVED_ACTION_LEASE_ID_LENGTH = 128;
|
||||
export const MAX_APPROVED_ACTION_RESULT_CODE_LENGTH = 64;
|
||||
export const MAX_APPROVED_ACTION_EXECUTION_PAGE_SIZE = 64;
|
||||
export const MAX_APPROVED_ACTION_LEASE_DURATION_MS = 10 * 60 * 1000;
|
||||
|
||||
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const RESULT_CODE_PATTERN = /^[a-z][a-z0-9_]{0,63}$/;
|
||||
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
|
||||
|
||||
export class InvalidApprovedActionExecutionError extends TypeError {
|
||||
readonly code = 'APPROVED_ACTION_EXECUTION_INVALID';
|
||||
|
||||
constructor(message: string) {
|
||||
super(`Approved Action execution is invalid: ${message}`);
|
||||
this.name = 'InvalidApprovedActionExecutionError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ApprovedActionExecutionFenceConflictError extends Error {
|
||||
readonly code = 'APPROVED_ACTION_EXECUTION_FENCE_CONFLICT';
|
||||
|
||||
constructor() {
|
||||
super('Approved Action execution fence changed');
|
||||
this.name = 'ApprovedActionExecutionFenceConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ApprovedActionExecutionStateConflictError extends Error {
|
||||
readonly code = 'APPROVED_ACTION_EXECUTION_STATE_CONFLICT';
|
||||
|
||||
constructor() {
|
||||
super('Approved Action execution is not in the required state');
|
||||
this.name = 'ApprovedActionExecutionStateConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ApprovedActionExecutionBindingConflictError extends Error {
|
||||
readonly code = 'APPROVED_ACTION_EXECUTION_BINDING_CONFLICT';
|
||||
|
||||
constructor() {
|
||||
super('Approved Action execution does not match its dispatch');
|
||||
this.name = 'ApprovedActionExecutionBindingConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ApprovedActionExecutionUnavailableError extends Error {
|
||||
readonly code = 'APPROVED_ACTION_EXECUTION_UNAVAILABLE';
|
||||
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('Approved Action execution authority is unavailable', options);
|
||||
this.name = 'ApprovedActionExecutionUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function dataRecord(value: unknown, label: string): Record<string, unknown> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
(Object.getPrototypeOf(value) !== Object.prototype &&
|
||||
Object.getPrototypeOf(value) !== null)
|
||||
) {
|
||||
throw new InvalidApprovedActionExecutionError(`${label} must be an object`);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function exactKeys(
|
||||
value: object,
|
||||
expected: readonly string[],
|
||||
optional: readonly string[],
|
||||
label: string,
|
||||
): void {
|
||||
const actual = Object.keys(value).sort();
|
||||
const required = [...expected].sort();
|
||||
const allowed = new Set([...expected, ...optional]);
|
||||
if (
|
||||
required.some((key) => !Object.hasOwn(value, key)) ||
|
||||
actual.some((key) => !allowed.has(key))
|
||||
) {
|
||||
throw new InvalidApprovedActionExecutionError(`${label} shape is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
function identifier(value: unknown, label: string): string {
|
||||
if (typeof value !== 'string' || !IDENTIFIER_PATTERN.test(value)) {
|
||||
throw new InvalidApprovedActionExecutionError(`${label} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function resultCode(value: unknown): string {
|
||||
if (typeof value !== 'string' || !RESULT_CODE_PATTERN.test(value)) {
|
||||
throw new InvalidApprovedActionExecutionError('result code is invalid');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function digest(value: unknown, label: string): string {
|
||||
if (typeof value !== 'string' || !DIGEST_PATTERN.test(value)) {
|
||||
throw new InvalidApprovedActionExecutionError(`${label} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function integer(
|
||||
value: unknown,
|
||||
label: string,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): number {
|
||||
if (
|
||||
!Number.isSafeInteger(value) ||
|
||||
(value as number) < minimum ||
|
||||
(value as number) > maximum
|
||||
) {
|
||||
throw new InvalidApprovedActionExecutionError(`${label} is invalid`);
|
||||
}
|
||||
return value as number;
|
||||
}
|
||||
|
||||
function timestamp(value: unknown, label: string): number {
|
||||
return integer(value, label, 0, Number.MAX_SAFE_INTEGER);
|
||||
}
|
||||
|
||||
function nullableTimestamp(value: unknown, label: string): number | null {
|
||||
return value === null ? null : timestamp(value, label);
|
||||
}
|
||||
|
||||
function nullableIdentifier(value: unknown, label: string): string | null {
|
||||
return value === null ? null : identifier(value, label);
|
||||
}
|
||||
|
||||
function nullableResultCode(value: unknown): string | null {
|
||||
return value === null ? null : resultCode(value);
|
||||
}
|
||||
|
||||
function nullableDigest(value: unknown, label: string): string | null {
|
||||
return value === null ? null : digest(value, label);
|
||||
}
|
||||
|
||||
function contractDigest(domain: string, value: unknown): string {
|
||||
return createHash('sha256')
|
||||
.update(domain)
|
||||
.update('\0')
|
||||
.update(JSON.stringify(value))
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
function withoutExecutionDigest(
|
||||
value: Omit<ApprovedActionExecutionRecord, 'executionDigest'>,
|
||||
): Omit<ApprovedActionExecutionRecord, 'executionDigest'> {
|
||||
return Object.freeze(value);
|
||||
}
|
||||
|
||||
function withExecutionDigest(
|
||||
value: Omit<ApprovedActionExecutionRecord, 'executionDigest'>,
|
||||
): Readonly<ApprovedActionExecutionRecord> {
|
||||
const normalized = withoutExecutionDigest(value);
|
||||
return Object.freeze({
|
||||
...normalized,
|
||||
executionDigest: contractDigest(
|
||||
'qinglong/approved-action-execution-digest@v1',
|
||||
normalized,
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
function executionWithoutDigest(
|
||||
value: Readonly<ApprovedActionExecutionRecord>,
|
||||
): Omit<ApprovedActionExecutionRecord, 'executionDigest'> {
|
||||
const { executionDigest: _executionDigest, ...record } = value;
|
||||
return record;
|
||||
}
|
||||
|
||||
function sameLease(
|
||||
record: Readonly<ApprovedActionExecutionRecord>,
|
||||
owner: string,
|
||||
leaseToken: string,
|
||||
expectedVersion: number,
|
||||
): boolean {
|
||||
return (
|
||||
record.leaseOwner === owner &&
|
||||
record.leaseToken === leaseToken &&
|
||||
record.version === expectedVersion
|
||||
);
|
||||
}
|
||||
|
||||
export function createApprovedActionExecution(
|
||||
dispatchValue: ApprovedActionDispatchRecord,
|
||||
maxAttempts = DEFAULT_APPROVED_ACTION_MAX_ATTEMPTS,
|
||||
): Readonly<ApprovedActionExecutionRecord> {
|
||||
const dispatch = normalizeApprovedActionDispatchRecord(dispatchValue);
|
||||
return withExecutionDigest({
|
||||
schema: APPROVED_ACTION_EXECUTION_SCHEMA,
|
||||
dispatchId: dispatch.id,
|
||||
dispatchDigest: approvedActionDispatchDigest(dispatch),
|
||||
projectId: dispatch.projectId,
|
||||
status: 'pending',
|
||||
version: 0,
|
||||
attemptCount: 0,
|
||||
maxAttempts: integer(
|
||||
maxAttempts,
|
||||
'maximum attempts',
|
||||
1,
|
||||
MAX_APPROVED_ACTION_ATTEMPTS,
|
||||
),
|
||||
eligibleAtMs: dispatch.createdAtMs,
|
||||
nextAttemptAtMs: null,
|
||||
leaseOwner: null,
|
||||
leaseToken: null,
|
||||
leaseExpiresAtMs: null,
|
||||
startedAtMs: null,
|
||||
resultMutationId: null,
|
||||
resultCode: null,
|
||||
resultDigest: null,
|
||||
completedAtMs: null,
|
||||
createdAtMs: dispatch.createdAtMs,
|
||||
updatedAtMs: dispatch.createdAtMs,
|
||||
});
|
||||
}
|
||||
|
||||
export function normalizeApprovedActionExecutionRecord(
|
||||
value: ApprovedActionExecutionRecord,
|
||||
): Readonly<ApprovedActionExecutionRecord> {
|
||||
const record = dataRecord(value, 'record');
|
||||
exactKeys(
|
||||
record,
|
||||
[
|
||||
'schema',
|
||||
'dispatchId',
|
||||
'dispatchDigest',
|
||||
'projectId',
|
||||
'status',
|
||||
'version',
|
||||
'attemptCount',
|
||||
'maxAttempts',
|
||||
'eligibleAtMs',
|
||||
'nextAttemptAtMs',
|
||||
'leaseOwner',
|
||||
'leaseToken',
|
||||
'leaseExpiresAtMs',
|
||||
'startedAtMs',
|
||||
'resultMutationId',
|
||||
'resultCode',
|
||||
'resultDigest',
|
||||
'completedAtMs',
|
||||
'createdAtMs',
|
||||
'updatedAtMs',
|
||||
'executionDigest',
|
||||
],
|
||||
[],
|
||||
'record',
|
||||
);
|
||||
if (
|
||||
value.schema !== APPROVED_ACTION_EXECUTION_SCHEMA ||
|
||||
!APPROVED_ACTION_EXECUTION_STATUSES.includes(value.status)
|
||||
) {
|
||||
throw new InvalidApprovedActionExecutionError(
|
||||
'schema or status is invalid',
|
||||
);
|
||||
}
|
||||
const normalized = {
|
||||
schema: APPROVED_ACTION_EXECUTION_SCHEMA,
|
||||
dispatchId: identifier(value.dispatchId, 'dispatch id'),
|
||||
dispatchDigest: digest(value.dispatchDigest, 'dispatch digest'),
|
||||
projectId: identifier(value.projectId, 'project id'),
|
||||
status: value.status,
|
||||
version: integer(
|
||||
value.version,
|
||||
'version',
|
||||
0,
|
||||
MAX_APPROVED_ACTION_EXECUTION_VERSION,
|
||||
),
|
||||
attemptCount: integer(
|
||||
value.attemptCount,
|
||||
'attempt count',
|
||||
0,
|
||||
MAX_APPROVED_ACTION_ATTEMPTS,
|
||||
),
|
||||
maxAttempts: integer(
|
||||
value.maxAttempts,
|
||||
'maximum attempts',
|
||||
1,
|
||||
MAX_APPROVED_ACTION_ATTEMPTS,
|
||||
),
|
||||
eligibleAtMs: nullableTimestamp(value.eligibleAtMs, 'eligible time'),
|
||||
nextAttemptAtMs: nullableTimestamp(
|
||||
value.nextAttemptAtMs,
|
||||
'next attempt time',
|
||||
),
|
||||
leaseOwner: nullableIdentifier(value.leaseOwner, 'lease owner'),
|
||||
leaseToken: nullableIdentifier(value.leaseToken, 'lease token'),
|
||||
leaseExpiresAtMs: nullableTimestamp(
|
||||
value.leaseExpiresAtMs,
|
||||
'lease expiry',
|
||||
),
|
||||
startedAtMs: nullableTimestamp(value.startedAtMs, 'start time'),
|
||||
resultMutationId: nullableIdentifier(
|
||||
value.resultMutationId,
|
||||
'result mutation id',
|
||||
),
|
||||
resultCode: nullableResultCode(value.resultCode),
|
||||
resultDigest: nullableDigest(value.resultDigest, 'result digest'),
|
||||
completedAtMs: nullableTimestamp(value.completedAtMs, 'completion time'),
|
||||
createdAtMs: timestamp(value.createdAtMs, 'creation time'),
|
||||
updatedAtMs: timestamp(value.updatedAtMs, 'update time'),
|
||||
} satisfies Omit<ApprovedActionExecutionRecord, 'executionDigest'>;
|
||||
const executionDigest = digest(value.executionDigest, 'execution digest');
|
||||
if (
|
||||
normalized.attemptCount > normalized.maxAttempts ||
|
||||
normalized.updatedAtMs < normalized.createdAtMs
|
||||
) {
|
||||
throw new InvalidApprovedActionExecutionError(
|
||||
'attempt or timestamp range is invalid',
|
||||
);
|
||||
}
|
||||
|
||||
const leaseValues = [
|
||||
normalized.leaseOwner,
|
||||
normalized.leaseToken,
|
||||
normalized.leaseExpiresAtMs,
|
||||
];
|
||||
const hasLease = leaseValues.every((entry) => entry !== null);
|
||||
if (
|
||||
leaseValues.some((entry) => entry !== null) !== hasLease ||
|
||||
(hasLease &&
|
||||
(normalized.leaseExpiresAtMs! <= normalized.updatedAtMs ||
|
||||
normalized.leaseExpiresAtMs! - normalized.updatedAtMs >
|
||||
MAX_APPROVED_ACTION_LEASE_DURATION_MS))
|
||||
) {
|
||||
throw new InvalidApprovedActionExecutionError('lease tuple is invalid');
|
||||
}
|
||||
|
||||
const resultValues = [
|
||||
normalized.resultMutationId,
|
||||
normalized.resultCode,
|
||||
];
|
||||
const hasResult = resultValues.every((entry) => entry !== null);
|
||||
if (resultValues.some((entry) => entry !== null) !== hasResult) {
|
||||
throw new InvalidApprovedActionExecutionError('result tuple is invalid');
|
||||
}
|
||||
const terminal = ['succeeded', 'failed', 'blocked'].includes(
|
||||
normalized.status,
|
||||
);
|
||||
if (
|
||||
(normalized.status === 'pending' &&
|
||||
(normalized.version !== 0 ||
|
||||
normalized.attemptCount !== 0 ||
|
||||
normalized.eligibleAtMs === null ||
|
||||
normalized.nextAttemptAtMs !== null ||
|
||||
hasLease ||
|
||||
normalized.startedAtMs !== null ||
|
||||
hasResult ||
|
||||
normalized.resultDigest !== null ||
|
||||
normalized.completedAtMs !== null)) ||
|
||||
(normalized.status === 'leased' &&
|
||||
(!hasLease ||
|
||||
normalized.attemptCount < 1 ||
|
||||
normalized.eligibleAtMs !== normalized.leaseExpiresAtMs ||
|
||||
normalized.nextAttemptAtMs !== null ||
|
||||
normalized.startedAtMs !== null ||
|
||||
hasResult ||
|
||||
normalized.resultDigest !== null ||
|
||||
normalized.completedAtMs !== null)) ||
|
||||
(normalized.status === 'executing' &&
|
||||
(!hasLease ||
|
||||
normalized.attemptCount < 1 ||
|
||||
normalized.eligibleAtMs !== null ||
|
||||
normalized.nextAttemptAtMs !== null ||
|
||||
normalized.startedAtMs === null ||
|
||||
hasResult ||
|
||||
normalized.resultDigest !== null ||
|
||||
normalized.completedAtMs !== null)) ||
|
||||
(normalized.status === 'retry_wait' &&
|
||||
(hasLease ||
|
||||
normalized.attemptCount < 1 ||
|
||||
normalized.attemptCount >= normalized.maxAttempts ||
|
||||
normalized.eligibleAtMs === null ||
|
||||
normalized.eligibleAtMs !== normalized.nextAttemptAtMs ||
|
||||
normalized.startedAtMs !== null ||
|
||||
!hasResult ||
|
||||
normalized.resultDigest !== null ||
|
||||
normalized.completedAtMs !== null)) ||
|
||||
(terminal &&
|
||||
(hasLease ||
|
||||
normalized.eligibleAtMs !== null ||
|
||||
normalized.nextAttemptAtMs !== null ||
|
||||
!hasResult ||
|
||||
normalized.completedAtMs === null))
|
||||
) {
|
||||
throw new InvalidApprovedActionExecutionError(
|
||||
`${normalized.status} tuple is invalid`,
|
||||
);
|
||||
}
|
||||
if (
|
||||
normalized.status === 'succeeded' &&
|
||||
(normalized.startedAtMs === null || normalized.resultDigest === null)
|
||||
) {
|
||||
throw new InvalidApprovedActionExecutionError(
|
||||
'successful execution has no start barrier or result digest',
|
||||
);
|
||||
}
|
||||
if (
|
||||
normalized.status === 'failed' &&
|
||||
normalized.startedAtMs === null
|
||||
) {
|
||||
throw new InvalidApprovedActionExecutionError(
|
||||
'failed execution has no start barrier',
|
||||
);
|
||||
}
|
||||
if (
|
||||
normalized.startedAtMs !== null &&
|
||||
(normalized.startedAtMs < normalized.createdAtMs ||
|
||||
normalized.startedAtMs > normalized.updatedAtMs)
|
||||
) {
|
||||
throw new InvalidApprovedActionExecutionError(
|
||||
'start timestamp is invalid',
|
||||
);
|
||||
}
|
||||
if (
|
||||
normalized.completedAtMs !== null &&
|
||||
(normalized.completedAtMs <
|
||||
(normalized.startedAtMs ?? normalized.createdAtMs) ||
|
||||
normalized.completedAtMs !== normalized.updatedAtMs)
|
||||
) {
|
||||
throw new InvalidApprovedActionExecutionError(
|
||||
'completion timestamp is invalid',
|
||||
);
|
||||
}
|
||||
if (
|
||||
executionDigest !==
|
||||
contractDigest(
|
||||
'qinglong/approved-action-execution-digest@v1',
|
||||
normalized,
|
||||
)
|
||||
) {
|
||||
throw new InvalidApprovedActionExecutionError(
|
||||
'execution digest does not match the record',
|
||||
);
|
||||
}
|
||||
return Object.freeze({ ...normalized, executionDigest });
|
||||
}
|
||||
|
||||
export function normalizeApprovedActionExecutionSnapshot(
|
||||
value: ApprovedActionExecutionSnapshot,
|
||||
): Readonly<ApprovedActionExecutionSnapshot> {
|
||||
const snapshot = dataRecord(value, 'snapshot');
|
||||
exactKeys(snapshot, ['dispatch', 'execution'], [], 'snapshot');
|
||||
const dispatch = normalizeApprovedActionDispatchRecord(value.dispatch);
|
||||
const execution = normalizeApprovedActionExecutionRecord(value.execution);
|
||||
if (
|
||||
execution.dispatchId !== dispatch.id ||
|
||||
execution.dispatchDigest !== approvedActionDispatchDigest(dispatch) ||
|
||||
execution.projectId !== dispatch.projectId
|
||||
) {
|
||||
throw new ApprovedActionExecutionBindingConflictError();
|
||||
}
|
||||
return Object.freeze({ dispatch, execution });
|
||||
}
|
||||
|
||||
export function normalizeApprovedActionExecutionCursor(
|
||||
value: ApprovedActionExecutionCursor,
|
||||
): Readonly<ApprovedActionExecutionCursor> {
|
||||
const cursor = dataRecord(value, 'cursor');
|
||||
exactKeys(cursor, ['eligibleAtMs', 'dispatchId'], [], 'cursor');
|
||||
return Object.freeze({
|
||||
eligibleAtMs: timestamp(value.eligibleAtMs, 'cursor eligible time'),
|
||||
dispatchId: identifier(value.dispatchId, 'cursor dispatch id'),
|
||||
});
|
||||
}
|
||||
|
||||
export function approvedActionExecutionEffectiveStatus(
|
||||
recordValue: ApprovedActionExecutionRecord,
|
||||
nowMsValue: number,
|
||||
): ApprovedActionExecutionEffectiveStatus {
|
||||
const record = normalizeApprovedActionExecutionRecord(recordValue);
|
||||
const nowMs = timestamp(nowMsValue, 'observation time');
|
||||
if (
|
||||
record.status === 'executing' &&
|
||||
record.leaseExpiresAtMs !== null &&
|
||||
nowMs >= record.leaseExpiresAtMs
|
||||
) {
|
||||
return 'recovery_required';
|
||||
}
|
||||
return record.status;
|
||||
}
|
||||
|
||||
export function claimApprovedActionExecution(
|
||||
recordValue: ApprovedActionExecutionRecord,
|
||||
commandValue: Omit<ClaimApprovedActionExecutionCommand, 'dispatchId'>,
|
||||
): Readonly<ApprovedActionExecutionRecord> {
|
||||
const record = normalizeApprovedActionExecutionRecord(recordValue);
|
||||
const command = dataRecord(commandValue, 'claim command');
|
||||
exactKeys(
|
||||
command,
|
||||
['owner', 'leaseToken', 'nowMs', 'leaseDurationMs'],
|
||||
[],
|
||||
'claim command',
|
||||
);
|
||||
const owner = identifier(commandValue.owner, 'lease owner');
|
||||
const leaseToken = identifier(commandValue.leaseToken, 'lease token');
|
||||
const nowMs = timestamp(commandValue.nowMs, 'claim time');
|
||||
const leaseDurationMs = integer(
|
||||
commandValue.leaseDurationMs,
|
||||
'lease duration',
|
||||
1,
|
||||
MAX_APPROVED_ACTION_LEASE_DURATION_MS,
|
||||
);
|
||||
const due =
|
||||
(record.status === 'pending' || record.status === 'retry_wait') &&
|
||||
record.eligibleAtMs !== null &&
|
||||
record.eligibleAtMs <= nowMs;
|
||||
const reclaimable =
|
||||
record.status === 'leased' &&
|
||||
record.leaseExpiresAtMs !== null &&
|
||||
record.leaseExpiresAtMs <= nowMs;
|
||||
if (!due && !reclaimable) {
|
||||
throw new ApprovedActionExecutionStateConflictError();
|
||||
}
|
||||
if (record.attemptCount >= record.maxAttempts) {
|
||||
throw new ApprovedActionExecutionStateConflictError();
|
||||
}
|
||||
const leaseExpiresAtMs = Math.min(
|
||||
Number.MAX_SAFE_INTEGER,
|
||||
nowMs + leaseDurationMs,
|
||||
);
|
||||
return withExecutionDigest({
|
||||
...executionWithoutDigest(record),
|
||||
status: 'leased',
|
||||
version: record.version + 1,
|
||||
attemptCount: record.attemptCount + 1,
|
||||
eligibleAtMs: leaseExpiresAtMs,
|
||||
nextAttemptAtMs: null,
|
||||
leaseOwner: owner,
|
||||
leaseToken,
|
||||
leaseExpiresAtMs,
|
||||
startedAtMs: null,
|
||||
resultMutationId: null,
|
||||
resultCode: null,
|
||||
resultDigest: null,
|
||||
completedAtMs: null,
|
||||
updatedAtMs: nowMs,
|
||||
});
|
||||
}
|
||||
|
||||
export function startApprovedActionExecution(
|
||||
snapshotValue: ApprovedActionExecutionSnapshot,
|
||||
commandValue: StartApprovedActionExecutionCommand,
|
||||
): Readonly<ApprovedActionExecutionRecord> {
|
||||
const snapshot = normalizeApprovedActionExecutionSnapshot(snapshotValue);
|
||||
const command = dataRecord(commandValue, 'start command');
|
||||
exactKeys(
|
||||
command,
|
||||
[
|
||||
'dispatchId',
|
||||
'approvalRequestId',
|
||||
'actionDigest',
|
||||
'owner',
|
||||
'leaseToken',
|
||||
'expectedVersion',
|
||||
'startedAtMs',
|
||||
],
|
||||
[],
|
||||
'start command',
|
||||
);
|
||||
const dispatchId = identifier(commandValue.dispatchId, 'dispatch id');
|
||||
const approvalRequestId = identifier(
|
||||
commandValue.approvalRequestId,
|
||||
'approval request id',
|
||||
);
|
||||
const actionDigest = digest(commandValue.actionDigest, 'action digest');
|
||||
const owner = identifier(commandValue.owner, 'lease owner');
|
||||
const leaseToken = identifier(commandValue.leaseToken, 'lease token');
|
||||
const expectedVersion = integer(
|
||||
commandValue.expectedVersion,
|
||||
'expected version',
|
||||
0,
|
||||
MAX_APPROVED_ACTION_EXECUTION_VERSION,
|
||||
);
|
||||
const startedAtMs = timestamp(commandValue.startedAtMs, 'start time');
|
||||
if (
|
||||
snapshot.execution.status !== 'leased' ||
|
||||
!sameLease(snapshot.execution, owner, leaseToken, expectedVersion) ||
|
||||
dispatchId !== snapshot.dispatch.id ||
|
||||
approvalRequestId !== snapshot.dispatch.approvalRequestId ||
|
||||
actionDigest !== snapshot.dispatch.action.actionDigest ||
|
||||
snapshot.execution.leaseExpiresAtMs === null ||
|
||||
startedAtMs < snapshot.execution.updatedAtMs ||
|
||||
startedAtMs >= snapshot.execution.leaseExpiresAtMs
|
||||
) {
|
||||
throw new ApprovedActionExecutionFenceConflictError();
|
||||
}
|
||||
return withExecutionDigest({
|
||||
...executionWithoutDigest(snapshot.execution),
|
||||
status: 'executing',
|
||||
version: snapshot.execution.version + 1,
|
||||
eligibleAtMs: null,
|
||||
startedAtMs,
|
||||
updatedAtMs: startedAtMs,
|
||||
});
|
||||
}
|
||||
|
||||
export function renewApprovedActionExecution(
|
||||
recordValue: ApprovedActionExecutionRecord,
|
||||
commandValue: Omit<RenewApprovedActionExecutionCommand, 'dispatchId'>,
|
||||
): Readonly<ApprovedActionExecutionRecord> {
|
||||
const record = normalizeApprovedActionExecutionRecord(recordValue);
|
||||
const command = dataRecord(commandValue, 'renew command');
|
||||
exactKeys(
|
||||
command,
|
||||
['owner', 'leaseToken', 'expectedVersion', 'nowMs', 'leaseDurationMs'],
|
||||
[],
|
||||
'renew command',
|
||||
);
|
||||
const owner = identifier(commandValue.owner, 'lease owner');
|
||||
const leaseToken = identifier(commandValue.leaseToken, 'lease token');
|
||||
const expectedVersion = integer(
|
||||
commandValue.expectedVersion,
|
||||
'expected version',
|
||||
0,
|
||||
MAX_APPROVED_ACTION_EXECUTION_VERSION,
|
||||
);
|
||||
const nowMs = timestamp(commandValue.nowMs, 'renewal time');
|
||||
const leaseDurationMs = integer(
|
||||
commandValue.leaseDurationMs,
|
||||
'lease duration',
|
||||
1,
|
||||
MAX_APPROVED_ACTION_LEASE_DURATION_MS,
|
||||
);
|
||||
if (
|
||||
(record.status !== 'leased' && record.status !== 'executing') ||
|
||||
!sameLease(record, owner, leaseToken, expectedVersion) ||
|
||||
record.leaseExpiresAtMs === null ||
|
||||
nowMs < record.updatedAtMs ||
|
||||
nowMs >= record.leaseExpiresAtMs
|
||||
) {
|
||||
throw new ApprovedActionExecutionFenceConflictError();
|
||||
}
|
||||
const leaseExpiresAtMs = Math.min(
|
||||
Number.MAX_SAFE_INTEGER,
|
||||
nowMs + leaseDurationMs,
|
||||
);
|
||||
return withExecutionDigest({
|
||||
...executionWithoutDigest(record),
|
||||
version: record.version + 1,
|
||||
eligibleAtMs:
|
||||
record.status === 'leased' ? leaseExpiresAtMs : record.eligibleAtMs,
|
||||
leaseExpiresAtMs,
|
||||
updatedAtMs: nowMs,
|
||||
});
|
||||
}
|
||||
|
||||
export function releaseApprovedActionExecutionBeforeStart(
|
||||
recordValue: ApprovedActionExecutionRecord,
|
||||
commandValue: Omit<
|
||||
ReleaseApprovedActionExecutionBeforeStartCommand,
|
||||
'dispatchId'
|
||||
>,
|
||||
): Readonly<ApprovedActionExecutionRecord> {
|
||||
const record = normalizeApprovedActionExecutionRecord(recordValue);
|
||||
const command = dataRecord(commandValue, 'release command');
|
||||
exactKeys(
|
||||
command,
|
||||
[
|
||||
'owner',
|
||||
'leaseToken',
|
||||
'expectedVersion',
|
||||
'resultMutationId',
|
||||
'resultCode',
|
||||
'atMs',
|
||||
],
|
||||
['retryAtMs'],
|
||||
'release command',
|
||||
);
|
||||
const owner = identifier(commandValue.owner, 'lease owner');
|
||||
const leaseToken = identifier(commandValue.leaseToken, 'lease token');
|
||||
const expectedVersion = integer(
|
||||
commandValue.expectedVersion,
|
||||
'expected version',
|
||||
0,
|
||||
MAX_APPROVED_ACTION_EXECUTION_VERSION,
|
||||
);
|
||||
const resultMutationId = identifier(
|
||||
commandValue.resultMutationId,
|
||||
'result mutation id',
|
||||
);
|
||||
const normalizedResultCode = resultCode(commandValue.resultCode);
|
||||
const atMs = timestamp(commandValue.atMs, 'release time');
|
||||
const retryAtMs =
|
||||
commandValue.retryAtMs === undefined
|
||||
? undefined
|
||||
: timestamp(commandValue.retryAtMs, 'retry time');
|
||||
if (
|
||||
record.status !== 'leased' ||
|
||||
!sameLease(record, owner, leaseToken, expectedVersion) ||
|
||||
record.startedAtMs !== null ||
|
||||
atMs < record.updatedAtMs ||
|
||||
(retryAtMs !== undefined && retryAtMs <= atMs)
|
||||
) {
|
||||
throw new ApprovedActionExecutionFenceConflictError();
|
||||
}
|
||||
const retry =
|
||||
retryAtMs !== undefined && record.attemptCount < record.maxAttempts;
|
||||
return withExecutionDigest({
|
||||
...executionWithoutDigest(record),
|
||||
status: retry ? 'retry_wait' : 'blocked',
|
||||
version: record.version + 1,
|
||||
eligibleAtMs: retry ? retryAtMs! : null,
|
||||
nextAttemptAtMs: retry ? retryAtMs! : null,
|
||||
leaseOwner: null,
|
||||
leaseToken: null,
|
||||
leaseExpiresAtMs: null,
|
||||
resultMutationId,
|
||||
resultCode: normalizedResultCode,
|
||||
resultDigest: null,
|
||||
completedAtMs: retry ? null : atMs,
|
||||
updatedAtMs: atMs,
|
||||
});
|
||||
}
|
||||
|
||||
export function completeApprovedActionExecution(
|
||||
recordValue: ApprovedActionExecutionRecord,
|
||||
commandValue: Omit<CompleteApprovedActionExecutionCommand, 'dispatchId'>,
|
||||
): Readonly<ApprovedActionExecutionRecord> {
|
||||
const record = normalizeApprovedActionExecutionRecord(recordValue);
|
||||
const command = dataRecord(commandValue, 'complete command');
|
||||
exactKeys(
|
||||
command,
|
||||
[
|
||||
'owner',
|
||||
'leaseToken',
|
||||
'expectedVersion',
|
||||
'resultMutationId',
|
||||
'outcome',
|
||||
'resultCode',
|
||||
'completedAtMs',
|
||||
],
|
||||
['resultDigest'],
|
||||
'complete command',
|
||||
);
|
||||
const owner = identifier(commandValue.owner, 'lease owner');
|
||||
const leaseToken = identifier(commandValue.leaseToken, 'lease token');
|
||||
const expectedVersion = integer(
|
||||
commandValue.expectedVersion,
|
||||
'expected version',
|
||||
0,
|
||||
MAX_APPROVED_ACTION_EXECUTION_VERSION,
|
||||
);
|
||||
const resultMutationId = identifier(
|
||||
commandValue.resultMutationId,
|
||||
'result mutation id',
|
||||
);
|
||||
const normalizedResultCode = resultCode(commandValue.resultCode);
|
||||
const completedAtMs = timestamp(commandValue.completedAtMs, 'completion time');
|
||||
const resultDigestValue =
|
||||
commandValue.resultDigest === undefined
|
||||
? null
|
||||
: digest(commandValue.resultDigest, 'result digest');
|
||||
if (
|
||||
!['succeeded', 'failed', 'indeterminate'].includes(commandValue.outcome) ||
|
||||
record.status !== 'executing' ||
|
||||
!sameLease(record, owner, leaseToken, expectedVersion) ||
|
||||
record.startedAtMs === null ||
|
||||
completedAtMs < record.updatedAtMs ||
|
||||
(commandValue.outcome === 'succeeded' && resultDigestValue === null) ||
|
||||
(commandValue.outcome !== 'succeeded' && resultDigestValue !== null)
|
||||
) {
|
||||
throw new ApprovedActionExecutionFenceConflictError();
|
||||
}
|
||||
const status =
|
||||
commandValue.outcome === 'indeterminate'
|
||||
? 'blocked'
|
||||
: commandValue.outcome;
|
||||
return withExecutionDigest({
|
||||
...executionWithoutDigest(record),
|
||||
status,
|
||||
version: record.version + 1,
|
||||
eligibleAtMs: null,
|
||||
nextAttemptAtMs: null,
|
||||
leaseOwner: null,
|
||||
leaseToken: null,
|
||||
leaseExpiresAtMs: null,
|
||||
resultMutationId,
|
||||
resultCode: normalizedResultCode,
|
||||
resultDigest: resultDigestValue,
|
||||
completedAtMs,
|
||||
updatedAtMs: completedAtMs,
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user